agent-evals 0.1.1

data/lib/agent/evals/cli/model_command.rb

@@ -0,0 +1,393 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "pathname"
+
+module Agent
+  module Evals
+    module CLI
+      # The options shared by the commands that call a model. Settings resolve
+      # flag first, then the config entry, then .agent-eval.config.yml's defaults.
+      #
+      # Every column that is not a metric is the gem's; the metric columns,
+      # their glossary and the ranking come from the suite's {Scorer#metrics}.
+      class ModelCommand < Command
+        # How close to the best primary metric a model must come to compete on
+        # price: 0.02 is two percentage points.
+        VALUE_MARGIN = 0.02
+
+        # What the gem's own leaderboard columns mean, printed under it with the metrics'.
+        GLOSSARY = {
+          "effort"  => "Reasoning effort sent to the model. \"default\" sends none, and the model decides; " \
+                       "a provider that takes none shows \"default\" too.",
+          "temp"    => "Sampling temperature sent to the model. \"default\" sends none.",
+          "errors"  => "Cases whose call failed outright (an API or network error), so they have no scores at all. " \
+                       "A reply the model got wrong is not an error; it scores 0.",
+          "time"    => "Wall-clock time for the experiment. Experiments run side by side and share the network, " \
+                       "so compare times within one run only.",
+          "tok_in"  => "Input tokens sent, summed over the cases, as the provider counted them.",
+          "tok_out" => "Output tokens received, summed over the cases; reasoning tokens are included where the " \
+                       "provider bills them as output.",
+          "cost"    => "US dollars, as the provider reported them, summed over the cases.",
+          "score"   => "The mean of each case's overall score, as the scorer computed it. The last table ranks " \
+                       "every experiment by it."
+        }.freeze
+
+        def self.inherited(subclass)
+          super
+          subclass.extend(Dry::CLI::UI::Flags)
+          subclass.flags(:dry_run, :output)
+          subclass.option :models,
+            type:    :array,
+            aliases: ["-m"],
+            default: [],
+            desc:    "Model ids, comma-separated, instead of the config file's vendors"
+          subclass.option :dataset,
+            type:    :array,
+            aliases: ["-d"],
+            default: [],
+            desc:    "Datasets, comma-separated; every dataset when omitted. Each model runs on each"
+          subclass.option :config, desc: "The lineup file (default: the suite's .agent-eval..agent-eval.config.yml)"
+          subclass.option :effort, aliases: ["-e"], desc: "Reasoning effort: default, or one the provider accepts (overrides the config file)"
+          subclass.option :temperature, aliases: ["-t"], desc: "Sampling temperature, or 'default' to send none (overrides the config file)"
+          subclass.option :max_tokens, desc: "Output token cap, reasoning included (overrides the config file)"
+          subclass.option :case,
+            type:    :array,
+            aliases: ["-c"],
+            default: [],
+            desc:    "Only these cases, by id prefix: -c clear or -c clear/003,vague/001"
+          subclass.option :parallelism, aliases: ["-p"], desc: "Cases in flight at once, per model (default: the config's defaults.parallelism)"
+          subclass.option :project,
+            desc: "Braintrust project name; default is the BRAINTRUST_PROJECT_ID project, else the config file's project"
+          subclass.option :open,
+            type:    :boolean,
+            default: true,
+            desc:    "Open the best experiment in the browser when done (no effect with --local)"
+          subclass.option :local,
+            type:    :boolean,
+            default: false,
+            desc:    "No remote tracker: results go to log/ only (still calls the models)"
+          subclass.option :tracker,
+            desc: "The remote tracker: braintrust, logfire, none, or one the suite registers " \
+                  "(default: the config's tracker:, else the one whose key is in the environment)"
+        end
+
+        private
+
+        # @return [Config] the lineup: --config, else the suite's
+        def lineup(options) = options[:config] ? Config.new(File.expand_path(options[:config])) : configuration.lineup
+
+        # @return [Providers::Registry]
+        def providers = configuration.providers
+
+        # @return [Scorer]
+        def scorer = configuration.scorer
+
+        # @return [Array<Metric>]
+        def metrics = scorer.metrics
+
+        # @return [Hash{String => Object}] the setting flags that were given
+        def overrides(options)
+          { "effort" => options[:effort], "temperature" => options[:temperature], "max_tokens" => options[:max_tokens] }
+        end
+
+        def experiment(contender, config, options, dataset:)
+          Experiment.new(
+            model:       providers.model_for(contender),
+            cases:       cases_for(options[:case], dataset: dataset),
+            adapter:     configuration.adapter,
+            scorer:      scorer,
+            parallelism: Integer(options[:parallelism] || config.defaults["parallelism"]),
+            trackers:    trackers_for(config, options)
+          )
+        end
+
+        # The local JSON log, always, then the one remote tracker if any,
+        # each experiment getting instances of its own.
+        #
+        # @return [Array<Tracker>]
+        def trackers_for(config, options)
+          context = { suite:      configuration.name,
+                      project:    options[:project] || config.project || configuration.project || configuration.name,
+                      project_id: options[:project] ? nil : ENV["BRAINTRUST_PROJECT_ID"].to_s[/\S+/] }
+          local = Trackers::Local.new(dir: File.join(Dry::CLI::UI::Invocation.root(Dir.pwd), "log", "agent-evals"), **context)
+          klass = remote_tracker(config, options)
+          remote = klass&.new(**context, options: config.tracker_options(klass.tracker_name))
+          [local, remote].compact
+        end
+
+        # The remote tracker class this run uses, chosen once: --tracker,
+        # then tracker: in the config, then the environment. --local is none.
+        #
+        # @return [Class<Tracker>, nil]
+        def remote_tracker(config, options)
+          return @remote_tracker if defined?(@remote_tracker)
+
+          chosen = configuration.trackers.choose(flag: options[:tracker], config: config.tracker, env: ENV, local: options[:local])
+          @remote_tracker = chosen&.first
+          err.puts "tracker: #{chosen ? "#{chosen.first.tracker_name}, because #{chosen.last}" : "none"}" if options[:verbose]
+          @remote_tracker
+        end
+
+        # The contenders a command runs: the models named on the command line,
+        # or every model .agent-eval.config.yml lists.
+        #
+        # @param models [Array<String>] as given to -m, possibly comma-separated
+        # @return [Array<Contender>]
+        # @raise [Error] when neither names any model
+        def contenders(config, models, options)
+          models = models.flat_map { it.split(",") }
+          found = models.empty? ? config.contenders(overrides(options)) : config.contenders_for(models, overrides(options))
+          raise Error, "no models: list some under vendors in #{config.path}, or pass -m" if found.empty?
+
+          admit(found, config)
+        end
+
+        # Keeps the contenders whose provider produces what the suite declares
+        # in `request_types`, and records each refusal in {#warnings} by name,
+        # so a refused model is reported rather than run, failed and billed.
+        #
+        # @return [Array<Contender>] the admitted contenders
+        # @raise [Error] when none is admitted, naming every refusal
+        def admit(found, config)
+          found.each { providers.for(it) } # an unserved id fails here, before anything is spent
+          admitted, refusals = providers.admit(found, request_types: config.request_types)
+          refusals.each { warnings.record(it.to_s) }
+          raise Error, "no admitted models: #{refusals.join("; ")}" if admitted.empty?
+
+          admitted
+        end
+
+        # Env, the tracker's gems and key, hooks, provider clients, then the catalogs.
+        def setup!(contenders, options)
+          configuration.runtime.setup!(providers: providers.used_by(contenders), tracker: remote_tracker(lineup(options), options))
+          providers.verify!(contenders)
+        end
+
+        # What -n prints: every experiment that would run, and where the
+        # refusals went, if there were any.
+        def matrix(datasets, contenders)
+          rows = datasets.product(contenders).map { |name, c| [name, c.model, c.effort, c.temperature, c.max_tokens] }
+          table(rows, header: %w[dataset model effort temp max_tokens])
+          save_warnings
+        end
+
+        # Runs the experiments side by side, a progress bar each. What would
+        # print as a warning meanwhile, failed cases included, goes to
+        # {#warnings} instead.
+        #
+        # @param runs [Array<Experiment>]
+        # @param concurrency [Integer] experiments at once; 0 runs them all together
+        # @return [Array<Experiment::Summary>]
+        def execute(runs, concurrency: 0)
+          summaries = Array.new(runs.size)
+          warnings.capture { bars(runs, summaries, concurrency) }
+          summaries.compact.each { |s| s.errors.each { warnings.record("[#{s.name}] #{it}") } }
+          summaries.compact
+        end
+
+        # @return [Warnings] what this command has held back from the screen
+        def warnings = @warnings ||= Warnings.new
+
+        def bars(runs, summaries, concurrency)
+          ProgressBars.for(ui).run("Running #{runs.size} experiment#{"s" unless runs.size == 1}",
+            concurrent: concurrency.zero? || concurrency) do |multi|
+            runs.each_with_index do |run, index|
+              multi.progress(run.name, total: run.cases.size) do |bar|
+                summaries[index] = run.run(on_case: -> { bar.advance })
+              end
+            end
+          end
+        end
+
+        # @param names [Array<String>] as given to -d, possibly comma-separated; empty means every dataset
+        # @return [Array<String>]
+        # @raise [Error] on an unknown dataset, before anything is spent
+        def datasets(names)
+          names = names.flat_map { it.split(",") }.uniq
+          return discovery.datasets.tap { raise Error, "no datasets under #{discovery.root}" if it.empty? } if names.empty?
+
+          names.each { discovery.cases(dataset: it) }
+        end
+
+        # Prints what the block writes, then the leaderboard, its glossary and
+        # the scores; then says where the warnings went, if any, and opens the
+        # best-scoring experiment.
+        #
+        # With -o, the block's output goes to the file alone, and the rest to
+        # both, each drawn for its own stream. The file is in color and ends
+        # with the time it was closed, as dry-cli-ui's own reports do.
+        #
+        # @param summaries [Array<Experiment::Summary>]
+        # @param output [String, nil] what -o was given: nil, "-", "" for the default file, or a path
+        # @param open [Boolean]
+        # @return [void]
+        def report(summaries, output:, open:, &details)
+          path = invocation.report_path(output)
+          if path
+            File.open(path, "w") do |file|
+              report = Dry::CLI::UI::Report.new(file)
+              writing_to(report) do
+                details&.call
+                leaderboard(summaries)
+              end
+              report.puts "Closed at #{Time.now.strftime("%Y-%m-%d %H:%M:%S %z")}"
+            end
+          else
+            details&.call
+          end
+          leaderboard(summaries)
+          status_box(:info, "Report written to #{relative(path)}", stream: err) if path
+          save_warnings
+          url = summaries.max_by { it.score || -1 }&.permalink
+          status_box(:warning, "could not open #{url}", stream: err) if open && url && !configuration.runtime.browse(url)
+        end
+
+        # Saves {#warnings}, if there are any, to
+        # log/<command>.warnings.<YYYY-MM-DD>.<HHMMSS>.log at the repository
+        # root, stamped with the time the process started, and says how many.
+        #
+        # @return [void]
+        def save_warnings
+          return if warnings.none?
+
+          stamp = Dry::CLI::UI.started_at.strftime("%Y-%m-%d.%H%M%S")
+          path = File.join(Dry::CLI::UI::Invocation.root(Dir.pwd), "log", "#{invocation.action}.warnings.#{stamp}.log")
+          FileUtils.mkdir_p(File.dirname(path))
+          warnings.save(path)
+          count = warnings.count
+          status_box(:warning, "#{count} #{count == 1 ? "Warning was" : "Warnings were"} recorded in #{relative(path)}", stream: err)
+        end
+
+        # @return [Dry::CLI::UI::Invocation] how the command was run, which names its files
+        def invocation = ui.send(:invocation)
+
+        def relative(path) = Pathname(path).relative_path_from(Dir.pwd).to_s
+
+        # Points the tables and boxes at another stream while the block runs.
+        # They keep the screen's width.
+        #
+        # @param io [IO, nil] nil to leave them on the screen
+        def writing_to(io)
+          return yield if io.nil?
+
+          screen = out
+          @screen = screen
+          @out = io
+          @pastel = nil
+          yield
+        ensure
+          if io
+            @out = screen
+            @screen = nil
+            @pastel = nil
+          end
+        end
+
+        def percent(value) = value.nil? ? "-" : format("%.1f%%", value * 100)
+
+        def dollars(value) = format("$%.4f", value.to_f)
+
+        def primary = scorer.primary
+
+        # The best value: the cheapest entry within {VALUE_MARGIN} of the best
+        # primary metric. Entries with errors, or no score, do not compete. A
+        # plain score-per-dollar ratio would crown whatever is nearly free,
+        # however wrong it is.
+        #
+        # @param entries [Array<Array(Float, Float, Integer)>] primary metric, cost, errors
+        # @return [Integer, nil] the winner's index
+        def best_value(entries)
+          eligible = entries.each_index.select { entries[it][0] && entries[it][2].zero? }
+          return nil if eligible.empty?
+
+          floor = eligible.map { entries[it][0] }.max - VALUE_MARGIN
+          eligible.select { entries[it][0] >= floor - 1e-9 }.min_by { entries[it][1] }
+        end
+
+        def value_line(label, value, cost)
+          out.puts "best value#{label}: #{format("%.1f%%", value * 100)} at #{dollars(cost)}, " \
+                   "the cheapest within #{(VALUE_MARGIN * 100).round} points of the best #{primary.name}"
+        end
+
+        def contender_label(summary) = "#{summary.model.id} effort=#{summary.model.effort || "default"}"
+
+        def temperature_cell(summary) = summary.model.temperature.nil? ? "default" : summary.model.temperature
+
+        # Tokens, cost and the mean primary metric per contender, over every
+        # dataset it ran.
+        def totals(summaries)
+          groups = summaries.group_by { [it.model.key, it.model.effort, it.model.temperature] }.values.map do |group|
+            values = group.filter_map { it.mean(primary) }
+            { first: group.first, errors: group.sum { it.errors.size },
+              value: values.size == group.size ? values.sum / values.size : nil,
+              usage: group.map(&:usage).reduce(Usage.zero, :+) }
+          end
+          groups.sort_by! { -(it[:value] || -1) }
+          rows = groups.map do |g|
+            s = g[:first]
+            [s.model.id, s.model.effort || "default", temperature_cell(s), percent(g[:value]),
+             g[:errors], g[:usage].input_tokens, g[:usage].output_tokens, dollars(g[:usage].cost_usd)]
+          end
+          best = best_value(groups.map { [it[:value], it[:usage].cost_usd, it[:errors]] })
+          table(rows,
+            header:    ["model", "effort", "temp", primary.name, "errors", "tok_in", "tok_out", "cost"],
+            highlight: [best].compact,
+            fill:      0)
+          return unless best
+
+          g = groups[best]
+          value_line(" overall (#{contender_label(g[:first])})", g[:value], g[:usage].cost_usd)
+        end
+
+        # Every experiment by its mean score, best first, with its cost and its
+        # Braintrust page. URLs are never wrapped, so they stay clickable.
+        def scores(summaries)
+          rows = summaries.sort_by { -(it.score || -1) }.map do |s|
+            [s.model.id, s.model.effort || "default", s.dataset, percent(s.score), dollars(s.usage.cost_usd), s.permalink || "-"]
+          end
+          table(rows, header: %w[model effort dataset score cost url], fill: 0)
+        end
+
+        # The best value in each dataset's block of the leaderboard.
+        #
+        # @param ranked [Array<Experiment::Summary>]
+        # @return [Array<Integer>] indices into ranked
+        def best_per_dataset(ranked)
+          ranked.each_index.group_by { ranked[it].dataset }.values.filter_map do |indices|
+            winner = best_value(indices.map { [ranked[it].mean(primary), ranked[it].usage.cost_usd, ranked[it].errors.size] })
+            winner && indices[winner]
+          end
+        end
+
+        # One row per experiment, best primary metric first, with the best
+        # value per dataset; the glossary; the totals, over several datasets;
+        # and every experiment by score.
+        def leaderboard(summaries)
+          ranked = summaries.sort_by { [it.dataset, -(it.mean(primary) || -1)] }
+          rows = ranked.map do |s|
+            [s.dataset, s.model.id, s.model.effort || "default", temperature_cell(s),
+             *metrics.map { percent(s.mean(it)) },
+             s.errors.size, format("%.1fs", s.duration), s.usage.input_tokens, s.usage.output_tokens, dollars(s.usage.cost_usd)]
+          end
+          best = best_per_dataset(ranked)
+          table(rows,
+            header:    ["dataset", "model", "effort", "temp", *metrics.map(&:name), "errors", "time", "tok_in", "tok_out", "cost"],
+            highlight: best)
+          best.each do |index|
+            s = ranked[index]
+            value_line(" [#{s.dataset}] (#{contender_label(s)})", s.mean(primary), s.usage.cost_usd)
+          end
+          table(glossary, header: %w[Term Description], fill: 1, wrap: true)
+          totals(summaries) if summaries.map(&:dataset).uniq.size > 1
+          scores(summaries)
+        end
+
+        # @return [Array<Array(String, String)>] the metrics' descriptions, then the gem's columns
+        def glossary
+          metrics.map { [it.name, it.description.to_s.empty? ? "(no description)" : it.description] } + GLOSSARY.to_a
+        end
+      end
+    end
+  end
+end

data/lib/agent/evals/cli/progress_bars.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+module Agent
+  module Evals
+    module CLI
+      # dry-cli-ui's multi_progress, drawn differently:
+      #
+      # - a flat list, the jobs indented under the headline, with no tree;
+      # - the headline's bar, which counts every case, in yellow;
+      # - a finished job keeps its bar, full, with the time it took where the
+      #   ETA was, instead of turning into a one-line summary;
+      # - labels padded to at least {LABEL_WIDTH} columns;
+      # - each frame overwrites the last line by line, inside a synchronized
+      #   update, rather than clearing the screen below and drawing it again,
+      #   which flickers with many rows.
+      class ProgressBars < Dry::CLI::UI::Widgets::MultiProgress
+        # The fewest columns a label is padded to.
+        LABEL_WIDTH = 40
+
+        # What each job's row starts with.
+        INDENT = "  "
+
+        # The headline's bar color.
+        HEADLINE_COLOR = :yellow
+
+        # Rows that have ended and drew a bar while they ran.
+        ENDED = %i[done failed].freeze
+
+        # Tells the terminal to show a frame only once all of it has arrived.
+        # Terminals that do not know these ignore them.
+        SYNC_START = "\e[?2026h"
+        SYNC_END = "\e[?2026l"
+
+        # @param console [Dry::CLI::UI::Console] the command's ui, whose stream, clock and settings it draws with
+        def self.for(console) = new(console.send(:err), clock: console.send(:clock), config: console.send(:config))
+
+        private
+
+        def label_width = [LABEL_WIDTH, title.length, *jobs.map { it.label.length }].max + INDENT.length
+
+        def rows
+          width = label_width
+          [headline(width), *visible.map { "#{INDENT}#{row(it, width - INDENT.length)}" }]
+        end
+
+        def redraw
+          drawn = rows
+          cursor = terminal.cursor
+          up = @drawn.positive? ? cursor.up(@drawn) : ""
+          lines = drawn.map { "\r#{it}#{cursor.clear_line_after}\n" }.join
+          below = drawn.size < @drawn ? cursor.clear_screen_down : ""
+          terminal.print("#{SYNC_START}#{up}#{lines}#{below}#{SYNC_END}")
+          @drawn = drawn.size
+        end
+
+        def headline(width)
+          return super unless live?
+
+          finished = state != :running
+          bar = full_meter(current, total, started, HEADLINE_COLOR, counts, (@seconds if finished))
+          "#{glyph(state)} #{title.ljust(width)} #{bar}#{finished ? noted(headline_note) : stopping}"
+        end
+
+        def row(job, width)
+          return super unless ENDED.include?(job.state) && !spinner?(job) && live?
+
+          handle = job.handle
+          bar = full_meter(handle.current, handle.total, job.started, handle.color, handle.counts, job.seconds.to_f)
+          "#{glyph(job.state)} #{job.label.ljust(width)} #{bar}#{noted(note(job))}"
+        end
+
+        # {#meter}, with the time taken in place of the ETA once seconds are known.
+        def full_meter(done, all, since, color, tally, seconds)
+          return meter(done, all, since, color, tally) if seconds.nil?
+
+          took = Dry::CLI::UI::Duration.format(seconds)
+          Dry::CLI::UI::Widgets::Progress.meter(terminal.pastel,
+            config,
+            done:        done,
+            all:         all,
+            columns:     bar_columns,
+            eta:         took,
+            color:       color,
+            counts:      tally,
+            count_width: count_width).sub("ETA #{took}", "in #{took}")
+        end
+      end
+    end
+  end
+end

data/lib/agent/evals/cli/run.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module Agent
+  module Evals
+    module CLI
+      # One experiment per model and dataset, each with a per-case breakdown.
+      # Without -m, every model .agent-eval.config.yml lists. {Compare} runs the same
+      # experiments and prints only the leaderboard.
+      class Run < ModelCommand
+        desc "Run every case against a model, or every model in .agent-eval.config.yml, with a per-case breakdown"
+
+        example [
+          "-m anthropic/claude-haiku-5.5",
+          "-m openai/gpt-4.1-mini -t 0 -c clear/00        # one variant's first cases",
+          "-m google/gemini-2.5-flash -e high --local      # score without Braintrust",
+          "-d colors,animals -o                           # both datasets; the breakdowns go to log/",
+          "-n                                             # show what would run, call nothing"
+        ]
+
+        def call(models:, dataset:, dry_run:, **options)
+          config = lineup(options)
+          contenders = contenders(config, models, options)
+          datasets = datasets(dataset)
+          return matrix(datasets, contenders) if dry_run
+
+          setup!(contenders, options)
+          runs = datasets.flat_map { |name| contenders.map { experiment(it, config, options, dataset: name) } }
+          summaries = execute(runs)
+          report(summaries, output: options[:output], open: options[:open]) do
+            summaries.each { breakdown(it, titled: summaries.size > 1) }
+          end
+        end
+
+        private
+
+        # One row per case: its score, and whatever the scorer diagnosed.
+        def breakdown(summary, titled: false)
+          status_box :info, summary.name if titled
+          rows = summary.results.map do |result|
+            [result.case.id, result.case.difficulty, result.score ? percent(result.score.value) : "-", diagnosis(result)]
+          end
+          table(rows, header: %w[case difficulty score diagnostics], fill: 3, wrap: true)
+        end
+
+        # The scorer's diagnostics, one per line, rendered without knowing
+        # their shape: "field: value" for a Hash, each item for an Array.
+        def diagnosis(result)
+          return result.error if result.failed?
+
+          result.score.diagnostics.map do |field, detail|
+            case detail
+            when Hash then "#{field}: #{detail.map { |k, v| "#{k}=#{v.inspect}" }.join(" ")}"
+            when Array then "#{field}: #{detail.join(", ")}"
+            else "#{field}: #{detail}"
+            end
+          end.join("\n")
+        end
+      end
+    end
+  end
+end

data/lib/agent/evals/cli/verify.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Agent
+  module Evals
+    module CLI
+      # Offline checks that every case agrees with what its suite demands of it.
+      class Verify < Command
+        desc "Check every case's input and expected answer, through the adapter and the scorer (no API calls)"
+
+        def call(**)
+          all = discovery.all
+          failures = Agent::Evals::Verify.new(configuration).problems
+          if failures.empty?
+            status_box :success, "#{all.size} cases verified across #{discovery.datasets.join(", ")}"
+            return
+          end
+
+          failures.each { |name, problems| problems.each { status_box :error, "#{name}: #{it}", stream: err } }
+          raise Error, "#{failures.size} case(s) failed verification"
+        end
+      end
+    end
+  end
+end

data/lib/agent/evals/cli/version.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Agent
+  module Evals
+    module CLI
+      class Version < Command
+        desc "Print the version"
+
+        def call(**)
+          out.puts Agent::Evals::VERSION
+        end
+      end
+    end
+  end
+end

data/lib/agent/evals/cli.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Agent
+  module Evals
+    # Every command the executable answers to. The help's title and
+    # description come from the suite's configuration, which the executable
+    # loads before this module is first referenced.
+    module CLI
+      extend Dry::CLI::Registry
+
+      PROGRAM = "agent-evals"
+
+      Dry::CLI::Help.configure do
+        configuration = Agent::Evals.configuration
+        title configuration.name == PROGRAM ? PROGRAM : "#{PROGRAM} (#{configuration.name})"
+        types = configuration.providers.uniq(&:request_type).map { "#{it.request_type} (#{it.request_type_description})" }
+        description "#{configuration.description} Request types: #{types.join("; ")}."
+
+        columns = IO.console&.winsize&.last || 80
+        width columns.clamp(60, 120)
+        exit_code_without_arguments 0
+
+        styles do
+          heading :bold, :cyan, case: :UPPERCASE
+          usage :yellow
+          example :yellow
+          option :green
+          example_comment :bright_black
+        end
+      end
+
+      register "run", Run
+      register "compare", Compare
+      register "cases", Cases
+      register "verify", Verify
+      register "version", Version, aliases: ["-V", "--version"]
+      register "completion", Dry::CLI::Autocomplete::Command[self, program_name: PROGRAM]
+    end
+  end
+end

data/lib/agent/evals/completion.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Agent
+  module Evals
+    # What a {Model} returns from one call: the text, what it cost, and the
+    # provider's raw response for an adapter that needs more than the text.
+    Completion = Data.define(:text, :usage, :raw) do
+      def initialize(text:, usage: Usage.zero, raw: nil) = super(text: text.to_s, usage: Usage.from(usage), raw: raw)
+    end
+  end
+end