llm-evals 0.4.0 → 0.5.0

data/lib/llm/evals/cli/model_command.rb

@@ -1,8 +1,6 @@
 # frozen_string_literal: true
 
 require "fileutils"
-require "pathname"
-require "uri"
 
 module LLM
   module Evals
@@ -11,374 +9,205 @@ module LLM
       # flag first, then the config entry, then .llm-evals.config.yml's defaults.
       #
       # Every column that is not a metric is the gem's; the metric columns,
-      # their glossary and the ranking come from the suite's {Scorer#metrics}.
+      # their glossary and the ranking come from the suite's {Scoring::Scorer#metrics}.
       class ModelCommand < Command
         include Formatting
 
+        # Gives each subclass the model flags, since dry-cli starts every subclass with no options.
+        #
+        # @param subclass [Class<ModelCommand>]
+        # @return [void]
         def self.inherited(subclass)
           super
-          subclass.extend(Dry::CLI::UI::Flags)
-          subclass.flags(:dry_run, :output)
-          subclass.option :models,
-            type:    :array,
-            aliases: ["-m"],
-            default: [],
-            desc:    "Model ids, comma-separated, instead of the config file's vendors"
-          subclass.option :dataset,
-            type:    :array,
-            aliases: ["-d"],
-            default: [],
-            desc:    "Datasets, comma-separated; every dataset when omitted. Each model runs on each"
-          subclass.option :config, aliases: ["-C"], desc: "The lineup file (default: the suite's .llm-evals.config.yml)"
-          subclass.option :effort, aliases: ["-e"], desc: "Reasoning effort: default, or one the provider accepts (overrides the config file)"
-          subclass.option :temperature, aliases: ["-t"], desc: "Sampling temperature, or 'default' to send none (overrides the config file)"
-          subclass.option :max_tokens, desc: "Output token cap, reasoning included (overrides the config file)"
-          subclass.option :case,
-            type:    :array,
-            aliases: ["-c"],
-            default: [],
-            desc:    "Only these cases, by id prefix: -c clear or -c clear/003,vague/001"
-          subclass.option :parallelism, aliases: ["-p"], desc: "Cases in flight at once, per model (default: the config's defaults.parallelism)"
-          subclass.option :concurrency,
-            aliases: ["-j"],
-            desc:    "Experiments running at once; 0 runs them all together (default: the number of CPUs)"
-          subclass.option :project,
-            desc: "Braintrust project name; default is the BRAINTRUST_PROJECT_ID project, else the config file's project"
-          subclass.option :open,
-            type:    :boolean,
-            default: true,
-            desc:    "Open the best experiment in the browser when done (no effect with --local)"
-          subclass.option :scores,
-            type:    :boolean,
-            default: false,
-            desc:    "Also print every experiment by score, with its cost and its tracker's link"
-          subclass.option :totals,
-            values:  %w[overall variant both],
-            default: "overall",
-            desc:    "Group TUI totals by overall results, case variant or both"
-          subclass.option :format,
-            values: %w[tui html json],
-            desc:   "Override the configured display: tui, html or json"
-          subclass.option :display,
-            values: %w[none tui browser],
-            desc:   "Override results.display from the lineup"
-          subclass.option :autosave,
-            type:    :boolean,
-            default: nil,
-            desc:    "Save a JSON snapshot regardless of results.autosave"
-          subclass.option :destination,
-            desc: "Override results.destination"
-          subclass.option :temp_dir,
-            desc: "Override results.temp_dir"
-          subclass.option :local,
-            type:    :boolean,
-            default: false,
-            desc:    "No remote tracker: results go to log/ only (still calls the models)"
-          subclass.option :tracker,
-            desc: "The remote tracker: braintrust, logfire, none, or one the suite registers " \
-                  "(default: the config's tracker:, else the one whose key is in the environment)"
+          ModelOptions.apply(subclass)
+        end
+
+        # The run every model command makes; subclasses add what prints per run
+        # through {#details}.
+        #
+        # @param models [Array<String>] as given to -m, possibly comma-separated
+        # @param dataset [Array<String>] as given to -d, possibly comma-separated
+        # @param dry_run [Boolean] print the matrix and spend nothing
+        # @param yes [Boolean] keep running through a streak of failures without asking
+        # @param options [Hash{Symbol => Object}] the other parsed flags
+        # @return [void]
+        # @raise [Error] when nothing can run, once the error budget is spent, or when told to stop
+        def call(models:, dataset:, dry_run:, yes: false, **options)
+          config = lineup(options)
+          weigh(config)
+          trackers = TrackerResolver.new(configuration:, config:, options:, err:)
+          planner = RunPlanner.new(configuration:, config:, options:, trackers:)
+          contenders = ContenderSelector.new(config:, providers:, warnings:).call(models, options)
+          datasets = planner.datasets(dataset)
+          return matrix(datasets, contenders) if dry_run
+
+          setup!(contenders, trackers)
+          runs = planner.experiments(datasets, contenders)
+          budget = config.error_budget
+          streak = Actions::ErrorStreak.new
+          pause = ErrorPause.new(console: ui, runs:, yes:, notice: ->(level, text) { status_box(level, text, stream: err) })
+          outcome = ExperimentPool.new(console: ui, colors: model_colors, warnings:)
+                                  .call(runs, concurrency: planner.concurrency, budget:, streak:, pause:)
+          record = RunRecord.new(command: invocation.action.to_s,
+            cli: { models:, dataset:, **options },
+            config:,
+            planner:,
+            trackers:,
+            providers:,
+            contenders:,
+            datasets:,
+            runs:,
+            seconds: outcome.seconds)
+          report(outcome.summaries,
+            output:      options[:output],
+            open:        options[:open],
+            scores:      options[:scores],
+            format:      options[:format],
+            config:,
+            options:,
+            run_options: record.to_h) { details(it) }
+          raise Error, budget.reason if budget&.spent?
+          raise Error, streak.reason if streak.stopped?
         end
 
         private
 
+        # What prints before the leaderboard; nothing, unless a subclass says.
+        #
+        # @param _summaries [Array<Actions::Experiment::Summary>]
+        # @return [void]
+        def details(_summaries) = nil
+
+        # A file named by --config may use the names of the suite's providers.
+        #
+        # @param options [Hash] the parsed flags
         # @return [Config] the lineup: --config, else the suite's
-        def lineup(options) = options[:config] ? Config.new(File.expand_path(options[:config])) : configuration.lineup
+        def lineup(options) = options[:config] ? Config.new(File.expand_path(options[:config]), providers:) : configuration.lineup
 
+        # The suite's, which every lineup and saved report is read against.
+        #
         # @return [Providers::Registry]
         def providers = configuration.providers
 
-        # @return [Scorer]
+        # The suite's; it supplies the metrics until a saved snapshot replaces them.
+        #
+        # @return [Scoring::Scorer]
         def scorer = configuration.scorer
 
-        # @return [Array<Metric>]
+        # The saved snapshot's once reporting has begun, so a report shows what was saved.
+        #
+        # @return [Array<Scoring::Metric>]
         def metrics = @report_metrics || scorer.metrics
 
         # Reads the file's weights before anything is spent, so a bad one fails
         # the run rather than the report at its end.
         #
-        # @return [WeightedScore, nil] nil when the file sets none
+        # @param config [Config]
+        # @return [Scoring::WeightedScore, nil] nil when the file sets none
         # @raise [Error] on a bad weight
         def weigh(config)
-          config.results # validate output settings before model calls start
+          config.validate!
           weights = config.score_weights
-          @weighted = weights.empty? ? nil : WeightedScore.new(weights, metrics: metrics.map(&:name))
-        end
-
-        # @return [Hash{String => Object}] the setting flags that were given
-        def overrides(options)
-          { "effort" => options[:effort], "temperature" => options[:temperature], "max_tokens" => options[:max_tokens] }
-        end
-
-        def experiment(contender, config, options, dataset:)
-          Experiment.new(
-            model:       providers.model_for(contender),
-            cases:       cases_for(options[:case], dataset: dataset),
-            adapter:     configuration.adapter,
-            scorer:      scorer,
-            parallelism: Integer(options[:parallelism] || config.defaults["parallelism"]),
-            trackers:    trackers_for(config, options)
-          )
-        end
-
-        # Captures the parsed flags and the settings that actually reached the
-        # experiments. The former preserves the invocation; the latter records
-        # defaults and expanded case/model selections that were implicit.
-        def serialize_run_options(config:, contenders:, datasets:, runs:, models:, dataset:, options:)
-          {
-            command:   invocation.action.to_s,
-            cli:       { models:, dataset:, **options },
-            effective: {
-              project:       options[:project] || config.project || configuration.project || configuration.name,
-              request_types: config.request_types.map(&:to_s),
-              datasets:,
-              cases:         runs.group_by(&:dataset).transform_values { |dataset_runs| dataset_runs.flat_map { it.cases.map(&:id) }.uniq },
-              contenders:    contenders.map do |contender|
-                settings = contender.settings
-                { model: contender.model, provider: providers.for(contender).name, effort: settings[:effort],
-                  temperature: settings[:temperature], max_tokens: settings[:max_tokens] }
-              end,
-              defaults:      config.defaults,
-              parallelism:   Integer(options[:parallelism] || config.defaults["parallelism"]),
-              concurrency:   concurrency(options),
-              tracker:       options[:local] ? "none" : remote_tracker(config, options)&.tracker_name
-            }
-          }
-        end
-
-        # The local JSON log, always, then the one remote tracker if any,
-        # each experiment getting instances of its own.
-        #
-        # @return [Array<Tracker>]
-        def trackers_for(config, options)
-          context = { suite:      configuration.name,
-                      project:    options[:project] || config.project || configuration.project || configuration.name,
-                      project_id: options[:project] ? nil : ENV["BRAINTRUST_PROJECT_ID"].to_s[/\S+/] }
-          local = Trackers::Local.new(dir: File.join(Dry::CLI::UI::Invocation.root(Dir.pwd), "log", "llm-evals"), **context)
-          klass = remote_tracker(config, options)
-          remote = klass&.new(**context, options: config.tracker_options(klass.tracker_name))
-          [local, remote].compact
-        end
-
-        # The remote tracker class this run uses, chosen once: --tracker,
-        # then tracker: in the config, then the environment. --local is none.
-        #
-        # @return [Class<Tracker>, nil]
-        def remote_tracker(config, options)
-          return @remote_tracker if defined?(@remote_tracker)
-
-          chosen = configuration.trackers.choose(flag: options[:tracker], config: config.tracker, env: ENV, local: options[:local])
-          @remote_tracker = chosen&.first
-          err.puts "tracker: #{chosen ? "#{chosen.first.tracker_name}, because #{chosen.last}" : "none"}" if options[:verbose]
-          @remote_tracker
-        end
-
-        # The contenders a command runs: the models named on the command line,
-        # or every model .llm-evals.config.yml lists.
-        #
-        # @param models [Array<String>] as given to -m, possibly comma-separated
-        # @return [Array<Contender>]
-        # @raise [Error] when neither names any model
-        def contenders(config, models, options)
-          models = models.flat_map { it.split(",") }
-          found = models.empty? ? config.contenders(overrides(options)) : config.contenders_for(models, overrides(options))
-          raise Error, "no models: list some under vendors in #{config.path}, or pass -m" if found.empty?
-
-          admit(found, config)
-        end
-
-        # Keeps the contenders whose provider produces what the suite declares
-        # in `request_types`, and records each refusal in {#warnings} by name,
-        # so a refused model is reported rather than run, failed and billed.
-        #
-        # @return [Array<Contender>] the admitted contenders
-        # @raise [Error] when none is admitted, naming every refusal
-        def admit(found, config)
-          found.each { providers.for(it) } # an unserved id fails here, before anything is spent
-          admitted, refusals = providers.admit(found, request_types: config.request_types)
-          refusals.each { warnings.record(it.to_s) }
-          raise Error, "no admitted models: #{refusals.join("; ")}" if admitted.empty?
-
-          admitted
+          @weighted = weights.empty? ? nil : Scoring::WeightedScore.new(weights, metrics: metrics.map(&:name))
         end
 
         # Env, the tracker's gems and key, hooks, provider clients, then the catalogs.
-        def setup!(contenders, options)
-          configuration.runtime.setup!(providers: providers.used_by(contenders), tracker: remote_tracker(lineup(options), options))
+        #
+        # @param contenders [Array<Models::Contender>]
+        # @param trackers [TrackerResolver]
+        # @return [void]
+        # @raise [Error] when a needed variable or gem is missing
+        def setup!(contenders, trackers)
+          configuration.runtime.setup!(providers: providers.used_by(contenders), tracker: trackers.remote)
           providers.verify!(contenders)
         end
 
         # What -n prints: every experiment that would run, and where the
         # refusals went, if there were any.
+        #
+        # @param datasets [Array<String>]
+        # @param contenders [Array<Models::Contender>]
+        # @return [void]
         def matrix(datasets, contenders)
           rows = datasets.product(contenders).map { |name, c| [name, c.model, c.effort, c.temperature, c.max_tokens] }
           table(rows, header: %w[dataset model effort temp max_tokens])
           save_warnings
         end
 
-        # Runs the experiments side by side, a progress bar each. What would
-        # print as a warning meanwhile, failed cases included, goes to
-        # {#warnings} instead.
+        # Filled by the selector and the pool, and written out only by {#save_warnings}.
         #
-        # @param runs [Array<Experiment>]
-        # @param concurrency [Integer] experiments at once; 0 runs them all together
-        # @return [Array<Experiment::Summary>]
-        def execute(runs, concurrency:)
-          summaries = Array.new(runs.size)
-          warnings.capture { bars(runs, summaries, concurrency) }
-          summaries.compact.each { |s| s.errors.each { warnings.record("[#{s.name}] #{it}") } }
-          summaries.compact
-        end
-
-        # -j, else one experiment per CPU. Uncapped, a full lineup started
-        # every model x dataset pair at once: 140 experiments, each with its
-        # own pool of case threads, all fighting for the network and the screen.
-        #
-        # @return [Integer] experiments at once; 0 means all of them
-        # @raise [Error] on a negative or non-numeric -j
-        def concurrency(options)
-          value = options[:concurrency] || Etc.nprocessors
-          Integer(value).tap { raise Error, "--concurrency must be 0 or more, got #{value}" if it.negative? }
-        rescue ArgumentError, TypeError
-          raise Error, "--concurrency must be a whole number, got #{value.inspect}"
-        end
-
-        # @return [Warnings] what this command has held back from the screen
-        def warnings = @warnings ||= Warnings.new
-
-        def bars(runs, summaries, concurrency)
-          ProgressBars.for(ui, colors: model_colors).run("Running #{runs.size} experiment#{"s" unless runs.size == 1}",
-            concurrent: concurrency.zero? || concurrency) do |multi|
-            labels = ProgressBars.labels(runs)
-            runs.each_with_index do |run, index|
-              multi.progress(labels[index], total: run.cases.size) do |bar|
-                summaries[index] = run.run(on_case: -> { bar.advance })
-              end
-            end
-          end
-        end
-
-        # @param names [Array<String>] as given to -d, possibly comma-separated; empty means every dataset
-        # @return [Array<String>]
-        # @raise [Error] on an unknown dataset, before anything is spent
-        def datasets(names)
-          names = names.flat_map { it.split(",") }.uniq
-          return discovery.datasets.tap { raise Error, "no datasets under #{discovery.root}" if it.empty? } if names.empty?
-
-          names.each { discovery.cases(dataset: it) }
-        end
+        # @return [Results::Warnings] what this command has held back from the screen
+        def warnings = @warnings ||= Results::Warnings.new
 
         # Prints what the block writes, then the leaderboard, its glossary and
-        # the scores; then says where the warnings went, if any, and opens the
-        # best-scoring experiment.
+        # the scores, when the run's display is the terminal; see {ReportDispatcher}.
         #
-        # Saves or renders through {Render::Reports::Pipeline}; the configured
-        # display and autosave policy are independent of model execution.
-        #
-        # @param summaries [Array<Experiment::Summary>]
+        # @param summaries [Array<Actions::Experiment::Summary>]
         # @param output [String, nil] what -o was given: nil, "-", "" for the default file, or a path
         # @param open [Boolean]
         # @param scores [Boolean] also print every experiment by score
-        # @param format [String] `tui`, `html` or `json`
+        # @param format [String, nil] `tui`, `html` or `json`
+        # @param config [Config]
+        # @param options [Hash] the parsed flags
+        # @param run_options [Hash] what the saved report says about its run; see {RunRecord#to_h}
+        # @param details [Proc, nil] prints what comes before the leaderboard, given the summaries shown
         # @return [void]
         def report(summaries, config:, options:, run_options:, output:, open:, scores: false, format: nil, &details)
-          results = effective_results(config, options)
-          results[:display] = { "tui" => "tui", "none" => "none", "html" => "browser", "json" => "none" }.fetch(format, results[:display]) if format
-          snapshot_options = run_options.merge(results:)
-          pipeline = render_pipeline(results, run_options)
-
-          if format == "html"
-            json_path = if results[:autosave]
-                          pipeline.save_json(summaries:, metrics:, weighted_score: @weighted, run_options: snapshot_options)
-                        end
-            json = if json_path
-                     File.read(json_path)
-                   else
-                     pipeline.write_json(summaries:, metrics:, weighted_score: @weighted, run_options: snapshot_options, path: nil)
-                   end
-            if json_path
-              status_box(:success, "JSON results written to #{relative(json_path)}", stream: err)
-            end
-            path = report_output_path(output, "html", pipeline)
-            document = pipeline.write_html(json:, path:)
-            path ? status_box(:success, "HTML report written to #{relative(path)}", stream: err) : out.write(document)
-            save_warnings
-            if open && path
-              url = URI::DEFAULT_PARSER.escape("file://#{path}")
-              status_box(:warning, "could not open #{path}", stream: err) unless configuration.runtime.browse(url)
-            end
-            return
-          end
-          if format == "json"
-            if output.nil? || output.empty?
-              path = pipeline.save_json(summaries:, metrics:, weighted_score: @weighted, run_options: snapshot_options)
-              status_box(:success, "JSON report written to #{relative(path)}", stream: err)
-            else
-              path = report_output_path(output, "json", pipeline)
-              document = pipeline.write_json(summaries:, metrics:, weighted_score: @weighted, run_options: snapshot_options, path:)
-              path ? status_box(:success, "JSON report written to #{relative(path)}", stream: err) : out.write(document)
-            end
-            save_warnings
-            return
-          end
-
-          result = pipeline.call(summaries:,
-            metrics:,
-            weighted_score: @weighted,
-            run_options: snapshot_options,
-            autosave: results[:autosave],
-            display: results[:display],
-            open:) do |report|
+          dispatcher = report_dispatcher(config, options)
+          # A run that also writes the page opens that, not a second window on the tracker.
+          open_tracker = open && dispatcher.display(format) == "tui"
+          dispatcher.call(summaries, metrics:, weighted_score: @weighted, run_options:, output:, open:, format:) do |report|
             @report_metrics = report.metrics
             @weighted = report.weighted_score
-            terminal_report(report.summaries, output:, open:, scores:, totals_mode: options[:totals]) do |shown|
+            terminal_report(report.summaries, output:, open: open_tracker, scores:, totals_mode: options[:totals], run_options:) do |shown|
               details&.call(shown)
             end
           end
-          if result.json_path
-            status_box(:success, "JSON results written to #{relative(result.json_path)}", stream: err)
-          end
-          if result.html_path
-            status_box(:success, "HTML report written to #{relative(result.html_path)}", stream: err)
-            status_box(:warning, "could not open #{result.html_path}", stream: err) if open && !result.browser_opened
-          end
-          save_warnings unless results[:display] == "tui"
         end
 
-        def terminal_report(summaries, output:, open:, scores:, totals_mode:, &details)
+        # Hands over the command's streams only through a {ReportDispatcher::Console},
+        # so that no output ever sees the command.
+        #
+        # @param config [Config]
+        # @param options [Hash] the parsed flags
+        # @return [ReportDispatcher]
+        def report_dispatcher(config, options)
+          ReportDispatcher.new(config:,
+            options:,
+            providers:,
+            runtime: configuration.runtime,
+            suite: configuration.name,
+            invocation:,
+            console: ReportDispatcher::Console.new(out:,
+              browser: configuration.runtime,
+              save_warnings: method(:save_warnings),
+              status: ->(level, text) { status_box(level, text, stream: err) }))
+        end
+
+        # Draws the terminal view, says where a report file went, saves the warnings,
+        # and opens the best experiment on its tracker.
+        #
+        # @param summaries [Array<Actions::Experiment::Summary>]
+        # @param output [String, nil] what -o was given
+        # @param open [Boolean] open the best experiment's tracker page
+        # @param scores [Boolean] also print every experiment by score
+        # @param totals_mode [String] `overall`, `variant` or `both`
+        # @param run_options [Hash] what the call statistics are read from
+        # @param details [Proc, nil] prints what comes before the statistics, given the summaries shown
+        # @return [void]
+        def terminal_report(summaries, output:, open:, scores:, totals_mode:, run_options:, &details)
           terminal = LLM::Evals::Render::Reports::Terminal.new(invocation:, writing_to: method(:writing_to), leaderboard: method(:leaderboard))
-          path = terminal.call(summaries:, output:, scores:, totals_mode:, &details)
+          path = terminal.call(summaries:, output:, scores:, totals_mode:) do |shown|
+            details&.call(shown)
+            stats = LLM::Evals::Render::Reports::CallStatsTable.new(Stats::CallStats.for_run(shown, run_options))
+            table(stats.rows, header: LLM::Evals::Render::Reports::CallStatsTable::HEADER, fill: 0)
+          end
           status_box(:info, "Report written to #{relative(path)}", stream: err) if path
           save_warnings
           url = summaries.max_by { it.score || -1 }&.permalink
           status_box(:warning, "could not open #{url}", stream: err) if open && url && !configuration.runtime.browse(url)
         end
 
-        def effective_results(config, options)
-          configured = config.results
-          {
-            autosave:    options[:autosave].nil? ? configured.autosave : options[:autosave],
-            destination: File.expand_path(options[:destination] || configured.destination),
-            temp_dir:    File.expand_path(options[:temp_dir] || configured.temp_dir),
-            display:     options[:display] || configured.display_mode
-          }
-        end
-
-        def render_pipeline(results, run_options)
-          project = run_options.dig(:effective, :project) || run_options.dig("effective", "project") || configuration.name
-          LLM::Evals::Render::Reports::Pipeline.new(providers:,
-            runtime: configuration.runtime,
-            destination: results[:destination],
-            temp_dir: results[:temp_dir],
-            project:)
-        end
-
-        def report_output_path(output, extension, pipeline)
-          return nil if output == "-"
-          return pipeline.report_path(File.dirname(invocation.report_path("")), extension) if output.nil? || output.empty?
-
-          invocation.report_path(output).sub(/\.log\z/, ".#{extension}")
-        end
-
         # Saves {#warnings}, if there are any, to
         # log/<command>.warnings.<YYYY-MM-DD>.<HHMMSS>.log at the repository
         # root, stamped with the time the process started, and says how many.
@@ -395,15 +224,12 @@ module LLM
           status_box(:warning, "#{count} #{count == 1 ? "Warning was" : "Warnings were"} recorded in #{relative(path)}", stream: err)
         end
 
-        # @return [Dry::CLI::UI::Invocation] how the command was run, which names its files
-        def invocation = ui.send(:invocation)
-
-        def relative(path) = Pathname(path).relative_path_from(Dir.pwd).to_s
-
         # Points the tables and boxes at another stream while the block runs.
         # They keep the screen's width.
         #
-        # @param io [IO, nil] nil to leave them on the screen
+        # @param io [IO, Dry::CLI::UI::Report, nil] nil to leave them on the screen
+        # @yield with the stream swapped
+        # @return [Object] what the block returns
         def writing_to(io)
           return yield if io.nil?
 
@@ -420,12 +246,23 @@ module LLM
           end
         end
 
+        # A snapshot's metrics may name their own primary; otherwise the scorer's stands.
+        #
+        # @return [Scoring::Metric]
         def primary = metrics.find(&:primary) || scorer.primary
 
+        # One per command, so the bars and the tables agree.
+        #
+        # @return [ModelColors]
         def model_colors = @model_colors ||= ModelColors.new
 
         # Built per call: {#report} swaps the metrics and the weights for those of
         # the saved snapshot, and {#writing_to} swaps the stream, before this runs.
+        #
+        # @param summaries [Array<Actions::Experiment::Summary>]
+        # @param scores [Boolean] also print every experiment by score
+        # @param totals_mode [String] `overall`, `variant` or `both`
+        # @return [void]
         def leaderboard(summaries, scores: false, totals_mode: "overall")
           standings = Standings.new(metrics:, primary:, weighted: @weighted)
           Leaderboard.new(standings:, table: method(:table), out:, pastel:, colors: model_colors)

data/lib/llm/evals/cli/model_options.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    module CLI
+      # The flags of every command that calls a model.
+      module ModelOptions
+        # Declares them on the command's class, which is where dry-cli keeps flags.
+        #
+        # @param command [Class<Command>]
+        # @return [void]
+        def self.apply(command)
+          command.extend(Dry::CLI::UI::Flags)
+          command.flags(:dry_run, :yes, :output)
+          command.option :models,
+            type:    :array,
+            aliases: ["-m"],
+            default: [],
+            desc:    "Model ids, comma-separated, instead of the config file's vendors"
+          command.option :dataset,
+            type:    :array,
+            aliases: ["-d"],
+            default: [],
+            desc:    "Datasets, comma-separated; every dataset when omitted. Each model runs on each"
+          command.option :config, aliases: ["-C"], desc: "The lineup file (default: the suite's .llm-evals.config.yml)"
+          command.option :effort, aliases: ["-e"], desc: "Reasoning effort: default, or one the provider accepts (overrides the config file)"
+          command.option :temperature, aliases: ["-t"], desc: "Sampling temperature, or 'default' to send none (overrides the config file)"
+          command.option :max_tokens, desc: "Output token cap, reasoning included (overrides the config file)"
+          command.option :case,
+            type:    :array,
+            aliases: ["-c"],
+            default: [],
+            desc:    "Only these cases, by id prefix: -c clear or -c clear/003,vague/001"
+          command.option :parallelism, aliases: ["-p"], desc: "Cases in flight at once, per model (default: the config's defaults.parallelism)"
+          command.option :concurrency,
+            aliases: ["-j"],
+            desc:    "Experiments running at once; 0 runs them all together (default: the number of CPUs)"
+          command.option :project,
+            desc: "Braintrust project name; default is the BRAINTRUST_PROJECT_ID project, else the config file's project"
+          command.option :open,
+            type:    :boolean,
+            default: true,
+            group:   :browser,
+            desc:    "Open the HTML report when done; with --display tui, the best experiment on its tracker instead"
+          command.option :scores,
+            type:    :boolean,
+            default: false,
+            group:   :tui,
+            desc:    "Also print every experiment by score, with its cost and its tracker's link"
+          command.option :totals,
+            values:  %w[overall variant both],
+            default: "overall",
+            group:   :tui,
+            desc:    "Group TUI totals by overall results, case variant or both"
+          command.option :format,
+            values: %w[tui html json],
+            desc:   "Override the configured display: tui, html or json"
+          command.option :display,
+            values: Config::RESULT_DISPLAYS,
+            desc:   "Override results.display: tui skips the HTML report, browser skips the terminal summary"
+          command.option :autosave,
+            type:    :boolean,
+            default: nil,
+            desc:    "Save a JSON snapshot regardless of results.autosave"
+          command.option :destination,
+            desc: "Override results.destination"
+          command.option :temp_dir,
+            desc: "Override results.temp_dir"
+          command.option :local,
+            type:    :boolean,
+            default: false,
+            desc:    "No remote tracker: results go to log/ only (still calls the models)"
+          command.option :tracker,
+            desc: "The remote tracker: braintrust, logfire, none, or one the suite registers " \
+                  "(default: the config's tracker:, else the one whose key is in the environment)"
+        end
+      end
+    end
+  end
+end