llm-evals 0.4.0 → 0.5.0

data/lib/llm/evals/cli/command.rb

@@ -2,8 +2,7 @@
 
 require "io/console"
 require "pastel"
-require "tty-box"
-require "tty-table"
+require "pathname"
 
 module LLM
   module Evals
@@ -12,27 +11,13 @@ module LLM
       class Command < Dry::CLI::Command
         include Dry::CLI::UI
 
-        # A cell that reads as a number: 42, 1,200, 97.1%, 3.5s, $0.0042.
-        NUMBER = /\A-?\$?\d[\d,]*(?:\.\d+)?[%s]?\z/
-
-        # Cells that stand in for a number without being one; they do not stop
-        # a column from right-aligning.
-        PLACEHOLDERS = ["", "-", "default"].freeze
-
         # The width assumed when the output is not a terminal.
         DEFAULT_WIDTH = 120
 
-        # The narrowest a wrapped column gets, however full the screen.
-        MIN_WRAP = 30
-
-        # Each status box's title and colors.
-        STATUS = {
-          info:    { title: " Info ", fg: :bright_white, bg: :blue },
-          success: { title: " Success ", fg: :black, bg: :green },
-          warning: { title: " Warning ", fg: :black, bg: :yellow },
-          error:   { title: " Error ", fg: :bright_white, bg: :red }
-        }.freeze
-
+        # Gives each subclass --verbose, since dry-cli starts every subclass with no options.
+        #
+        # @param subclass [Class<Command>]
+        # @return [void]
         def self.inherited(subclass)
           super
           subclass.option :verbose, type: :boolean, default: false, aliases: ["-v"], desc: "Print the full backtrace on failure"
@@ -40,81 +25,34 @@ module LLM
 
         private
 
-        # @return [Configuration] the suite, which must be loaded
+        # Fails here rather than deep inside a command when no helper set a suite up.
+        #
+        # @return [Runtime::Configuration] the suite, which must be loaded
         def configuration = LLM::Evals.configuration.require_suite!
 
-        # @return [Discovery]
+        # Where the suite's cases are found.
+        #
+        # @return [Datasets::Discovery]
         def discovery = configuration.discovery
 
-        # ui.table, as wide as the terminal (and so as the info and warning
-        # boxes), with every all-numeric column right-aligned and a line between
-        # rows once any cell spans several. dry-cli-ui does none of that, so this
-        # renders through tty-table, as it does.
-        #
-        # The fill column takes whatever width the others leave. With wrap, its
-        # text is word-wrapped to fit; without, it is only ever widened, so an
-        # id is never broken and a table too wide for the screen stays so.
+        # Prints nothing at all for no rows, not even the header.
         #
+        # @see Table#render
         # @param rows [Array<Array<#to_s>>]
-        # @param header [Array<String>]
-        # @param highlight [Array<Integer>] rows drawn in bold yellow (when color is on)
-        # @param fill [Integer] the column that absorbs the spare width
-        # @param wrap [Boolean] word-wrap the fill column to fit
         # @return [void]
-        def table(rows, header:, highlight: [], fill: 1, wrap: false)
-          return if rows.empty?
-
-          cells = rows.map { |row| row.map(&:to_s) }
-          widths = column_widths([header] + cells)
-          spare = terminal_width - (widths.sum - widths[fill]) - ((3 * widths.size) + 1)
-          if wrap
-            widths[fill] = [spare, MIN_WRAP].max
-            cells.each { it[fill] = wrap(it[fill], widths[fill]) }
-          else
-            widths[fill] = [spare, widths[fill]].max
-          end
-          alignments = header.each_index.map do |index|
-            column = cells.map { it[index] }.reject { PLACEHOLDERS.include?(it) }
-            !column.empty? && column.all? { NUMBER.match?(it) } ? :right : :left
-          end
-          multiline = cells.flatten.any? { it.include?("\n") }
-          cells = cells.each_with_index.map { |row, index| highlight.include?(index) ? row.map { pastel.bold.yellow(it) } : row }
-          rendered = TTY::Table.new(header: header.map { pastel.bold(it) }, rows: cells).render(
-            :unicode,
-            padding:       [0, 1],
-            width:         Dry::CLI::UI::Widgets::Table::UNLIMITED,
-            alignments:    alignments,
-            multiline:     multiline,
-            column_widths: widths
-          ) { |renderer| renderer.border.separator = :each_row if multiline }
-          out.puts rendered
+        def table(rows, **)
+          out.puts Table.new(pastel:, width: terminal_width).render(rows, **) unless rows.empty?
         end
 
-        # @return [Array<Integer>] each column's widest line
-        def column_widths(rows)
-          rows.transpose.map { |column| column.flat_map { it.to_s.split("\n") }.map(&:length).max.to_i }
-        end
-
-        # ui.info, ui.success, ui.warn and ui.error, painted on their color
-        # across the full width of the terminal. dry-cli-ui colors only a box's
-        # title, so this draws the box with tty-box, which it uses too. Without
-        # color it is a plain framed box.
+        # Prints one box as wide as the terminal.
         #
-        # @param level [Symbol] a key of {STATUS}
+        # @see StatusBox#render
+        # @param level [Symbol] a key of {StatusBox::LEVELS}
         # @param text [String]
         # @param stream [IO] out by default; warnings and errors go to err
         # @return [void]
         def status_box(level, text, stream: out)
-          look = STATUS.fetch(level)
-          colored = color?(stream)
-          style = colored ? { fg: look[:fg], bg: look[:bg], border: { fg: look[:fg], bg: look[:bg] } } : {}
-          # tty-box drops a word wider than the box, such as a long path; cut it first
-          stream.puts TTY::Box.frame(wrap(text, terminal_width - 4),
-            width:        terminal_width,
-            padding:      [0, 1],
-            title:        { top_left: look[:title] },
-            style:        style,
-            enable_color: colored)
+          stream.puts StatusBox.new(width: terminal_width, color: color?(stream)).render(level, text)
         end
 
         # A report written with -o is in color, as dry-cli-ui writes its own.
@@ -125,8 +63,13 @@ module LLM
           (stream.respond_to?(:color?) && stream.color?) || (stream.respond_to?(:tty?) && stream.tty? && !ENV.key?("NO_COLOR"))
         end
 
+        # Follows {#color?} for the current stream; {ModelCommand#writing_to} clears it when that changes.
+        #
+        # @return [Pastel]
         def pastel = @pastel ||= Pastel.new(enabled: color?)
 
+        # A terminal that cannot report its size counts as not being one.
+        #
         # @return [Integer] columns available on this terminal; {DEFAULT_WIDTH} when it is not one.
         #   While writing to a report, the screen's.
         def terminal_width
@@ -137,26 +80,16 @@ module LLM
           DEFAULT_WIDTH
         end
 
-        # Word-wraps text to a width, keeping its own line breaks. A word
-        # longer than the width is cut.
+        # Memoized for the life of the command.
         #
-        # @param text [String]
-        # @param width [Integer]
-        # @return [String]
-        def wrap(text, width)
-          text.split("\n").flat_map { |line| line.scan(/\S.{0,#{width - 1}}(?=\s|\z)|\S{#{width}}/).map(&:strip) }.join("\n")
-        end
-
-        # @param only [Array<String>, nil] case prefixes from --case
-        # @param dataset [String]
-        # @return [Array<Case>]
-        # @raise [Error] when nothing matches
-        def cases_for(only, dataset:)
-          list = discovery.cases(dataset: dataset, only: Array(only))
-          raise Error, "no cases match #{Array(only).join(", ")} in #{dataset}" if list.empty?
+        # @return [Dry::CLI::UI::Invocation] how the command was run, which names its files
+        def invocation = @invocation ||= Dry::CLI::UI::Invocation.for(self)
 
-          list
-        end
+        # Shortens a path for a message to the user.
+        #
+        # @param path [String]
+        # @return [String] relative to the working directory
+        def relative(path) = Pathname(path).relative_path_from(Dir.pwd).to_s
       end
     end
   end

data/lib/llm/evals/cli/compare.rb

@@ -14,26 +14,6 @@ module LLM
           "-e low -c clear -j 2                 # two experiments at a time",
           "-n                                   # show the matrix, spend nothing"
         ]
-
-        def call(models:, dataset:, dry_run:, **options)
-          config = lineup(options)
-          weigh(config)
-          contenders = contenders(config, models, options)
-          datasets = datasets(dataset)
-          return matrix(datasets, contenders) if dry_run
-
-          setup!(contenders, options)
-          runs = datasets.flat_map { |name| contenders.map { experiment(it, config, options, dataset: name) } }
-          summaries = execute(runs, concurrency: concurrency(options))
-          report(summaries,
-            output:      options[:output],
-            open:        options[:open],
-            scores:      options[:scores],
-            format:      options[:format],
-            config:,
-            options:,
-            run_options: serialize_run_options(config:, contenders:, datasets:, runs:, models:, dataset:, options:))
-        end
       end
     end
   end

data/lib/llm/evals/cli/contender_selector.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    module CLI
+      # Who runs: the models named on the command line, or every model the lineup
+      # lists, less those whose provider cannot produce what the suite declares.
+      class ContenderSelector
+        # Holds only collaborators; nothing is resolved until {#call}.
+        #
+        # @param config [Config]
+        # @param providers [Providers::Registry]
+        # @param warnings [Results::Warnings] records each refusal
+        def initialize(config:, providers:, warnings:)
+          @config = config
+          @providers = providers
+          @warnings = warnings
+        end
+
+        # Settles the lineup before anything is spent on it.
+        #
+        # @param models [Array<String>] as given to -m, possibly comma-separated
+        # @param options [Hash] the parsed flags, whose settings override the lineup's
+        # @return [Array<Models::Contender>]
+        # @raise [Error] when nothing names a model, or none is admitted
+        def call(models, options)
+          named = models.flat_map { it.split(",") }
+          settings = overrides(options)
+          found = named.empty? ? @config.contenders(settings) : @config.contenders_for(named, settings)
+          raise Error, "no models: list some under a provider in #{@config.path}, or pass -m" if found.empty?
+
+          admit(found)
+        end
+
+        private
+
+        # Spelled as the lineup spells them. The lineup ignores a nil, so an absent flag changes nothing.
+        #
+        # @param options [Hash] the parsed flags
+        # @return [Hash{String => Object}] the setting flags that were given
+        def overrides(options)
+          { "effort" => options[:effort], "temperature" => options[:temperature], "max_tokens" => options[:max_tokens] }
+        end
+
+        # A refused model is reported rather than run, failed and billed.
+        #
+        # @param found [Array<Models::Contender>]
+        # @return [Array<Models::Contender>] those the suite can ask
+        # @raise [Error] when no provider serves one of them, or none is admitted
+        def admit(found)
+          found.each { @providers.for(it) } # an unserved id fails here, before anything is spent
+          admitted, refusals = @providers.admit(found, request_types: @config.request_types)
+          refusals.each { @warnings.record(it.to_s) }
+          raise Error, "no admitted models: #{refusals.join("; ")}" if admitted.empty?
+
+          admitted
+        end
+      end
+    end
+  end
+end

data/lib/llm/evals/cli/error_pause.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    module CLI
+      # Says what is failing and asks whether to carry on. With nobody to ask, as
+      # in CI, it stops: nothing suggests the remaining calls would fare better.
+      class ErrorPause
+        # The question put to the user.
+        QUESTION = "Keep running?"
+
+        # The most error messages shown; the rest are only counted.
+        MESSAGES = 3
+
+        # The most names listed per dimension; the rest are only counted.
+        NAMES = 5
+
+        # How much of one message is shown.
+        MESSAGE_WIDTH = 300
+
+        # @param console [Dry::CLI::UI::Console] asks the question
+        # @param notice [#call] given the box's level and text
+        # @param runs [Array<Actions::Experiment>] the whole run
+        # @param yes [Boolean] carry on without asking
+        # @param interactive [Boolean] whether anyone can answer
+        def initialize(console:, notice:, runs:, yes: false, interactive: $stdin.tty?)
+          @console = console
+          @notice = notice
+          @runs = runs
+          @yes = yes
+          @interactive = interactive
+        end
+
+        # @param failures [Array<Results::RunResult>] the streak
+        # @return [Boolean] whether the run should carry on
+        def call(failures)
+          @notice.call(:error, describe(Actions::ErrorDiagnosis.new(failures:, runs: @runs), failures.size))
+          return true if @yes
+          return false unless @interactive
+
+          @console.confirm(QUESTION, default: false)
+        end
+
+        private
+
+        # @param diagnosis [Actions::ErrorDiagnosis]
+        # @param count [Integer] the failures in the streak
+        # @return [String] the box's text
+        def describe(diagnosis, count)
+          scopes = { "Models" => diagnosis.models, "Datasets" => diagnosis.datasets, "Cases" => diagnosis.cases }
+          lines = ["#{count} calls in a row failed. Paused: no new calls will start.", *errors(diagnosis.messages)]
+          lines.concat(scopes.map { |name, scope| "#{name}: #{extent(scope)}" })
+          lines << "The others have not failed so far, or have not run yet." unless scopes.values.all?(&:all?)
+          lines.join("\n")
+        end
+
+        # @param messages [Array<Array(String, Integer)>]
+        # @return [Array<String>]
+        def errors(messages)
+          shown = messages.first(MESSAGES).map { |message, count| "#{count} x #{message[0, MESSAGE_WIDTH]}" }
+          hidden = messages.size - shown.size
+          if hidden.positive?
+            [*shown, "and #{hidden} other #{hidden == 1 ? "error" : "errors"}"]
+          else
+            shown
+          end
+        end
+
+        # @param scope [Actions::ErrorDiagnosis::Scope]
+        # @return [String] "all 3 affected", or how many and which
+        def extent(scope)
+          return "all #{scope.total} affected" if scope.all?
+
+          hidden = scope.failed.size - NAMES
+          names = scope.failed.first(NAMES).join(", ")
+          "#{scope.failed.size} of #{scope.total} affected: #{names}#{" and #{hidden} more" if hidden.positive?}"
+        end
+      end
+    end
+  end
+end

data/lib/llm/evals/cli/experiment_pool.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    module CLI
+      # Runs the experiments side by side, a progress bar each. What would print
+      # as a warning meanwhile, failed cases included, goes to the warnings instead.
+      class ExperimentPool
+        # What can halt a run: the lineup's budget, which ends it, and the streak,
+        # which pauses it until the pause says otherwise. Either may be absent.
+        #
+        # @!attribute [r] budget
+        #   @return [Actions::ErrorBudget, nil]
+        # @!attribute [r] streak
+        #   @return [Actions::ErrorStreak, nil]
+        # @!attribute [r] pause
+        #   @return [#call, nil] given the streak; answers whether to carry on
+        Halts = Data.define(:budget, :streak, :pause) do
+          # Waits out a pause first, so a worker acts on the answer.
+          #
+          # @return [Boolean] whether no new case should start
+          def stop?
+            streak&.hold
+            budget&.spent? || streak&.stopped? || false
+          end
+
+          # Counts a finished call. The bars stay still while the question is up.
+          #
+          # @param result [Results::RunResult]
+          # @param widget [ProgressBars]
+          # @return [void]
+          def record(result, widget)
+            budget&.record(result)
+            streak&.record(result) { |failures| widget.pausing { pause.call(failures) } }
+          end
+        end
+
+        # Holds only collaborators; nothing runs until {#call}.
+        #
+        # @param console [Dry::CLI::UI::Console]
+        # @param colors [ModelColors]
+        # @param warnings [Results::Warnings]
+        def initialize(console:, colors:, warnings:)
+          @console = console
+          @colors = colors
+          @warnings = warnings
+        end
+
+        # Returns once every experiment has ended; one the budget stopped before it began has no summary.
+        #
+        # @param runs [Array<Actions::Experiment>]
+        # @param concurrency [Integer] experiments at once; 0 runs them all together
+        # @param budget [Actions::ErrorBudget, nil] stops the run once spent
+        # @param streak [Actions::ErrorStreak, nil] pauses the run on a streak of failures
+        # @param pause [#call, nil] given the streak; answers whether to carry on
+        # @return [Results::RunOutcome]
+        def call(runs, concurrency:, budget: nil, streak: nil, pause: nil)
+          summaries = Array.new(runs.size)
+          started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          @warnings.capture { bars(runs, summaries, concurrency, Halts.new(budget:, streak:, pause:)) }
+          seconds = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started
+          summaries.compact.each { |s| s.errors.each { @warnings.record("[#{s.name}] #{it}") } }
+          Results::RunOutcome.new(summaries: summaries.compact, seconds:)
+        end
+
+        private
+
+        # Draws a bar per run and fills summaries by position, so they keep the order
+        # given whichever finishes first. A run the budget stopped before it began leaves a nil.
+        #
+        # @param runs [Array<Actions::Experiment>]
+        # @param summaries [Array<Actions::Experiment::Summary, nil>] written in place, one slot per run
+        # @param concurrency [Integer] experiments at once; 0 runs them all together
+        # @param halts [Halts]
+        # @return [void]
+        def bars(runs, summaries, concurrency, halts)
+          widget = ProgressBars.for(@console, colors: @colors)
+          widget.run("Running #{runs.size} experiment#{"s" unless runs.size == 1}",
+            concurrent: concurrency.zero? || concurrency) do |multi|
+            labels = ProgressBars.labels(runs)
+            runs.each_with_index do |run, index|
+              multi.progress(labels[index], total: run.cases.size) do |bar|
+                next if halts.stop?
+
+                summaries[index] = run.run(stop: -> { halts.stop? },
+                  on_case: lambda { |result|
+                    bar.advance(as: result.errored? ? :failed : :ok)
+                    halts.record(result, widget)
+                  })
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/llm/evals/cli/formatting.rb

@@ -7,8 +7,16 @@ module LLM
       module Formatting
         private
 
+        # A missing number prints as a dash, never as 0%, which would read as a score.
+        #
+        # @param value [Float, nil] a fraction, where 1.0 is 100%
+        # @return [String]
         def percent(value) = value.nil? ? "-" : format("%.1f%%", value * 100)
 
+        # Four decimals, since a single cheap experiment can cost a fraction of a cent.
+        #
+        # @param value [Numeric]
+        # @return [String]
         def dollars(value) = format("$%.4f", value.to_f)
       end
     end

data/lib/llm/evals/cli/leaderboard.rb

@@ -6,7 +6,7 @@ module LLM
       # The terminal summary of a run: one row per experiment, the glossary, the
       # selected totals view, and the winners. Every column that is not a metric
       # is the gem's; the metric columns, their glossary and the ranking come
-      # from the suite's {Scorer#metrics}.
+      # from the suite's {Scoring::Scorer#metrics}.
       class Leaderboard
         include Formatting
 
@@ -27,9 +27,11 @@ module LLM
                        "ranks every experiment by it."
         }.freeze
 
+        # Built per call by {ModelCommand#leaderboard}, since the metrics and the stream can change between calls.
+        #
         # @param standings [Standings]
         # @param table [#call] draws rows, as {Command#table}
-        # @param out [IO] where the lines between tables go
+        # @param out [IO, Dry::CLI::UI::Report] where the lines between tables go
         # @param pastel [Pastel]
         # @param colors [ModelColors] shared with the progress bars, so a model keeps its color
         def initialize(standings:, table:, out:, pastel:, colors:)
@@ -44,7 +46,7 @@ module LLM
         # file sets weights; the glossary; the selected totals view; with
         # scores, every experiment by score; and, last, the winners.
         #
-        # @param summaries [Array<Experiment::Summary>]
+        # @param summaries [Array<Actions::Experiment::Summary>]
         # @param scores [Boolean] also print every experiment by score
         # @param totals_mode [String] `overall`, `variant` or `both`
         # @return [void]
@@ -52,7 +54,7 @@ module LLM
           weights = standings.weighted_scores(summaries)
           ranked = summaries.sort_by { [it.dataset, -((weighted ? weights[it] : it.mean(primary)) || -1)] }
           rows = ranked.map do |s|
-            [s.dataset, s.model.id, s.model.effort || "default", temperature_cell(s),
+            [s.dataset, s.model.id, s.model.effort_label, s.model.temperature_label,
              *(percent(weights[s]) if weighted),
              *metrics.map { percent(s.mean(it)) },
              s.errors.size, format("%.1fs", s.duration), s.usage.input_tokens, s.usage.output_tokens, dollars(s.usage.cost_usd)]
@@ -75,6 +77,8 @@ module LLM
           overall&.call
         end
 
+        # A metric named like one of the gem's columns replaces its entry, and wscore leads when the file sets weights.
+        #
         # @return [Array<Array(String, String)>] the metrics' descriptions, then the gem's columns, less any a metric took the name of
         def glossary
           described = metrics.map { [it.name, it.description.to_s.empty? ? "(no description)" : it.description] } + GLOSSARY.reject { |term, _| metrics.any? { it.name == term } }.to_a
@@ -85,27 +89,71 @@ module LLM
 
         attr_reader :standings, :table, :out, :pastel, :colors
 
+        # @!attribute [r] standings
+        #   The arithmetic behind every ranking here.
+        #   @return [Standings]
+
+        # @!attribute [r] table
+        #   Draws rows through the command, so they follow its stream.
+        #   @return [#call]
+
+        # @!attribute [r] out
+        #   Where the lines between tables go.
+        #   @return [IO, Dry::CLI::UI::Report]
+
+        # @!attribute [r] pastel
+        #   Paints the winners' models.
+        #   @return [Pastel]
+
+        # @!attribute [r] colors
+        #   Shared with the progress bars.
+        #   @return [ModelColors]
+
+        # The columns the suite scores.
+        #
+        # @return [Array<Scoring::Metric>]
         def metrics = standings.metrics
 
+        # The metric the ranking reads.
+        #
+        # @return [Scoring::Metric]
         def primary = standings.primary
 
+        # Doubles as the question of whether the file sets weights at all.
+        #
+        # @return [Scoring::WeightedScore, nil] nil when it does not
         def weighted = standings.weighted
 
+        # Prints the best value over every dataset.
+        #
+        # @param label [String] what follows "best value", with its leading space
+        # @param value [Float] the primary metric, as a fraction
+        # @param cost [Float] in US dollars
+        # @return [void]
         def value_line(label, value, cost)
           out.puts "best value#{label}: #{format("%.1f%%", value * 100)} at #{dollars(cost)}, " \
                    "the cheapest within #{(Standings::VALUE_MARGIN * 100).round} points of the best #{primary.name}"
         end
 
+        # Prints the best weighted score over every dataset.
+        #
+        # @param label [String] what follows "best wscore", with its leading space
+        # @param value [Float] the weighted score, as a fraction
+        # @return [void]
         def weighted_line(label, value)
           out.puts "best wscore#{label}: #{format("%.1f%%", value * 100)}, by the weights in weighted_score"
         end
 
-        def contender_label(summary) = "#{summary.model.id} effort=#{summary.model.effort || "default"}"
-
-        def temperature_cell(summary) = summary.model.temperature.nil? ? "default" : summary.model.temperature
+        # Names a contender in the conclusions over every dataset, by model and effort.
+        #
+        # @param summary [Actions::Experiment::Summary] any of the contender's experiments
+        # @return [String]
+        def contender_label(summary) = "#{summary.model.id} effort=#{summary.model.effort_label}"
 
         # The selected totals view: one aggregate table, one per variant, or both.
         #
+        # @param summaries [Array<Actions::Experiment::Summary>]
+        # @param mode [String] `overall`, `variant` or `both`
         # @return [Proc, nil] prints the conclusions over every dataset, for after
         #   everything else; nil over a single dataset
         def totals(summaries, mode: "overall")
@@ -133,12 +181,14 @@ module LLM
           end
         end
 
+        # One row per contender, with the best value highlighted.
+        #
         # @param groups [Array<Hash>] from {Standings#contender_groups}
         # @return [void]
         def contender_table(groups)
           rows = groups.map do |g|
             s = g[:first]
-            [s.model.id, s.model.effort || "default", temperature_cell(s), *(percent(g[:weighted]) if weighted), percent(g[:value]),
+            [s.model.id, s.model.effort_label, s.model.temperature_label, *(percent(g[:weighted]) if weighted), percent(g[:value]),
              g[:errors], g[:usage].input_tokens, g[:usage].output_tokens, dollars(g[:usage].cost_usd)]
           end
           table.call(rows,
@@ -149,10 +199,13 @@ module LLM
 
         # Every experiment by its mean score, best first, with its cost and its
         # Braintrust page. URLs are never wrapped, so they stay clickable.
+        #
+        # @param summaries [Array<Actions::Experiment::Summary>]
+        # @return [void]
         def score_table(summaries)
           weights = standings.weighted_scores(summaries)
           rows = summaries.sort_by { -(it.score || -1) }.map do |s|
-            [s.model.id, s.model.effort || "default", s.dataset, percent(s.score), *(percent(weights[s]) if weighted), dollars(s.usage.cost_usd), s.permalink || "-"]
+            [s.model.id, s.model.effort_label, s.dataset, percent(s.score), *(percent(weights[s]) if weighted), dollars(s.usage.cost_usd), s.permalink || "-"]
           end
           table.call(rows, header: ["model", "effort", "dataset", "score", *("wscore" if weighted), "cost", "url"], fill: 0)
         end
@@ -161,14 +214,15 @@ module LLM
         # the color of its progress bars.
         #
         # @param title [String]
-        # @param summaries [Array<Experiment::Summary>]
+        # @param summaries [Array<Actions::Experiment::Summary>]
         # @param label [String] the name of what the block reads
-        # @yieldparam summary [Experiment::Summary]
+        # @yieldparam summary [Actions::Experiment::Summary]
         # @yieldreturn [Float]
+        # @return [void]
         def winners(title, summaries, label)
           return if summaries.empty?
 
-          rows = summaries.map { ["[#{it.dataset}]", "effort=#{it.model.effort || "default"}", "#{label}=#{percent(yield(it))}", dollars(it.usage.cost_usd)] }
+          rows = summaries.map { ["[#{it.dataset}]", "effort=#{it.model.effort_label}", "#{label}=#{percent(yield(it))}", dollars(it.usage.cost_usd)] }
           widths = rows.transpose.map { |column| column.map(&:length).max }
           out.puts title
           rows.zip(summaries).each do |(dataset, effort, value, cost), summary|

data/lib/llm/evals/cli/model_colors.rb

@@ -9,11 +9,14 @@ module LLM
         # What a model can be drawn in. More models than these share them.
         COLORS = %i[bright_red bright_green bright_yellow bright_blue bright_magenta bright_cyan bright_white].freeze
 
+        # Shuffles once, so the colors change between runs and hold within one.
         def initialize
           palette = COLORS.shuffle
           @colors = Hash.new { |colors, model| colors[model] = palette[colors.size % palette.size] }
         end
 
+        # Hands out the next color the first time a model is asked about.
+        #
         # @param model [String] the model's id
         # @return [Symbol] a Pastel style
         def [](model) = @colors[model]