llm-evals 0.4.0 → 0.5.0

data/lib/llm/evals/cli/run_planner.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+require "etc"
+
+module LLM
+  module Evals
+    module CLI
+      # What a run will do: which datasets, how many cases and experiments at
+      # once, and one experiment per dataset and contender.
+      class RunPlanner
+        # Holds only collaborators; nothing is decided until it is asked.
+        #
+        # @param configuration [Runtime::Configuration]
+        # @param config [Config]
+        # @param options [Hash] the parsed flags
+        # @param trackers [TrackerResolver]
+        def initialize(configuration:, config:, options:, trackers:)
+          @configuration = configuration
+          @config = config
+          @options = options
+          @trackers = trackers
+        end
+
+        # Named datasets are checked, so a typo fails before the first call.
+        #
+        # @param names [Array<String>] as given to -d, possibly comma-separated; empty means every dataset
+        # @return [Array<String>]
+        # @raise [Error] on an unknown dataset, before anything is spent
+        def datasets(names)
+          names = names.flat_map { it.split(",") }.uniq
+          return every_dataset if names.empty?
+
+          names.each { discovery.cases(dataset: it) }
+        end
+
+        # Every dataset crossed with every contender, datasets first.
+        #
+        # @param datasets [Array<String>]
+        # @param contenders [Array<Models::Contender>]
+        # @return [Array<Actions::Experiment>]
+        def experiments(datasets, contenders)
+          datasets.flat_map { |dataset| contenders.map { experiment(it, dataset) } }
+        end
+
+        # -p, else the lineup's defaults.parallelism.
+        #
+        # @return [Integer] cases in flight at once, per model
+        def parallelism = Integer(@options[:parallelism] || @config.defaults["parallelism"])
+
+        # -j, else one experiment per CPU. Uncapped, a full lineup started
+        # every model x dataset pair at once: 140 experiments, each with its
+        # own pool of case threads, all fighting for the network and the screen.
+        #
+        # @return [Integer] experiments at once; 0 means all of them
+        # @raise [Error] on a negative or non-numeric -j
+        def concurrency
+          value = @options[:concurrency] || Etc.nprocessors
+          Integer(value).tap { raise Error, "--concurrency must be 0 or more, got #{value}" if it.negative? }
+        rescue ArgumentError, TypeError
+          raise Error, "--concurrency must be a whole number, got #{value.inspect}"
+        end
+
+        private
+
+        # Read through the configuration on each call.
+        #
+        # @return [Datasets::Discovery]
+        def discovery = @configuration.discovery
+
+        # Used when -d names none.
+        #
+        # @return [Array<String>]
+        # @raise [Error] when the suite has no datasets
+        def every_dataset
+          discovery.datasets.tap { raise Error, "no datasets under #{discovery.root}" if it.empty? }
+        end
+
+        # Trackers are made new for each experiment; the adapter and the scorer are shared by all.
+        #
+        # @param contender [Models::Contender]
+        # @param dataset [String]
+        # @return [Actions::Experiment]
+        # @raise [Error] when no case in the dataset matches -c
+        def experiment(contender, dataset)
+          Actions::Experiment.new(
+            model:       @configuration.providers.model_for(contender, throttle: @config.throttle),
+            cases:       discovery.cases!(dataset:, only: @options[:case]),
+            adapter:     @configuration.adapter,
+            scorer:      @configuration.scorer,
+            parallelism:,
+            trackers:    @trackers.for_experiment
+          )
+        end
+      end
+    end
+  end
+end

data/lib/llm/evals/cli/run_record.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    module CLI
+      # What a saved report says about its run: the parsed flags, which preserve
+      # the invocation, and the settings that actually reached the experiments,
+      # which record the defaults and expanded selections that were implicit.
+      #
+      # @!attribute [r] command
+      #   @return [String] the action that ran, `run` or `compare`
+      # @!attribute [r] cli
+      #   @return [Hash{Symbol => Object}] the parsed flags
+      # @!attribute [r] config
+      #   @return [Config]
+      # @!attribute [r] planner
+      #   @return [RunPlanner]
+      # @!attribute [r] trackers
+      #   @return [TrackerResolver]
+      # @!attribute [r] providers
+      #   @return [Providers::Registry]
+      # @!attribute [r] contenders
+      #   @return [Array<Models::Contender>]
+      # @!attribute [r] datasets
+      #   @return [Array<String>]
+      # @!attribute [r] runs
+      #   @return [Array<Actions::Experiment>]
+      # @!attribute [r] seconds
+      #   @return [Float] how long the experiments took, by the wall clock
+      RunRecord = Data.define(:command, :cli, :config, :planner, :trackers, :providers, :contenders, :datasets, :runs, :seconds) do
+        # Replaces `Data#to_h`, so the collaborators stay out of what gets saved.
+        #
+        # @return [Hash]
+        def to_h
+          { command:, cli:, effective: effective }
+        end
+
+        private
+
+        # What reached the experiments once every default and selection was expanded.
+        #
+        # @return [Hash{Symbol => Object}]
+        def effective
+          { project:       trackers.project,
+            request_types: config.request_types.map(&:to_s),
+            datasets:,
+            cases:         runs.group_by(&:dataset).transform_values { |group| group.flat_map { it.cases.map(&:id) }.uniq },
+            contenders:    contenders.map { contender(it) },
+            defaults:      config.defaults,
+            parallelism:   planner.parallelism,
+            concurrency:   planner.concurrency,
+            tracker:       cli[:local] ? "none" : trackers.remote&.tracker_name,
+            seconds: }
+        end
+
+        # Names the provider too, which a model id alone does not settle.
+        #
+        # @param entry [Models::Contender]
+        # @return [Hash{Symbol => Object}]
+        def contender(entry)
+          settings = entry.settings
+          { model: entry.model, provider: providers.for(entry).name, effort: settings[:effort],
+            temperature: settings[:temperature], max_tokens: settings[:max_tokens] }
+        end
+      end
+    end
+  end
+end

data/lib/llm/evals/cli/standings.rb

@@ -11,9 +11,23 @@ module LLM
 
         attr_reader :metrics, :primary, :weighted
 
-        # @param metrics [Array<Metric>]
-        # @param primary [Metric] the metric the ranking reads
-        # @param weighted [WeightedScore, nil] nil when the file sets no weights
+        # @!attribute [r] metrics
+        #   The columns the suite scores.
+        #   @return [Array<Scoring::Metric>]
+
+        # @!attribute [r] primary
+        #   The metric the ranking reads.
+        #   @return [Scoring::Metric]
+
+        # @!attribute [r] weighted
+        #   Doubles as the question of whether the file sets weights at all.
+        #   @return [Scoring::WeightedScore, nil]
+
+        # Holds no results; every method answers from what it is given.
+        #
+        # @param metrics [Array<Scoring::Metric>]
+        # @param primary [Scoring::Metric] the metric the ranking reads
+        # @param weighted [Scoring::WeightedScore, nil] nil when the file sets no weights
         def initialize(metrics:, primary:, weighted: nil)
           @metrics = metrics
           @primary = primary
@@ -25,7 +39,7 @@ module LLM
         # plain score-per-dollar ratio would crown whatever is nearly free,
         # however wrong it is.
         #
-        # @param entries [Array<Array(Float, Float, Integer)>] primary metric, cost, errors
+        # @param entries [Array<Array(Float, Float, Integer)>] primary metric (nil when unscored), cost, errors
         # @return [Integer, nil] the winner's index
         def best_value(entries)
           eligible = entries.each_index.select { entries[it][0] && entries[it][2].zero? }
@@ -37,14 +51,14 @@ module LLM
 
         # Each contender's numbers over the given experiments, best first.
         #
-        # @param summaries [Array<Experiment::Summary>]
+        # @param summaries [Array<Actions::Experiment::Summary>]
         # @return [Array<Hash>]
         def contender_groups(summaries)
           groups = summaries.group_by { [it.model.key, it.model.effort, it.model.temperature] }.values.map do |group|
             values = group.filter_map { it.mean(primary) }
             { first: group.first, errors: group.sum { it.errors.size },
               value: values.size == group.size ? values.sum / values.size : nil,
-              usage: group.map(&:usage).reduce(Usage.zero, :+),
+              usage: group.map(&:usage).reduce(Stats::Usage.zero, :+),
               row:   overall_row(group) }
           end
           scores = weighted_by(groups.map { it[:row] })
@@ -52,12 +66,16 @@ module LLM
           groups.sort_by { -((weighted ? it[:weighted] : it[:value]) || -1) }
         end
 
+        # The same rule as {#best_value}, over contenders rather than experiments.
+        #
         # @param groups [Array<Hash>] from {#contender_groups}
         # @return [Integer, nil] the index of the best value
         def best_contender(groups) = best_value(groups.map { [it[:value], it[:usage].cost_usd, it[:errors]] })
 
-        # @param summaries [Array<Experiment::Summary>]
-        # @return [Hash{Experiment::Summary => Float, nil}] by identity, each compared within its dataset
+        # Weighted scores are relative, so an experiment is compared only within its dataset.
+        #
+        # @param summaries [Array<Actions::Experiment::Summary>]
+        # @return [Hash{Actions::Experiment::Summary => Float, nil}] by identity, each compared within its dataset
         def weighted_scores(summaries)
           summaries.group_by(&:dataset).values.each_with_object({}.compare_by_identity) do |group, scores|
             group.zip(weighted_by(group.map { weighted_row(it) })).each { |summary, value| scores[summary] = value }
@@ -66,7 +84,7 @@ module LLM
 
         # The best value in each dataset's block of the leaderboard.
         #
-        # @param ranked [Array<Experiment::Summary>]
+        # @param ranked [Array<Actions::Experiment::Summary>]
         # @return [Array<Integer>] indices into ranked
         def best_per_dataset(ranked)
           ranked.each_index.group_by { ranked[it].dataset }.values.filter_map do |indices|
@@ -77,22 +95,24 @@ module LLM
 
         private
 
-        # What {WeightedScore} reads from one experiment.
+        # What {Scoring::WeightedScore} reads from one experiment.
         #
-        # @param summary [Experiment::Summary]
+        # @param summary [Actions::Experiment::Summary]
         # @return [Hash{String => Numeric, nil}]
         def weighted_row(summary)
           metrics.to_h { [it.name, summary.mean(it)] }.merge(
-            "score"  => summary.score,
-            "errors" => summary.errors.size,
-            "time"   => summary.duration,
-            "cost"   => summary.usage.cost_usd.to_f
+            "score"   => summary.score,
+            "errors"  => summary.errors.size,
+            "time"    => summary.duration,
+            "cost"    => summary.usage.cost_usd.to_f,
+            "latency" => summary.latency,
+            "p95"     => summary.latency_p95
           )
         end
 
         # One contender over every dataset: errors and magnitudes add up, the rest average.
         #
-        # @param group [Array<Experiment::Summary>]
+        # @param group [Array<Actions::Experiment::Summary>]
         # @return [Hash{String => Numeric, nil}]
         def overall_row(group)
           rows = group.map { weighted_row(it) }
@@ -101,10 +121,12 @@ module LLM
             total = values.sum(&:to_f)
             next [name, nil] if values.any?(&:nil?)
 
-            [name, ["errors", *WeightedScore::MAGNITUDES].include?(name) ? total : total / values.size]
+            [name, Scoring::WeightedScore::SUMMED.include?(name) ? total : total / values.size]
           end
         end
 
+        # Scores rows together, since a weighted score depends on the rows beside it.
+        #
         # @param rows [Array<Hash>]
         # @return [Array<Float, nil>] nils when the file sets no weights
         def weighted_by(rows) = weighted ? weighted.call(rows) : Array.new(rows.size)

data/lib/llm/evals/cli/status_box.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require "tty-box"
+
+module LLM
+  module Evals
+    module CLI
+      # ui.info, ui.success, ui.warn and ui.error, painted on their color across
+      # the full width of the terminal. dry-cli-ui colors only a box's title, so
+      # this draws the box with tty-box, which it uses too. Without color it is a
+      # plain framed box.
+      class StatusBox
+        # Each level's title and colors.
+        LEVELS = {
+          info:    { title: " Info ", fg: :bright_white, bg: :blue },
+          success: { title: " Success ", fg: :black, bg: :green },
+          found:   { title: " Info ", fg: :black, bg: :green },
+          warning: { title: " Warning ", fg: :black, bg: :yellow },
+          error:   { title: " Error ", fg: :bright_white, bg: :red }
+        }.freeze
+
+        # Told its width and color, rather than asking the terminal itself.
+        #
+        # @param width [Integer] the box's whole width
+        # @param color [Boolean]
+        def initialize(width:, color:)
+          @width = width
+          @color = color
+        end
+
+        # Cuts words too wide for the box, before tty-box can drop them.
+        #
+        # @param level [Symbol] a key of {LEVELS}
+        # @param text [String]
+        # @return [String]
+        # @raise [KeyError] on an unknown level
+        def render(level, text)
+          look = LEVELS.fetch(level)
+          style = @color ? { fg: look[:fg], bg: look[:bg], border: { fg: look[:fg], bg: look[:bg] } } : {}
+          # tty-box drops a word wider than the box, such as a long path; cut it first
+          TTY::Box.frame(Text.wrap(text, @width - 4),
+            width:        @width,
+            padding:      [0, 1],
+            title:        { top_left: look[:title] },
+            style:        style,
+            enable_color: @color)
+        end
+      end
+    end
+  end
+end

data/lib/llm/evals/cli/table.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+require "tty-table"
+
+module LLM
+  module Evals
+    module CLI
+      # ui.table, as wide as the terminal (and so as the status boxes), with every
+      # all-numeric column right-aligned and a line between rows once any cell
+      # spans several. dry-cli-ui does none of that, so this renders through
+      # tty-table, as it does.
+      #
+      # The fill column takes whatever width the others leave. With wrap, its text
+      # is word-wrapped to fit; without, it is only ever widened, so an id is never
+      # broken and a table too wide for the screen stays so.
+      class Table
+        # A cell that reads as a number: 42, 1,200, 97.1%, 3.5s, $0.0042.
+        NUMBER = /\A-?\$?\d[\d,]*(?:\.\d+)?[%s]?\z/
+
+        # Cells that stand in for a number without being one; they do not stop
+        # a column from right-aligning.
+        PLACEHOLDERS = ["", "-", "default"].freeze
+
+        # The narrowest a wrapped column gets, however full the screen.
+        MIN_WRAP = 30
+
+        # Holds no rows, so one instance can draw any number of tables.
+        #
+        # @param pastel [Pastel]
+        # @param width [Integer] the columns available
+        def initialize(pastel:, width:)
+          @pastel = pastel
+          @width = width
+        end
+
+        # Draws one table, with widths measured before any highlight is painted on.
+        #
+        # @param rows [Array<Array<#to_s>>]
+        # @param header [Array<String>]
+        # @param highlight [Array<Integer>] rows drawn in bold yellow (when color is on)
+        # @param fill [Integer] the column that absorbs the spare width
+        # @param wrap [Boolean] word-wrap the fill column to fit
+        # @return [String]
+        def render(rows, header:, highlight: [], fill: 1, wrap: false)
+          cells = rows.map { |row| row.map(&:to_s) }
+          widths = fitted_widths(header, cells, fill, wrap)
+          alignments = alignments(header, cells)
+          multiline = cells.flatten.any? { it.include?("\n") }
+          cells = cells.each_with_index.map { |row, index| highlight.include?(index) ? row.map { @pastel.bold.yellow(it) } : row }
+          TTY::Table.new(header: header.map { @pastel.bold(it) }, rows: cells).render(
+            :unicode,
+            padding:       [0, 1],
+            width:         Dry::CLI::UI::Widgets::Table::UNLIMITED,
+            alignments:    alignments,
+            multiline:     multiline,
+            column_widths: widths
+          ) { |renderer| renderer.border.separator = :each_row if multiline }
+        end
+
+        private
+
+        # Wraps the fill column's cells in place when asked to.
+        #
+        # @param header [Array<String>]
+        # @param cells [Array<Array<String>>] modified when wrapping
+        # @param fill [Integer] the column that absorbs the spare width
+        # @param wrap [Boolean] word-wrap the fill column to fit
+        # @return [Array<Integer>] each column's width
+        def fitted_widths(header, cells, fill, wrap)
+          widths = column_widths([header] + cells)
+          spare = @width - (widths.sum - widths[fill]) - ((3 * widths.size) + 1)
+          if wrap
+            widths[fill] = [spare, MIN_WRAP].max
+            cells.each { it[fill] = Text.wrap(it[fill], widths[fill]) }
+          else
+            widths[fill] = [spare, widths[fill]].max
+          end
+          widths
+        end
+
+        # A column right-aligns when all it holds reads as a number. One holding only placeholders stays left.
+        #
+        # @param header [Array<String>]
+        # @param cells [Array<Array<String>>]
+        # @return [Array<Symbol>] `:left` or `:right`, one per column
+        def alignments(header, cells)
+          header.each_index.map do |index|
+            column = cells.map { it[index] }.reject { PLACEHOLDERS.include?(it) }
+            !column.empty? && column.all? { NUMBER.match?(it) } ? :right : :left
+          end
+        end
+
+        # Measures past any color, which takes up no columns.
+        #
+        # @param rows [Array<Array<#to_s>>]
+        # @return [Array<Integer>] each column's widest line
+        def column_widths(rows)
+          rows.transpose.map { |column| column.flat_map { @pastel.strip(it.to_s).split("\n") }.map(&:length).max.to_i }
+        end
+      end
+    end
+  end
+end

data/lib/llm/evals/cli/text.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    module CLI
+      # Text fitted to a width, for tables and boxes alike.
+      module Text
+        module_function
+
+        # Word-wraps text to a width, keeping its own line breaks. A word
+        # longer than the width is cut.
+        #
+        # @param text [String]
+        # @param width [Integer]
+        # @return [String]
+        def wrap(text, width)
+          text.split("\n").flat_map { |line| line.scan(/\S.{0,#{width - 1}}(?=\s|\z)|\S{#{width}}/).map(&:strip) }.join("\n")
+        end
+      end
+    end
+  end
+end

data/lib/llm/evals/cli/tracker_resolver.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    module CLI
+      # Where a run's results go: the local JSON log, always, then one remote
+      # tracker if any, chosen once per run.
+      class TrackerResolver
+        # Holds only collaborators; nothing is chosen until {#remote} is asked.
+        #
+        # @param configuration [Runtime::Configuration]
+        # @param config [Config]
+        # @param options [Hash] the parsed flags
+        # @param err [IO] says which tracker was chosen, under --verbose
+        # @param env [#[]]
+        def initialize(configuration:, config:, options:, err:, env: ENV)
+          @configuration = configuration
+          @config = config
+          @options = options
+          @err = err
+          @env = env
+        end
+
+        # --tracker, then tracker: in the config, then the environment. --local is none.
+        #
+        # @return [Class<Trackers::Base>, nil]
+        def remote
+          return @remote if defined?(@remote)
+
+          chosen = @configuration.trackers.choose(flag: @options[:tracker], config: @config.tracker, env: @env, local: @options[:local])
+          @err.puts "tracker: #{chosen ? "#{chosen.first.tracker_name}, because #{chosen.last}" : "none"}" if @options[:verbose]
+          @remote = chosen&.first
+        end
+
+        # --project, the lineup's, the suite's, and last the suite's name.
+        #
+        # @return [String]
+        def project = @options[:project] || @config.project || @configuration.project || @configuration.name
+
+        # Each experiment gets instances of its own.
+        #
+        # @return [Array<Trackers::Base>]
+        def for_experiment
+          context = { suite: @configuration.name, project:, project_id: @options[:project] ? nil : Trackers::Braintrust.project_id(@env) }
+          local = Trackers::Local.new(dir: File.join(Dry::CLI::UI::Invocation.root(Dir.pwd), "log", "llm-evals"), **context)
+          klass = remote
+          [local, klass&.new(**context, options: @config.tracker_options(klass.tracker_name))].compact
+        end
+      end
+    end
+  end
+end

data/lib/llm/evals/cli/upgrade_config.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    module CLI
+      # Moves a lineup file to the current format, keeping its comments and
+      # leaving the old file beside it as <file>.bak.
+      class UpgradeConfig < Command
+        desc "Rewrite the config file in the current format, keeping its comments"
+
+        example [
+          "                                  # the suite's .llm-evals.config.yml",
+          "-C evals/.llm-evals.config.yml    # another lineup file",
+          "-n                                # print the result, change nothing"
+        ]
+
+        option :config, aliases: ["-C"], desc: "The lineup file (default: the suite's .llm-evals.config.yml)"
+        option :dry_run, type: :boolean, default: false, aliases: ["-n"], desc: "Print the result, change nothing"
+
+        # Leaves a file already in the current format alone, and refuses to overwrite an earlier backup.
+        #
+        # @param config [String, nil] the lineup file; nil for the suite's
+        # @param dry_run [Boolean] print the result, change nothing
+        # @return [void]
+        # @raise [Error] when the file is missing, or its backup already exists
+        def call(config: nil, dry_run: false, **)
+          path = config ? File.expand_path(config) : configuration.lineup.path
+          raise Error, "no config file at #{path}" unless File.exist?(path)
+
+          text = File.read(path)
+          upgrade = Config::Upgrade.new(text, path:)
+          return status_box(:info, "#{path} is already in the current format") unless upgrade.needed?
+
+          upgraded = upgrade.call
+          return out.write(upgraded) if dry_run
+
+          backup = "#{path}.bak"
+          raise Error, "#{backup} already exists; move it away first" if File.exist?(backup)
+
+          File.write(backup, text)
+          replace(path, upgraded)
+          status_box(:success, "#{path} upgraded; the old file is #{backup}")
+        end
+
+        private
+
+        # A rename, so that a failed write cannot leave half a lineup.
+        #
+        # @param path [String]
+        # @param text [String]
+        # @return [void]
+        def replace(path, text)
+          temporary = "#{path}.tmp"
+          File.write(temporary, text)
+          File.rename(temporary, path)
+        end
+      end
+    end
+  end
+end

data/lib/llm/evals/cli/verify.rb

@@ -7,9 +7,13 @@ module LLM
       class Verify < Command
         desc "Check every case's input and expected answer, through the adapter and the scorer (no API calls)"
 
+        # Boxes every problem on STDERR, then fails, so a script sees the exit status.
+        #
+        # @return [void]
+        # @raise [Error] when any case has a problem
         def call(**)
           all = discovery.all
-          failures = LLM::Evals::Verify.new(configuration).problems
+          failures = LLM::Evals::Actions::Verify.new(configuration).problems
           if failures.empty?
             status_box :success, "#{all.size} cases verified across #{discovery.datasets.join(", ")}"
             return

data/lib/llm/evals/cli/version.rb

@@ -3,9 +3,13 @@
 module LLM
   module Evals
     module CLI
+      # `evals version`, also reachable as -V and --version.
       class Version < Command
         desc "Print the version"
 
+        # Prints the bare number, so a script can read it.
+        #
+        # @return [void]
         def call(**)
           out.puts LLM::Evals::VERSION
         end

data/lib/llm/evals/cli/weights.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    module CLI
+      # How the lineup's weights divide the weighted score, without calling
+      # anything: a weight is relative, so its share is what it really counts for.
+      class Weights < Command
+        desc "Print how the config file's weights divide the weighted score"
+
+        example [
+          "                                  # the suite's .llm-evals.config.yml",
+          "-C evals/.llm-evals.config.yml    # another lineup file"
+        ]
+
+        option :config, aliases: ["-C"], desc: "The lineup file (default: the suite's .llm-evals.config.yml)"
+
+        # Prints each weighted column's share, and in red the metrics no weight covers.
+        #
+        # @param config [String, nil] the lineup file; nil for the suite's
+        # @return [void]
+        # @raise [Error] when the file sets no weights
+        def call(config: nil, **)
+          lineup = config ? Config.new(File.expand_path(config)) : configuration.lineup
+          weights = lineup.score_weights
+          raise Error, "#{lineup.path}: no weighted_score.weights to show" if weights.empty?
+
+          metrics = configuration.scorer.metrics.map(&:name)
+          weighted = Scoring::WeightedScore.new(weights, metrics:)
+          rows = weighted.shares.map { |name, share| [direction(name, weighted.weights[name]), weighted.weights[name], format("%.1f%%", share * 100)] }
+          unweighted = (metrics - weighted.weights.keys).map { [pastel.red(it), "", ""] }
+          table(rows + unweighted, header: %w[Column Weight Share], fill: 0)
+        end
+
+        private
+
+        # A negative weight means less is better.
+        #
+        # @param name [String]
+        # @param weight [Numeric]
+        # @return [String] the column, marked with the direction that is good for it
+        def direction(name, weight)
+          weight.negative? ? pastel.bold.blue("↓ #{name}") : pastel.bold.yellow("↑ #{name}")
+        end
+      end
+    end
+  end
+end