llm-evals 0.4.0

data/exe/evals

@@ -0,0 +1,9 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# No Bundler here: a Gemfile in the user's directory must not decide which
+# gems load. The suite's own eval_helper.rb may `require "bundler/setup"`.
+$LOAD_PATH.unshift(File.expand_path("../lib", __dir__))
+require "llm/evals"
+
+LLM::Evals::Launcher.new(ARGV).execute!

data/exe/llm-evals

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# No Bundler here: a Gemfile in the user's directory must not decide which
+# gems load. The suite's own eval_helper.rb may `require "bundler/setup"`.
+$LOAD_PATH.unshift(File.expand_path("../lib", __dir__))
+require "llm/evals"
+
+puts "\e[1;31mERROR: \e[0;33mThis gem comes with an executable named 'evals'"
+puts "       You invoked 'llm-evals'. Please use 'evals' to use the gems's CLI.\n\e[0m"
+
+puts "Thanks for using llm-evals!"
+
+exit 0

data/lib/llm/evals/adapter.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    # What a suite implements to run one case: everything the model sees, and
+    # everything done to its answer before it is scored. An adapter never
+    # scores; that is the {Scorer}'s, and keeping them apart is what lets one
+    # suite compare prompts and another compare models with the same runner.
+    #
+    # @abstract Subclass and override {#call}.
+    class Adapter
+      # Called once per dataset before its cases run; load what the dataset
+      # shares (a schema, a flow definition) here, outside the threads.
+      #
+      # @param dataset [String]
+      # @return [void]
+      def prepare(_dataset) = nil
+
+      # Runs one case against one model.
+      #
+      # @param kase [Case]
+      # @param model [Model]
+      # @return [Reply] with `error:` set when the model failed to answer; raise for a transport failure
+      def call(kase, model:)
+        raise NotImplementedError, "#{self.class}#call(kase, model:) must return an LLM::Evals::Reply"
+      end
+
+      # What the scorer needs to know about a case beyond the case itself: a
+      # schema, field types, the prompt. Passed to {Scorer#call} as `context:`.
+      #
+      # @param kase [Case]
+      # @return [Hash]
+      def context(_kase) = {}
+
+      # Offline checks of a dataset's shared files. `llm-evals verify` prints them.
+      #
+      # @param dataset [String]
+      # @return [Array<String>] problems; empty when the dataset is sound
+      def verify(_dataset) = []
+
+      # Recorded with each Braintrust experiment, e.g. which prompt version ran.
+      #
+      # @return [Hash]
+      def metadata = {}
+    end
+  end
+end

data/lib/llm/evals/case.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    # One eval case, as {Discovery} found it on disk: the text the model is
+    # given, the answer a perfect reply would produce, and where it came from.
+    #
+    # @!attribute [r] id
+    #   @return [String] unique within its dataset, e.g. "clear/003"
+    # @!attribute [r] dataset
+    #   @return [String] the folder under datasets/, e.g. "colors"
+    # @!attribute [r] variant
+    #   @return [String] the folder under cases/, e.g. "clear" or "vague"
+    # @!attribute [r] input
+    #   @return [String] input.txt, stripped
+    # @!attribute [r] expected
+    #   @return [Object, nil] expected.json parsed; nil when missing or unparsable
+    # @!attribute [r] path
+    #   @return [String] the case's directory
+    # @!attribute [r] metadata
+    #   @return [Hash{Symbol => Object}] number, difficulty, and any `expected_error`
+    Case = Data.define(:id, :dataset, :variant, :input, :expected, :path, :metadata) do
+      # @return [Integer, nil] the NNN/ level as a number, when the case has one
+      def number = metadata[:number]
+
+      # @return [Integer] the number, else 0; what the tables sort and show difficulty by
+      def difficulty = metadata.fetch(:difficulty, number || 0)
+
+      # @return [String] "dataset/id", unique across the suite
+      def key = "#{dataset}/#{id}"
+
+      # @return [Boolean] whether expected.json was there and parsed
+      def expected? = !expected.nil?
+    end
+  end
+end

data/lib/llm/evals/cli/cases.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    module CLI
+      # The cases, without calling anything.
+      class Cases < Command
+        desc "List the eval cases in every dataset"
+
+        option :dataset, type: :array, aliases: ["-d"], default: [], desc: "Only these datasets, comma-separated"
+
+        def call(dataset: [], **)
+          scorer = configuration.scorer
+          names = dataset.flat_map { it.split(",") }
+          names = discovery.datasets if names.empty?
+          rows = names.flat_map { discovery.cases(dataset: it) }.map do |kase|
+            [kase.dataset, kase.id, kase.difficulty, scorer.summary(kase) || summary(kase), kase.input.split.size]
+          end
+          raise Error, "no cases found under #{discovery.root} with #{discovery.glob}" if rows.empty?
+
+          table(rows, header: %w[dataset case difficulty expected words])
+          refusals
+        end
+
+        private
+
+        # Names, on STDERR, every lineup entry the suite cannot ask, so the
+        # mismatch is visible before anyone reaches for `run`.
+        def refusals
+          config = configuration.lineup
+          _, refused = configuration.providers.admit(config.contenders, request_types: config.request_types)
+          refused.map(&:to_s).uniq.each { status_box :warning, it, stream: err }
+        end
+
+        # The generic cell when the scorer has nothing to say: how many fields
+        # the expected answer holds, or what kind of thing it is.
+        def summary(kase)
+          case kase.expected
+          when Hash then "#{kase.expected.size} fields"
+          when Array then "#{kase.expected.size} items"
+          when nil then "-"
+          else kase.expected.class.name.downcase
+          end
+        end
+      end
+    end
+  end
+end

data/lib/llm/evals/cli/command.rb

@@ -0,0 +1,163 @@
+# frozen_string_literal: true
+
+require "io/console"
+require "pastel"
+require "tty-box"
+require "tty-table"
+
+module LLM
+  module Evals
+    module CLI
+      # What every command inherits: the ui helpers, and the flags they all share.
+      class Command < Dry::CLI::Command
+        include Dry::CLI::UI
+
+        # A cell that reads as a number: 42, 1,200, 97.1%, 3.5s, $0.0042.
+        NUMBER = /\A-?\$?\d[\d,]*(?:\.\d+)?[%s]?\z/
+
+        # Cells that stand in for a number without being one; they do not stop
+        # a column from right-aligning.
+        PLACEHOLDERS = ["", "-", "default"].freeze
+
+        # The width assumed when the output is not a terminal.
+        DEFAULT_WIDTH = 120
+
+        # The narrowest a wrapped column gets, however full the screen.
+        MIN_WRAP = 30
+
+        # Each status box's title and colors.
+        STATUS = {
+          info:    { title: " Info ", fg: :bright_white, bg: :blue },
+          success: { title: " Success ", fg: :black, bg: :green },
+          warning: { title: " Warning ", fg: :black, bg: :yellow },
+          error:   { title: " Error ", fg: :bright_white, bg: :red }
+        }.freeze
+
+        def self.inherited(subclass)
+          super
+          subclass.option :verbose, type: :boolean, default: false, aliases: ["-v"], desc: "Print the full backtrace on failure"
+        end
+
+        private
+
+        # @return [Configuration] the suite, which must be loaded
+        def configuration = LLM::Evals.configuration.require_suite!
+
+        # @return [Discovery]
+        def discovery = configuration.discovery
+
+        # ui.table, as wide as the terminal (and so as the info and warning
+        # boxes), with every all-numeric column right-aligned and a line between
+        # rows once any cell spans several. dry-cli-ui does none of that, so this
+        # renders through tty-table, as it does.
+        #
+        # The fill column takes whatever width the others leave. With wrap, its
+        # text is word-wrapped to fit; without, it is only ever widened, so an
+        # id is never broken and a table too wide for the screen stays so.
+        #
+        # @param rows [Array<Array<#to_s>>]
+        # @param header [Array<String>]
+        # @param highlight [Array<Integer>] rows drawn in bold yellow (when color is on)
+        # @param fill [Integer] the column that absorbs the spare width
+        # @param wrap [Boolean] word-wrap the fill column to fit
+        # @return [void]
+        def table(rows, header:, highlight: [], fill: 1, wrap: false)
+          return if rows.empty?
+
+          cells = rows.map { |row| row.map(&:to_s) }
+          widths = column_widths([header] + cells)
+          spare = terminal_width - (widths.sum - widths[fill]) - ((3 * widths.size) + 1)
+          if wrap
+            widths[fill] = [spare, MIN_WRAP].max
+            cells.each { it[fill] = wrap(it[fill], widths[fill]) }
+          else
+            widths[fill] = [spare, widths[fill]].max
+          end
+          alignments = header.each_index.map do |index|
+            column = cells.map { it[index] }.reject { PLACEHOLDERS.include?(it) }
+            !column.empty? && column.all? { NUMBER.match?(it) } ? :right : :left
+          end
+          multiline = cells.flatten.any? { it.include?("\n") }
+          cells = cells.each_with_index.map { |row, index| highlight.include?(index) ? row.map { pastel.bold.yellow(it) } : row }
+          rendered = TTY::Table.new(header: header.map { pastel.bold(it) }, rows: cells).render(
+            :unicode,
+            padding:       [0, 1],
+            width:         Dry::CLI::UI::Widgets::Table::UNLIMITED,
+            alignments:    alignments,
+            multiline:     multiline,
+            column_widths: widths
+          ) { |renderer| renderer.border.separator = :each_row if multiline }
+          out.puts rendered
+        end
+
+        # @return [Array<Integer>] each column's widest line
+        def column_widths(rows)
+          rows.transpose.map { |column| column.flat_map { it.to_s.split("\n") }.map(&:length).max.to_i }
+        end
+
+        # ui.info, ui.success, ui.warn and ui.error, painted on their color
+        # across the full width of the terminal. dry-cli-ui colors only a box's
+        # title, so this draws the box with tty-box, which it uses too. Without
+        # color it is a plain framed box.
+        #
+        # @param level [Symbol] a key of {STATUS}
+        # @param text [String]
+        # @param stream [IO] out by default; warnings and errors go to err
+        # @return [void]
+        def status_box(level, text, stream: out)
+          look = STATUS.fetch(level)
+          colored = color?(stream)
+          style = colored ? { fg: look[:fg], bg: look[:bg], border: { fg: look[:fg], bg: look[:bg] } } : {}
+          # tty-box drops a word wider than the box, such as a long path; cut it first
+          stream.puts TTY::Box.frame(wrap(text, terminal_width - 4),
+            width:        terminal_width,
+            padding:      [0, 1],
+            title:        { top_left: look[:title] },
+            style:        style,
+            enable_color: colored)
+        end
+
+        # A report written with -o is in color, as dry-cli-ui writes its own.
+        #
+        # @param stream [IO]
+        # @return [Boolean] whether the stream takes color
+        def color?(stream = out)
+          (stream.respond_to?(:color?) && stream.color?) || (stream.respond_to?(:tty?) && stream.tty? && !ENV.key?("NO_COLOR"))
+        end
+
+        def pastel = @pastel ||= Pastel.new(enabled: color?)
+
+        # @return [Integer] columns available on this terminal; {DEFAULT_WIDTH} when it is not one.
+        #   While writing to a report, the screen's.
+        def terminal_width
+          screen = @screen || out
+          width = screen.respond_to?(:winsize) && screen.tty? ? screen.winsize[1] : 0
+          width.positive? ? width : DEFAULT_WIDTH
+        rescue SystemCallError
+          DEFAULT_WIDTH
+        end
+
+        # Word-wraps text to a width, keeping its own line breaks. A word
+        # longer than the width is cut.
+        #
+        # @param text [String]
+        # @param width [Integer]
+        # @return [String]
+        def wrap(text, width)
+          text.split("\n").flat_map { |line| line.scan(/\S.{0,#{width - 1}}(?=\s|\z)|\S{#{width}}/).map(&:strip) }.join("\n")
+        end
+
+        # @param only [Array<String>, nil] case prefixes from --case
+        # @param dataset [String]
+        # @return [Array<Case>]
+        # @raise [Error] when nothing matches
+        def cases_for(only, dataset:)
+          list = discovery.cases(dataset: dataset, only: Array(only))
+          raise Error, "no cases match #{Array(only).join(", ")} in #{dataset}" if list.empty?
+
+          list
+        end
+      end
+    end
+  end
+end

data/lib/llm/evals/cli/compare.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    module CLI
+      # Several models at one setting, side by side: one experiment each,
+      # with either a terminal leaderboard or a saved HTML/JSON report.
+      class Compare < ModelCommand
+        desc "Run every case against several models and report the results"
+
+        example [
+          "                                     # everything .llm-evals.config.yml lists",
+          "-m anthropic/claude-haiku-5.5,openai/gpt-4o-mini -t 0",
+          "-e low -c clear -j 2                 # two experiments at a time",
+          "-n                                   # show the matrix, spend nothing"
+        ]
+
+        def call(models:, dataset:, dry_run:, **options)
+          config = lineup(options)
+          weigh(config)
+          contenders = contenders(config, models, options)
+          datasets = datasets(dataset)
+          return matrix(datasets, contenders) if dry_run
+
+          setup!(contenders, options)
+          runs = datasets.flat_map { |name| contenders.map { experiment(it, config, options, dataset: name) } }
+          summaries = execute(runs, concurrency: concurrency(options))
+          report(summaries,
+            output:      options[:output],
+            open:        options[:open],
+            scores:      options[:scores],
+            format:      options[:format],
+            config:,
+            options:,
+            run_options: serialize_run_options(config:, contenders:, datasets:, runs:, models:, dataset:, options:))
+        end
+      end
+    end
+  end
+end

data/lib/llm/evals/cli/formatting.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    module CLI
+      # How a command prints the numbers every report shares.
+      module Formatting
+        private
+
+        def percent(value) = value.nil? ? "-" : format("%.1f%%", value * 100)
+
+        def dollars(value) = format("$%.4f", value.to_f)
+      end
+    end
+  end
+end

data/lib/llm/evals/cli/leaderboard.rb

@@ -0,0 +1,183 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    module CLI
+      # The terminal summary of a run: one row per experiment, the glossary, the
+      # selected totals view, and the winners. Every column that is not a metric
+      # is the gem's; the metric columns, their glossary and the ranking come
+      # from the suite's {Scorer#metrics}.
+      class Leaderboard
+        include Formatting
+
+        # What the gem's own leaderboard columns mean, printed under it with the metrics'.
+        GLOSSARY = {
+          "effort"  => "Reasoning effort sent to the model. \"default\" sends none, and the model decides; " \
+                       "a provider that takes none shows \"default\" too.",
+          "temp"    => "Sampling temperature sent to the model. \"default\" sends none.",
+          "errors"  => "Cases whose call failed outright (an API or network error), so they have no scores at all. " \
+                       "A reply the model got wrong is not an error; it scores 0.",
+          "time"    => "Wall-clock time for the experiment. Experiments run side by side and share the network, " \
+                       "so compare times within one run only.",
+          "tok_in"  => "Input tokens sent, summed over the cases, as the provider counted them.",
+          "tok_out" => "Output tokens received, summed over the cases; reasoning tokens are included where the " \
+                       "provider bills them as output.",
+          "cost"    => "US dollars, as the provider reported them, summed over the cases.",
+          "score"   => "The mean of each case's overall score, as the scorer computed it. With --scores, a table " \
+                       "ranks every experiment by it."
+        }.freeze
+
+        # @param standings [Standings]
+        # @param table [#call] draws rows, as {Command#table}
+        # @param out [IO] where the lines between tables go
+        # @param pastel [Pastel]
+        # @param colors [ModelColors] shared with the progress bars, so a model keeps its color
+        def initialize(standings:, table:, out:, pastel:, colors:)
+          @standings = standings
+          @table = table
+          @out = out
+          @pastel = pastel
+          @colors = colors
+        end
+
+        # One row per experiment, best primary metric first, or best wscore when the
+        # file sets weights; the glossary; the selected totals view; with
+        # scores, every experiment by score; and, last, the winners.
+        #
+        # @param summaries [Array<Experiment::Summary>]
+        # @param scores [Boolean] also print every experiment by score
+        # @param totals_mode [String] `overall`, `variant` or `both`
+        # @return [void]
+        def call(summaries, scores: false, totals_mode: "overall")
+          weights = standings.weighted_scores(summaries)
+          ranked = summaries.sort_by { [it.dataset, -((weighted ? weights[it] : it.mean(primary)) || -1)] }
+          rows = ranked.map do |s|
+            [s.dataset, s.model.id, s.model.effort || "default", temperature_cell(s),
+             *(percent(weights[s]) if weighted),
+             *metrics.map { percent(s.mean(it)) },
+             s.errors.size, format("%.1fs", s.duration), s.usage.input_tokens, s.usage.output_tokens, dollars(s.usage.cost_usd)]
+          end
+          best = standings.best_per_dataset(ranked)
+          table.call(rows,
+            header:    ["dataset", "model", "effort", "temp", *("wscore" if weighted), *metrics.map(&:name),
+                        "errors", "time", "tok_in", "tok_out", "cost"],
+            highlight: best)
+          table.call(glossary, header: %w[Term Description], fill: 1, wrap: true)
+          overall = totals(summaries, mode: totals_mode)
+          score_table(summaries) if scores
+          winners("Best value: the cheapest within #{(Standings::VALUE_MARGIN * 100).round} points of the best #{primary.name}",
+            best.map { ranked[it] },
+            primary.name) { it.mean(primary) }
+          if weighted
+            leaders = ranked.group_by(&:dataset).values.filter_map { |group| group.find { weights[it] && it.errors.empty? } }
+            winners("Best wscore: by the weights in weighted_score", leaders, "wscore") { weights[it] }
+          end
+          overall&.call
+        end
+
+        # @return [Array<Array(String, String)>] the metrics' descriptions, then the gem's columns, less any a metric took the name of
+        def glossary
+          described = metrics.map { [it.name, it.description.to_s.empty? ? "(no description)" : it.description] } + GLOSSARY.reject { |term, _| metrics.any? { it.name == term } }.to_a
+          weighted ? [["wscore", weighted.description], *described] : described
+        end
+
+        private
+
+        attr_reader :standings, :table, :out, :pastel, :colors
+
+        def metrics = standings.metrics
+
+        def primary = standings.primary
+
+        def weighted = standings.weighted
+
+        def value_line(label, value, cost)
+          out.puts "best value#{label}: #{format("%.1f%%", value * 100)} at #{dollars(cost)}, " \
+                   "the cheapest within #{(Standings::VALUE_MARGIN * 100).round} points of the best #{primary.name}"
+        end
+
+        def weighted_line(label, value)
+          out.puts "best wscore#{label}: #{format("%.1f%%", value * 100)}, by the weights in weighted_score"
+        end
+
+        def contender_label(summary) = "#{summary.model.id} effort=#{summary.model.effort || "default"}"
+
+        def temperature_cell(summary) = summary.model.temperature.nil? ? "default" : summary.model.temperature
+
+        # The selected totals view: one aggregate table, one per variant, or both.
+        #
+        # @return [Proc, nil] prints the conclusions over every dataset, for after
+        #   everything else; nil over a single dataset
+        def totals(summaries, mode: "overall")
+          variants = summaries.flat_map { |summary| summary.results.map { it.case.variant } }.uniq.sort
+          several_datasets = summaries.map(&:dataset).uniq.size > 1
+          has_multiple_groups = several_datasets || variants.size > 1
+          if has_multiple_groups && %w[overall both].include?(mode)
+            out.puts "[all cases]"
+            contender_table(standings.contender_groups(summaries))
+          end
+          if has_multiple_groups && %w[variant both].include?(mode)
+            variants.each do |variant|
+              out.puts "[#{variant}]"
+              contender_table(standings.contender_groups(summaries.map { it.for_variant(variant, metrics) }))
+            end
+          end
+          return unless several_datasets
+
+          groups = standings.contender_groups(summaries)
+          best = standings.best_contender(groups)
+          winner = groups.find { it[:weighted] && it[:errors].zero? }
+          lambda do
+            value_line(" overall (#{contender_label(groups[best][:first])})", groups[best][:value], groups[best][:usage].cost_usd) if best
+            weighted_line(" overall (#{contender_label(winner[:first])})", winner[:weighted]) if weighted && winner
+          end
+        end
+
+        # @param groups [Array<Hash>] from {Standings#contender_groups}
+        # @return [void]
+        def contender_table(groups)
+          rows = groups.map do |g|
+            s = g[:first]
+            [s.model.id, s.model.effort || "default", temperature_cell(s), *(percent(g[:weighted]) if weighted), percent(g[:value]),
+             g[:errors], g[:usage].input_tokens, g[:usage].output_tokens, dollars(g[:usage].cost_usd)]
+          end
+          table.call(rows,
+            header:    ["model", "effort", "temp", *("wscore" if weighted), primary.name, "errors", "tok_in", "tok_out", "cost"],
+            highlight: [standings.best_contender(groups)].compact,
+            fill:      0)
+        end
+
+        # Every experiment by its mean score, best first, with its cost and its
+        # Braintrust page. URLs are never wrapped, so they stay clickable.
+        def score_table(summaries)
+          weights = standings.weighted_scores(summaries)
+          rows = summaries.sort_by { -(it.score || -1) }.map do |s|
+            [s.model.id, s.model.effort || "default", s.dataset, percent(s.score), *(percent(weights[s]) if weighted), dollars(s.usage.cost_usd), s.permalink || "-"]
+          end
+          table.call(rows, header: ["model", "effort", "dataset", "score", *("wscore" if weighted), "cost", "url"], fill: 0)
+        end
+
+        # One line per dataset's winner, each column aligned, with its model last in
+        # the color of its progress bars.
+        #
+        # @param title [String]
+        # @param summaries [Array<Experiment::Summary>]
+        # @param label [String] the name of what the block reads
+        # @yieldparam summary [Experiment::Summary]
+        # @yieldreturn [Float]
+        def winners(title, summaries, label)
+          return if summaries.empty?
+
+          rows = summaries.map { ["[#{it.dataset}]", "effort=#{it.model.effort || "default"}", "#{label}=#{percent(yield(it))}", dollars(it.usage.cost_usd)] }
+          widths = rows.transpose.map { |column| column.map(&:length).max }
+          out.puts title
+          rows.zip(summaries).each do |(dataset, effort, value, cost), summary|
+            model = summary.model.id
+            out.puts "  #{dataset.ljust(widths[0])}  #{effort.ljust(widths[1])}  #{value.rjust(widths[2])}  #{cost.rjust(widths[3])}  " \
+                     "#{pastel.public_send(colors[model], model)}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/llm/evals/cli/model_colors.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    module CLI
+      # Each model's bright color for one run, chosen at random and kept, so a
+      # model looks the same in its progress bars and in the tables after them.
+      class ModelColors
+        # What a model can be drawn in. More models than these share them.
+        COLORS = %i[bright_red bright_green bright_yellow bright_blue bright_magenta bright_cyan bright_white].freeze
+
+        def initialize
+          palette = COLORS.shuffle
+          @colors = Hash.new { |colors, model| colors[model] = palette[colors.size % palette.size] }
+        end
+
+        # @param model [String] the model's id
+        # @return [Symbol] a Pastel style
+        def [](model) = @colors[model]
+      end
+    end
+  end
+end