agent-evals 0.1.1

data/lib/agent/evals/adapter.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Agent
+  module Evals
+    # What a suite implements to run one case: everything the model sees, and
+    # everything done to its answer before it is scored. An adapter never
+    # scores; that is the {Scorer}'s, and keeping them apart is what lets one
+    # suite compare prompts and another compare models with the same runner.
+    #
+    # @abstract Subclass and override {#call}.
+    class Adapter
+      # Called once per dataset before its cases run; load what the dataset
+      # shares (a schema, a flow definition) here, outside the threads.
+      #
+      # @param dataset [String]
+      # @return [void]
+      def prepare(_dataset) = nil
+
+      # Runs one case against one model.
+      #
+      # @param kase [Case]
+      # @param model [Model]
+      # @return [Reply] with `error:` set when the model failed to answer; raise for a transport failure
+      def call(kase, model:)
+        raise NotImplementedError, "#{self.class}#call(kase, model:) must return an Agent::Evals::Reply"
+      end
+
+      # What the scorer needs to know about a case beyond the case itself: a
+      # schema, field types, the prompt. Passed to {Scorer#call} as `context:`.
+      #
+      # @param kase [Case]
+      # @return [Hash]
+      def context(_kase) = {}
+
+      # Offline checks of a dataset's shared files. `agent-evals verify` prints them.
+      #
+      # @param dataset [String]
+      # @return [Array<String>] problems; empty when the dataset is sound
+      def verify(_dataset) = []
+
+      # Recorded with each Braintrust experiment, e.g. which prompt version ran.
+      #
+      # @return [Hash]
+      def metadata = {}
+    end
+  end
+end

data/lib/agent/evals/case.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Agent
+  module Evals
+    # One eval case, as {Discovery} found it on disk: the text the model is
+    # given, the answer a perfect reply would produce, and where it came from.
+    #
+    # @!attribute [r] id
+    #   @return [String] unique within its dataset, e.g. "clear/003"
+    # @!attribute [r] dataset
+    #   @return [String] the folder under datasets/, e.g. "colors"
+    # @!attribute [r] variant
+    #   @return [String] the folder under cases/, e.g. "clear" or "vague"
+    # @!attribute [r] input
+    #   @return [String] input.txt, stripped
+    # @!attribute [r] expected
+    #   @return [Object, nil] expected.json parsed; nil when missing or unparsable
+    # @!attribute [r] path
+    #   @return [String] the case's directory
+    # @!attribute [r] metadata
+    #   @return [Hash{Symbol => Object}] number, difficulty, and any `expected_error`
+    Case = Data.define(:id, :dataset, :variant, :input, :expected, :path, :metadata) do
+      # @return [Integer, nil] the NNN/ level as a number, when the case has one
+      def number = metadata[:number]
+
+      # @return [Integer] the number, else 0; what the tables sort and show difficulty by
+      def difficulty = metadata.fetch(:difficulty, number || 0)
+
+      # @return [String] "dataset/id", unique across the suite
+      def key = "#{dataset}/#{id}"
+
+      # @return [Boolean] whether expected.json was there and parsed
+      def expected? = !expected.nil?
+    end
+  end
+end

data/lib/agent/evals/cli/cases.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Agent
+  module Evals
+    module CLI
+      # The cases, without calling anything.
+      class Cases < Command
+        desc "List the eval cases in every dataset"
+
+        option :dataset, type: :array, aliases: ["-d"], default: [], desc: "Only these datasets, comma-separated"
+
+        def call(dataset: [], **)
+          scorer = configuration.scorer
+          names = dataset.flat_map { it.split(",") }
+          names = discovery.datasets if names.empty?
+          rows = names.flat_map { discovery.cases(dataset: it) }.map do |kase|
+            [kase.dataset, kase.id, kase.difficulty, scorer.summary(kase) || summary(kase), kase.input.split.size]
+          end
+          raise Error, "no cases found under #{discovery.root} with #{discovery.glob}" if rows.empty?
+
+          table(rows, header: %w[dataset case difficulty expected words])
+          refusals
+        end
+
+        private
+
+        # Names, on STDERR, every lineup entry the suite cannot ask, so the
+        # mismatch is visible before anyone reaches for `run`.
+        def refusals
+          config = configuration.lineup
+          _, refused = configuration.providers.admit(config.contenders, request_types: config.request_types)
+          refused.map(&:to_s).uniq.each { status_box :warning, it, stream: err }
+        end
+
+        # The generic cell when the scorer has nothing to say: how many fields
+        # the expected answer holds, or what kind of thing it is.
+        def summary(kase)
+          case kase.expected
+          when Hash then "#{kase.expected.size} fields"
+          when Array then "#{kase.expected.size} items"
+          when nil then "-"
+          else kase.expected.class.name.downcase
+          end
+        end
+      end
+    end
+  end
+end

data/lib/agent/evals/cli/command.rb

@@ -0,0 +1,163 @@
+# frozen_string_literal: true
+
+require "io/console"
+require "pastel"
+require "tty-box"
+require "tty-table"
+
+module Agent
+  module Evals
+    module CLI
+      # What every command inherits: the ui helpers, and the flags they all share.
+      class Command < Dry::CLI::Command
+        include Dry::CLI::UI
+
+        # A cell that reads as a number: 42, 1,200, 97.1%, 3.5s, $0.0042.
+        NUMBER = /\A-?\$?\d[\d,]*(?:\.\d+)?[%s]?\z/
+
+        # Cells that stand in for a number without being one; they do not stop
+        # a column from right-aligning.
+        PLACEHOLDERS = ["", "-", "default"].freeze
+
+        # The width assumed when the output is not a terminal.
+        DEFAULT_WIDTH = 120
+
+        # The narrowest a wrapped column gets, however full the screen.
+        MIN_WRAP = 30
+
+        # Each status box's title and colors.
+        STATUS = {
+          info:    { title: " Info ", fg: :bright_white, bg: :blue },
+          success: { title: " Success ", fg: :black, bg: :green },
+          warning: { title: " Warning ", fg: :black, bg: :yellow },
+          error:   { title: " Error ", fg: :bright_white, bg: :red }
+        }.freeze
+
+        def self.inherited(subclass)
+          super
+          subclass.option :verbose, type: :boolean, default: false, aliases: ["-v"], desc: "Print the full backtrace on failure"
+        end
+
+        private
+
+        # @return [Configuration] the suite, which must be loaded
+        def configuration = Agent::Evals.configuration.require_suite!
+
+        # @return [Discovery]
+        def discovery = configuration.discovery
+
+        # ui.table, as wide as the terminal (and so as the info and warning
+        # boxes), with every all-numeric column right-aligned and a line between
+        # rows once any cell spans several. dry-cli-ui does none of that, so this
+        # renders through tty-table, as it does.
+        #
+        # The fill column takes whatever width the others leave. With wrap, its
+        # text is word-wrapped to fit; without, it is only ever widened, so an
+        # id is never broken and a table too wide for the screen stays so.
+        #
+        # @param rows [Array<Array<#to_s>>]
+        # @param header [Array<String>]
+        # @param highlight [Array<Integer>] rows drawn in bold yellow (when color is on)
+        # @param fill [Integer] the column that absorbs the spare width
+        # @param wrap [Boolean] word-wrap the fill column to fit
+        # @return [void]
+        def table(rows, header:, highlight: [], fill: 1, wrap: false)
+          return if rows.empty?
+
+          cells = rows.map { |row| row.map(&:to_s) }
+          widths = column_widths([header] + cells)
+          spare = terminal_width - (widths.sum - widths[fill]) - ((3 * widths.size) + 1)
+          if wrap
+            widths[fill] = [spare, MIN_WRAP].max
+            cells.each { it[fill] = wrap(it[fill], widths[fill]) }
+          else
+            widths[fill] = [spare, widths[fill]].max
+          end
+          alignments = header.each_index.map do |index|
+            column = cells.map { it[index] }.reject { PLACEHOLDERS.include?(it) }
+            !column.empty? && column.all? { NUMBER.match?(it) } ? :right : :left
+          end
+          multiline = cells.flatten.any? { it.include?("\n") }
+          cells = cells.each_with_index.map { |row, index| highlight.include?(index) ? row.map { pastel.bold.yellow(it) } : row }
+          rendered = TTY::Table.new(header: header.map { pastel.bold(it) }, rows: cells).render(
+            :unicode,
+            padding:       [0, 1],
+            width:         Dry::CLI::UI::Widgets::Table::UNLIMITED,
+            alignments:    alignments,
+            multiline:     multiline,
+            column_widths: widths
+          ) { |renderer| renderer.border.separator = :each_row if multiline }
+          out.puts rendered
+        end
+
+        # @return [Array<Integer>] each column's widest line
+        def column_widths(rows)
+          rows.transpose.map { |column| column.flat_map { it.to_s.split("\n") }.map(&:length).max.to_i }
+        end
+
+        # ui.info, ui.success, ui.warn and ui.error, painted on their color
+        # across the full width of the terminal. dry-cli-ui colors only a box's
+        # title, so this draws the box with tty-box, which it uses too. Without
+        # color it is a plain framed box.
+        #
+        # @param level [Symbol] a key of {STATUS}
+        # @param text [String]
+        # @param stream [IO] out by default; warnings and errors go to err
+        # @return [void]
+        def status_box(level, text, stream: out)
+          look = STATUS.fetch(level)
+          colored = color?(stream)
+          style = colored ? { fg: look[:fg], bg: look[:bg], border: { fg: look[:fg], bg: look[:bg] } } : {}
+          # tty-box drops a word wider than the box, such as a long path; cut it first
+          stream.puts TTY::Box.frame(wrap(text, terminal_width - 4),
+            width:        terminal_width,
+            padding:      [0, 1],
+            title:        { top_left: look[:title] },
+            style:        style,
+            enable_color: colored)
+        end
+
+        # A report written with -o is in color, as dry-cli-ui writes its own.
+        #
+        # @param stream [IO]
+        # @return [Boolean] whether the stream takes color
+        def color?(stream = out)
+          (stream.respond_to?(:color?) && stream.color?) || (stream.respond_to?(:tty?) && stream.tty? && !ENV.key?("NO_COLOR"))
+        end
+
+        def pastel = @pastel ||= Pastel.new(enabled: color?)
+
+        # @return [Integer] columns available on this terminal; {DEFAULT_WIDTH} when it is not one.
+        #   While writing to a report, the screen's.
+        def terminal_width
+          screen = @screen || out
+          width = screen.respond_to?(:winsize) && screen.tty? ? screen.winsize[1] : 0
+          width.positive? ? width : DEFAULT_WIDTH
+        rescue SystemCallError
+          DEFAULT_WIDTH
+        end
+
+        # Word-wraps text to a width, keeping its own line breaks. A word
+        # longer than the width is cut.
+        #
+        # @param text [String]
+        # @param width [Integer]
+        # @return [String]
+        def wrap(text, width)
+          text.split("\n").flat_map { |line| line.scan(/\S.{0,#{width - 1}}(?=\s|\z)|\S{#{width}}/).map(&:strip) }.join("\n")
+        end
+
+        # @param only [Array<String>, nil] case prefixes from --case
+        # @param dataset [String]
+        # @return [Array<Case>]
+        # @raise [Error] when nothing matches
+        def cases_for(only, dataset:)
+          list = discovery.cases(dataset: dataset, only: Array(only))
+          raise Error, "no cases match #{Array(only).join(", ")} in #{dataset}" if list.empty?
+
+          list
+        end
+      end
+    end
+  end
+end

data/lib/agent/evals/cli/compare.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Agent
+  module Evals
+    module CLI
+      # Several models at one setting, side by side: one experiment each,
+      # and only the leaderboard.
+      class Compare < ModelCommand
+        desc "Run every case against several models and print a leaderboard"
+
+        option :concurrency, default: "0", desc: "Models running at once; 0 runs them all together"
+
+        example [
+          "                                     # everything .agent-eval.config.yml lists",
+          "-m anthropic/claude-haiku-5.5,openai/gpt-4o-mini -t 0",
+          "-e low -c clear --concurrency 2",
+          "-n                                   # show the matrix, spend nothing"
+        ]
+
+        def call(models:, dataset:, dry_run:, concurrency:, **options)
+          config = lineup(options)
+          contenders = contenders(config, models, options)
+          datasets = datasets(dataset)
+          return matrix(datasets, contenders) if dry_run
+
+          setup!(contenders, options)
+          runs = datasets.flat_map { |name| contenders.map { experiment(it, config, options, dataset: name) } }
+          report(execute(runs, concurrency: Integer(concurrency)), output: options[:output], open: options[:open])
+        end
+      end
+    end
+  end
+end