llm-evals 0.4.0

data/lib/llm/evals/cli/run.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    module CLI
+      # One experiment per model and dataset, each with a per-case breakdown.
+      # Without -m, every model .llm-evals.config.yml lists. {Compare} runs the same
+      # experiments and prints only the leaderboard.
+      class Run < ModelCommand
+        desc "Run every case against a model, or every model in .llm-evals.config.yml, with a per-case breakdown"
+
+        example [
+          "-m anthropic/claude-haiku-5.5",
+          "-m openai/gpt-4.1-mini -t 0 -c clear/00        # one variant's first cases",
+          "-m google/gemini-2.5-flash -e high --local      # score without Braintrust",
+          "-d colors,animals -o                           # both datasets; the breakdowns go to log/",
+          "-n                                             # show what would run, call nothing"
+        ]
+
+        def call(models:, dataset:, dry_run:, **options)
+          config = lineup(options)
+          weigh(config)
+          contenders = contenders(config, models, options)
+          datasets = datasets(dataset)
+          return matrix(datasets, contenders) if dry_run
+
+          setup!(contenders, options)
+          runs = datasets.flat_map { |name| contenders.map { experiment(it, config, options, dataset: name) } }
+          summaries = execute(runs, concurrency: concurrency(options))
+          report(summaries,
+            output:      options[:output],
+            open:        options[:open],
+            scores:      options[:scores],
+            format:      options[:format],
+            config:,
+            options:,
+            run_options: serialize_run_options(config:, contenders:, datasets:, runs:, models:, dataset:, options:)) do |display_summaries|
+            display_summaries.each { breakdown(it, titled: display_summaries.size > 1) }
+          end
+        end
+
+        private
+
+        # One row per case: its score, and whatever the scorer diagnosed.
+        def breakdown(summary, titled: false)
+          status_box :info, summary.name if titled
+          rows = summary.results.map do |result|
+            [result.case.id, result.case.difficulty, result.score ? percent(result.score.value) : "-", diagnosis(result)]
+          end
+          table(rows, header: %w[case difficulty score diagnostics], fill: 3, wrap: true)
+        end
+
+        # The scorer's diagnostics, one per line, rendered without knowing
+        # their shape: "field: value" for a Hash, each item for an Array.
+        def diagnosis(result)
+          return result.error if result.failed?
+
+          result.score.diagnostics.map do |field, detail|
+            case detail
+            when Hash then "#{field}: #{detail.map { |k, v| "#{k}=#{v.inspect}" }.join(" ")}"
+            when Array then "#{field}: #{detail.join(", ")}"
+            else "#{field}: #{detail}"
+            end
+          end.join("\n")
+        end
+      end
+    end
+  end
+end

data/lib/llm/evals/cli/standings.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    module CLI
+      # Who ranks where: the arithmetic behind {Leaderboard}, which prints nothing.
+      class Standings
+        # How close to the best primary metric a model must come to compete on
+        # price: 0.02 is two percentage points.
+        VALUE_MARGIN = 0.02
+
+        attr_reader :metrics, :primary, :weighted
+
+        # @param metrics [Array<Metric>]
+        # @param primary [Metric] the metric the ranking reads
+        # @param weighted [WeightedScore, nil] nil when the file sets no weights
+        def initialize(metrics:, primary:, weighted: nil)
+          @metrics = metrics
+          @primary = primary
+          @weighted = weighted
+        end
+
+        # The best value: the cheapest entry within {VALUE_MARGIN} of the best
+        # primary metric. Entries with errors, or no score, do not compete. A
+        # plain score-per-dollar ratio would crown whatever is nearly free,
+        # however wrong it is.
+        #
+        # @param entries [Array<Array(Float, Float, Integer)>] primary metric, cost, errors
+        # @return [Integer, nil] the winner's index
+        def best_value(entries)
+          eligible = entries.each_index.select { entries[it][0] && entries[it][2].zero? }
+          return nil if eligible.empty?
+
+          floor = eligible.map { entries[it][0] }.max - VALUE_MARGIN
+          eligible.select { entries[it][0] >= floor - 1e-9 }.min_by { entries[it][1] }
+        end
+
+        # Each contender's numbers over the given experiments, best first.
+        #
+        # @param summaries [Array<Experiment::Summary>]
+        # @return [Array<Hash>]
+        def contender_groups(summaries)
+          groups = summaries.group_by { [it.model.key, it.model.effort, it.model.temperature] }.values.map do |group|
+            values = group.filter_map { it.mean(primary) }
+            { first: group.first, errors: group.sum { it.errors.size },
+              value: values.size == group.size ? values.sum / values.size : nil,
+              usage: group.map(&:usage).reduce(Usage.zero, :+),
+              row:   overall_row(group) }
+          end
+          scores = weighted_by(groups.map { it[:row] })
+          groups.each_with_index { |g, index| g[:weighted] = scores[index] }
+          groups.sort_by { -((weighted ? it[:weighted] : it[:value]) || -1) }
+        end
+
+        # @param groups [Array<Hash>] from {#contender_groups}
+        # @return [Integer, nil] the index of the best value
+        def best_contender(groups) = best_value(groups.map { [it[:value], it[:usage].cost_usd, it[:errors]] })
+
+        # @param summaries [Array<Experiment::Summary>]
+        # @return [Hash{Experiment::Summary => Float, nil}] by identity, each compared within its dataset
+        def weighted_scores(summaries)
+          summaries.group_by(&:dataset).values.each_with_object({}.compare_by_identity) do |group, scores|
+            group.zip(weighted_by(group.map { weighted_row(it) })).each { |summary, value| scores[summary] = value }
+          end
+        end
+
+        # The best value in each dataset's block of the leaderboard.
+        #
+        # @param ranked [Array<Experiment::Summary>]
+        # @return [Array<Integer>] indices into ranked
+        def best_per_dataset(ranked)
+          ranked.each_index.group_by { ranked[it].dataset }.values.filter_map do |indices|
+            winner = best_value(indices.map { [ranked[it].mean(primary), ranked[it].usage.cost_usd, ranked[it].errors.size] })
+            winner && indices[winner]
+          end
+        end
+
+        private
+
+        # What {WeightedScore} reads from one experiment.
+        #
+        # @param summary [Experiment::Summary]
+        # @return [Hash{String => Numeric, nil}]
+        def weighted_row(summary)
+          metrics.to_h { [it.name, summary.mean(it)] }.merge(
+            "score"  => summary.score,
+            "errors" => summary.errors.size,
+            "time"   => summary.duration,
+            "cost"   => summary.usage.cost_usd.to_f
+          )
+        end
+
+        # One contender over every dataset: errors and magnitudes add up, the rest average.
+        #
+        # @param group [Array<Experiment::Summary>]
+        # @return [Hash{String => Numeric, nil}]
+        def overall_row(group)
+          rows = group.map { weighted_row(it) }
+          rows.first.keys.to_h do |name|
+            values = rows.map { it[name] }
+            total = values.sum(&:to_f)
+            next [name, nil] if values.any?(&:nil?)
+
+            [name, ["errors", *WeightedScore::MAGNITUDES].include?(name) ? total : total / values.size]
+          end
+        end
+
+        # @param rows [Array<Hash>]
+        # @return [Array<Float, nil>] nils when the file sets no weights
+        def weighted_by(rows) = weighted ? weighted.call(rows) : Array.new(rows.size)
+      end
+    end
+  end
+end

data/lib/llm/evals/cli/verify.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    module CLI
+      # Offline checks that every case agrees with what its suite demands of it.
+      class Verify < Command
+        desc "Check every case's input and expected answer, through the adapter and the scorer (no API calls)"
+
+        def call(**)
+          all = discovery.all
+          failures = LLM::Evals::Verify.new(configuration).problems
+          if failures.empty?
+            status_box :success, "#{all.size} cases verified across #{discovery.datasets.join(", ")}"
+            return
+          end
+
+          failures.each { |name, problems| problems.each { status_box :error, "#{name}: #{it}", stream: err } }
+          raise Error, "#{failures.size} case(s) failed verification"
+        end
+      end
+    end
+  end
+end

data/lib/llm/evals/cli/version.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    module CLI
+      class Version < Command
+        desc "Print the version"
+
+        def call(**)
+          out.puts LLM::Evals::VERSION
+        end
+      end
+    end
+  end
+end

data/lib/llm/evals/cli.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    # Every command the executable answers to. The help's title and
+    # description come from the suite's configuration, which the executable
+    # loads before this module is first referenced.
+    module CLI
+      extend Dry::CLI::Registry
+
+      # What users type, so what the help and shell completion are written for.
+      # exe/llm-evals exists only for a system where `evals` is taken.
+      PROGRAM = "evals"
+
+      Dry::CLI::Help.configure do
+        configuration = LLM::Evals.configuration
+        title configuration.name == Configuration::DEFAULT_NAME ? PROGRAM : "#{PROGRAM} (#{configuration.name})"
+        types = configuration.providers.uniq(&:request_type).map { "#{it.request_type} (#{it.request_type_description})" }
+        description "#{configuration.description} Request types: #{types.join("; ")}."
+
+        columns = IO.console&.winsize&.last || 80
+        width columns.clamp(60, 120)
+        exit_code_without_arguments 0
+
+        styles do
+          heading :bold, :cyan, case: :UPPERCASE
+          usage :yellow
+          example :yellow
+          option :green
+          example_comment :bright_black
+        end
+      end
+
+      register "run", Run
+      register "compare", Compare
+      register "render", Render
+      register "cases", Cases
+      register "verify", Verify
+      register "version", Version, aliases: ["-V", "--version"]
+      register "completion", Dry::CLI::Autocomplete::Command[self, program_name: PROGRAM]
+    end
+  end
+end

data/lib/llm/evals/completion.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    # What a {Model} returns from one call: the text, what it cost, and the
+    # provider's raw response for an adapter that needs more than the text.
+    Completion = Data.define(:text, :usage, :raw) do
+      def initialize(text:, usage: Usage.zero, raw: nil) = super(text: text.to_s, usage: Usage.from(usage), raw: raw)
+    end
+  end
+end

data/lib/llm/evals/config.rb

@@ -0,0 +1,198 @@
+# frozen_string_literal: true
+
+require "yaml"
+
+module LLM
+  module Evals
+    # The suite's `.llm-evals.config.yml`: which models to compare, at which
+    # settings, and which kinds of request the suite can produce.
+    #
+    #   project: my-evals            # Braintrust project name
+    #   request_types: [prompt]      # required: what the adapter sends; a provider must match
+    #   tracker: braintrust          # optional: the one remote tracker; else the environment decides
+    #   logfire:                     # optional: a tracker's own settings
+    #     include_replies: true
+    #   require: my_gem/evals        # optional: a path (or list) to require, for a suite with no eval_helper.rb
+    #   adapter: MyGem::Evals::Adapter   # optional: constant names, instantiated with no arguments
+    #   scorer: MyGem::Evals::Scorer
+    #   defaults:
+    #     effort: default
+    #     temperature: 0.2
+    #     max_tokens: 4096
+    #     parallelism: 4
+    #   results:
+    #     autosave: true           # completed runs are saved as JSON (default)
+    #     destination: ./evals/results/
+    #     temp_dir: ./tmp           # temporary JSON used when autosave is off
+    #     display: browser          # none, tui or browser
+    #   weighted_score:              # optional: what matters most; see {WeightedScore}
+    #     weights:
+    #       score: 0.6
+    #       cost: -0.4
+    #   vendors:                   # the OpenRouter list; every entry takes a prompt
+    #     anthropic:
+    #       - model: claude-haiku-5.5
+    #         effort: [default, low]  # a list expands into one contender per value
+    #   direct:                      # a vendor's own API, one request type each
+    #     typesafe:
+    #       - model: jev-latest
+    #
+    # Each entry becomes "<vendor>/<model>", the id a provider recognises. A
+    # key under `direct:` names a provider; its entries carry that provider.
+    class Config
+      # The request types a suite may declare. `prompt` is a chat completion:
+      # system and user text in, text out. `typesafe` is TypeSafe's typed
+      # question API, whose request shape the first such suite will define.
+      REQUEST_TYPES = %w[prompt typesafe].freeze
+
+      BUILTIN = {
+        "effort" => "default", "temperature" => 0.2, "max_tokens" => 4096, "parallelism" => 4
+      }.freeze
+
+      Results = Data.define(:autosave, :destination, :temp_dir, :display_mode)
+      RESULT_DISPLAYS = %w[none tui browser].freeze
+
+      attr_reader :path, :direct_providers
+
+      # @param path [String] read when it exists; an absent file is an empty lineup
+      # @param direct_providers [Array<String>] the provider names `direct:` may use
+      def initialize(path, direct_providers: Providers::Registry.default.direct_names)
+        @path = path
+        @direct_providers = direct_providers
+        @data = File.exist?(path) ? (YAML.safe_load_file(path) || {}) : {}
+      end
+
+      # @return [String, nil] the Braintrust project name
+      def project = @data["project"]
+
+      # The remote tracker the file chooses; a list of more than one is an
+      # error, since exactly one remote tracker runs, or none.
+      #
+      # @return [String, nil]
+      # @raise [Error] on a list of several
+      def tracker
+        listed = Array(@data["tracker"]).map(&:to_s)
+        raise Error, "#{path}: tracker: names #{listed.join(", ")}; exactly one remote tracker runs, or none" if listed.size > 1
+
+        listed.first
+      end
+
+      # A tracker's own section of the file, e.g. `logfire: { include_replies: true }`.
+      #
+      # @param name [String]
+      # @return [Hash]
+      def tracker_options(name) = (@data[name.to_s].is_a?(Hash) ? @data[name.to_s] : {}).transform_keys(&:to_s)
+
+      # @return [Array<String>] the `require:` key, one path or a list
+      def requires = Array(@data["require"]).map(&:to_s)
+
+      # @return [String, nil] the `adapter:` constant name
+      def adapter_name = @data["adapter"]&.to_s
+
+      # @return [String, nil] the `scorer:` constant name
+      def scorer_name = @data["scorer"]&.to_s
+
+      # The kinds of request the suite's adapter produces. A provider whose
+      # request type is not listed cannot serve this suite, whatever model it
+      # names, so the list is required: without it every contender would be
+      # admitted on faith and fail at the first call.
+      #
+      # @return [Array<Symbol>]
+      # @raise [Error] when the key is missing, empty, or names an unknown type
+      def request_types
+        listed = Array(@data["request_types"]).map(&:to_s)
+        raise Error, "#{path}: request_types is required; list one or more of #{REQUEST_TYPES.join(", ")}" if listed.empty?
+
+        unknown = listed - REQUEST_TYPES
+        raise Error, "#{path}: unknown request_types #{unknown.join(", ")}; known: #{REQUEST_TYPES.join(", ")}" if unknown.any?
+
+        listed.map(&:to_sym)
+      end
+
+      # @return [Hash{String => Object}] the built-in defaults under the file's own
+      def defaults = BUILTIN.merge(@data.fetch("defaults", nil) || {})
+
+      # Where completed runs are saved and how they are presented. Paths are
+      # rooted at the caller's working directory so the same config works from
+      # a project root or when invoked by a task runner.
+      #
+      # @return [Results]
+      # @raise [Error] when a setting has the wrong type or an unknown display
+      def results
+        section = @data.fetch("results", {})
+        raise Error, "#{path}: results must be a mapping" unless section.is_a?(Hash)
+
+        autosave = section.fetch("autosave", true)
+        unless [true, false].include?(autosave)
+          raise Error, "#{path}: results.autosave must be true or false"
+        end
+
+        display = section.fetch("display", "tui").to_s
+        unless RESULT_DISPLAYS.include?(display)
+          raise Error, "#{path}: results.display must be one of #{RESULT_DISPLAYS.join(", ")}"
+        end
+
+        destination = section.fetch("destination", "./evals/results").to_s
+        temp_dir = section.fetch("temp_dir", "./tmp").to_s
+        Results.new(autosave:,
+          destination: File.expand_path(destination, Dir.pwd),
+          temp_dir: File.expand_path(temp_dir, Dir.pwd),
+          display_mode: display)
+      end
+
+      # @return [Hash{String => Object}] `weighted_score.weights`; see {WeightedScore}
+      # @raise [Error] when either level is not a mapping
+      def score_weights
+        section = @data["weighted_score"]
+        return {} if section.nil?
+
+        weights = section["weights"] if section.is_a?(Hash)
+        raise Error, "#{path}: weighted_score needs a weights: mapping of column to number" unless weights.is_a?(Hash)
+
+        weights.transform_keys(&:to_s)
+      end
+
+      # Every vendor/model/effort/temperature combination the file lists,
+      # `vendors:` first, then `direct:`.
+      #
+      # @param overrides [Hash{String => Object}] values from the command line; nil entries are ignored
+      # @return [Array<Contender>]
+      # @raise [Error] on a key under `direct:` that names no provider
+      def contenders(overrides = {})
+        vendors = (@data["vendors"] || {}).flat_map do |vendor, entries|
+          Array(entries).flat_map { expand("#{vendor}/#{it.fetch("model")}", it, overrides) }
+        end
+        direct = (@data["direct"] || {}).flat_map do |name, entries|
+          unless direct_providers.include?(name.to_s)
+            raise Error, "#{path}: unknown provider #{name.inspect} under direct:; known: #{direct_providers.join(", ")}"
+          end
+
+          Array(entries).flat_map { expand("#{name}/#{it.fetch("model")}", it, overrides, provider: name.to_sym) }
+        end
+        vendors + direct
+      end
+
+      # The models given on the command line, at the file's default settings.
+      #
+      # @param models [Array<String>] model ids
+      # @param overrides [Hash{String => Object}]
+      # @return [Array<Contender>]
+      def contenders_for(models, overrides = {})
+        models.flat_map { expand(it, {}, overrides) }
+      end
+
+      private
+
+      def expand(model, entry, overrides, provider: nil)
+        settings = defaults.merge(entry).merge(overrides.compact)
+        Array(settings["effort"]).product(Array(settings["temperature"])).map do |effort, temperature|
+          Contender.new(model: model,
+            effort: effort.to_s,
+            temperature: temperature,
+            max_tokens: Integer(settings["max_tokens"]),
+            provider: provider)
+        end
+      end
+    end
+  end
+end

data/lib/llm/evals/configuration.rb

@@ -0,0 +1,173 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    # Everything a suite tells the gem, through {LLM::Evals.configure}.
+    # Paths are relative to `root`, the directory the eval helper lives in.
+    class Configuration
+      DEFAULT_GLOB = "datasets/*/cases/**/input.txt"
+      DEFAULT_NAME = "llm-evals"
+      DEFAULT_CONFIG = ".llm-evals.config.yml"
+
+      attr_writer :adapter, :scorer
+      attr_reader :name, :root, :env_files, :cases_glob, :hooks, :providers, :trackers, :description
+
+      def initialize
+        @name = DEFAULT_NAME
+        @root = Dir.pwd
+        @env_files = []
+        @cases_glob = DEFAULT_GLOB
+        @config_path = DEFAULT_CONFIG
+        @hooks = []
+        @providers = Providers::Registry.default
+        @trackers = Trackers::Registry.default
+        @description = "Run a suite of cases against one or more models and score the replies."
+        @project = nil
+      end
+
+      # Names the suite. The name titles the help and tags the experiments.
+      #
+      # @param name [String]
+      # @param root [String] where .llm-evals.config.yml and datasets/ live; the helper's directory
+      # @param env_files [Array<String>] dotenv files to load, relative to root; the first that exists wins nothing,
+      #   all that exist load, and an exported variable beats them all
+      # @param description [String, nil] one line for the help's header
+      # @return [void]
+      def suite(name, root:, env_files: [], description: nil)
+        @name = name
+        @root = File.expand_path(root)
+        @env_files = env_files.map { File.expand_path(it, @root) }
+        @description = description if description
+      end
+
+      # Where the cases are: a glob relative to root ending in the input file.
+      # Each match is a case; the expected answer is `expected.json` beside it.
+      #
+      # @param glob [String]
+      # @return [void]
+      def cases(glob)
+        @cases_glob = glob
+      end
+
+      # @param object [Adapter]
+      # @return [void]
+
+      # @param object [Adapter, nil] setting; reading when nil
+      # @return [Adapter]
+      def adapter(object = nil)
+        @adapter = object unless object.nil?
+        @adapter
+      end
+
+      # @param object [Scorer, nil] setting; reading when nil
+      # @return [Scorer]
+      def scorer(object = nil)
+        @scorer = object unless object.nil?
+        @scorer
+      end
+
+      # @param object [Scorer]
+      # @return [void]
+
+      # Runs once, after the env files are loaded and before any model is
+      # called. The place to reset a library that read ENV at require time.
+      #
+      # @yield
+      # @return [void]
+      def before_run(&block)
+        @hooks << block
+      end
+
+      # Adds a provider, ahead of the defaults, so a suite's own transport
+      # wins the ids it claims.
+      #
+      # @param provider [Providers::Base]
+      # @return [void]
+      def provider(provider)
+        @providers.register(provider)
+      end
+
+      # Adds a tracker class, so a run can choose it by name with `--tracker`
+      # or `tracker:`, or have its env key select it.
+      #
+      # @param klass [Class<Tracker>]
+      # @return [void]
+      def tracker(klass)
+        @trackers.register(klass)
+      end
+
+      # @param path [String, nil] setting, relative to root; reading when nil
+      # @return [String] the absolute path of the lineup file, `.llm-evals.config.yml` at the root by default
+      def config(path = nil)
+        @config_path = path unless path.nil?
+        File.expand_path(@config_path, root)
+      end
+
+      # @param name [String, nil] setting; reading when nil
+      # @return [String, nil] the Braintrust project, when .llm-evals.config.yml does not name one
+      def project(name = nil)
+        @project = name unless name.nil?
+        @project
+      end
+
+      # @return [Discovery] the cases, as the glob finds them
+      def discovery = Discovery.new(root: root, glob: cases_glob)
+
+      # @return [Config] the lineup file
+      def lineup = Config.new(config, direct_providers: providers.direct_names)
+
+      # @return [Runtime]
+      def runtime = Runtime.new(env_files: env_files, hooks: hooks, app_name: name)
+
+      # @return [Boolean] whether an adapter and a scorer are registered
+      def complete? = !adapter.nil? && !scorer.nil?
+
+      # Fails a command that needs the suite when no helper was loaded, with
+      # the fix in the message, instead of a NoMethodError on nil.
+      #
+      # @return [self]
+      # @raise [Error]
+      def require_suite!
+        resolve_from_config!
+        return self if complete?
+
+        raise Error,
+          "no suite is configured: register an adapter and a scorer in evals/eval_helper.rb, or set " \
+          "adapter: and scorer: (with require:) in #{config}; --helper PATH or LLM_EVALS_HELPER names another helper"
+      end
+
+      # Fills whatever the helper left unset from the config file's `require:`,
+      # `adapter:` and `scorer:` keys, so a suite can be config-only: its Ruby
+      # lives in its own gem and `evals/` holds the file and the datasets. The
+      # helper, when there is one, was loaded first and wins.
+      #
+      # @return [void]
+      # @raise [Error] naming the key and the value when a require fails or a constant is unknown
+      def resolve_from_config!
+        return if complete? || !File.exist?(config)
+
+        file = lineup
+        file.requires.each do |feature|
+          require feature
+        rescue LoadError => e
+          raise Error, "#{config}: require: #{feature} could not be loaded (#{e.message}); is its gem in the bundle?"
+        end
+        @adapter ||= instantiate("adapter", file.adapter_name)
+        @scorer ||= instantiate("scorer", file.scorer_name) # rubocop:disable Naming/MemoizedInstanceVariableName
+      end
+
+      private
+
+      # @return [Object, nil] a new instance of the named constant; nil when no name is set
+      def instantiate(key, name)
+        return nil if name.nil? || name.empty?
+
+        Object.const_get(name).new
+      rescue NameError, ArgumentError => e
+        raise Error,
+          "#{config}: #{key}: #{name} could not be instantiated (#{e.class}: #{e.message}); " \
+          "name a constant the require: key loads, taking no arguments"
+      end
+    end
+  end
+end

data/lib/llm/evals/contender.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    # One model at one setting: what a single experiment runs. Built by
+    # {Config} from the lineup file, or from `-m` on the command line.
+    #
+    # `provider` is nil for a `vendors:` entry, which OpenRouter serves, and
+    # the provider's name for a `direct:` entry. It travels with the
+    # contender rather than being read back out of the id, so `typesafe/jev-router`
+    # (Jev on OpenRouter) and a direct `typesafe/jev-latest` can never be confused.
+    Contender = Data.define(:model, :effort, :temperature, :max_tokens, :provider) do
+      def initialize(model:, effort: "default", temperature: "default", max_tokens: nil, provider: nil) = super
+
+      # @return [Boolean] whether the entry came from `direct:`
+      def direct? = !provider.nil?
+
+      # @return [String] unique across the lineup: "typesafe:typesafe/jev-latest" for a direct entry
+      def key = direct? ? "#{provider}:#{model}" : model
+
+      # The settings as a {Model} takes them: "default" means nil, send nothing.
+      #
+      # @return [Hash{Symbol => Object}]
+      def settings
+        { effort:      effort.to_s == "default" || effort.nil? ? nil : effort.to_s,
+          temperature: temperature.to_s == "default" || temperature.nil? ? nil : Float(temperature),
+          max_tokens:  max_tokens.nil? ? nil : Integer(max_tokens) }
+      end
+    end
+  end
+end