llm-evals 0.4.0 → 0.5.0

data/lib/llm/evals/config.rb

@@ -24,41 +24,79 @@ module LLM
     #     autosave: true           # completed runs are saved as JSON (default)
     #     destination: ./evals/results/
     #     temp_dir: ./tmp           # temporary JSON used when autosave is off
-    #     display: browser          # none, tui or browser
-    #   weighted_score:              # optional: what matters most; see {WeightedScore}
+    #     display: both             # none, tui, browser or both (default)
+    #   errors:                      # optional: when to give up; see {Actions::ErrorBudget}
+    #     abort_after:
+    #       total: 1000
+    #       percent: 60
+    #   weighted_score:              # optional: what matters most; see {Scoring::WeightedScore}
     #     weights:
     #       score: 0.6
     #       cost: -0.4
-    #   vendors:                   # the OpenRouter list; every entry takes a prompt
-    #     anthropic:
-    #       - model: claude-haiku-5.5
-    #         effort: [default, low]  # a list expands into one contender per value
-    #   direct:                      # a vendor's own API, one request type each
-    #     typesafe:
-    #       - model: jev-latest
+    #   openrouter:                  # a provider's name; this one needs OPENROUTER_API_KEY
+    #     rate_limit: { burst: 1000, per_minute: 500 }
+    #     vendors:
+    #       anthropic:
+    #         rate_limit: { burst: 10, per_minute: 50 }
+    #         models:
+    #           - model: claude-haiku-5.5
+    #             rate_limit: { burst: 5, per_minute: 20 }
+    #             effort: [default, low]  # a list expands into one contender per value
+    #   typesafe:                    # a provider with no vendors; this one needs TYPESAFE_API_KEY
+    #     models:
+    #       - jev-1.14.0             # a bare name is the same as `- model: jev-1.14.0`
     #
-    # Each entry becomes "<vendor>/<model>", the id a provider recognises. A
-    # key under `direct:` names a provider; its entries carry that provider.
+    # An entry under `vendors:` becomes "<vendor>/<model>", the id a provider
+    # recognises; one under `models:` becomes the provider's prefix and the name.
+    # A call waits on every `rate_limit` that matches it; see {Models::Throttle}.
     class Config
       # The request types a suite may declare. `prompt` is a chat completion:
       # system and user text in, text out. `typesafe` is TypeSafe's typed
       # question API, whose request shape the first such suite will define.
       REQUEST_TYPES = %w[prompt typesafe].freeze
 
+      # The settings a contender gets when neither the file's `defaults:` nor its entry says.
       BUILTIN = {
         "effort" => "default", "temperature" => 0.2, "max_tokens" => 4096, "parallelism" => 4
       }.freeze
 
-      Results = Data.define(:autosave, :destination, :temp_dir, :display_mode)
-      RESULT_DISPLAYS = %w[none tui browser].freeze
+      # The values `results.display` accepts.
+      RESULT_DISPLAYS = %w[none tui browser both].freeze
 
-      attr_reader :path, :direct_providers
+      # The displays that draw in the terminal, and those that write the HTML report.
+      TERMINAL_DISPLAYS = %w[tui both].freeze
+      # The displays that write the HTML report.
+      BROWSER_DISPLAYS = %w[browser both].freeze
+
+      # What a lineup that says nothing gets: the terminal summary, and the HTML report opened in the browser.
+      DEFAULT_DISPLAY = "both"
+
+      attr_reader :path, :providers
+
+      # @!attribute [r] path
+      #   The lineup file, named in every error, whether or not it exists.
+      #   @return [String]
+
+      # @!attribute [r] providers
+      #   Whose names the file's sections may use.
+      #   @return [Providers::Registry]
+
+      # Reads every section a run needs before anything is spent, so a bad one
+      # fails the run rather than the report at its end.
+      #
+      # @return [void]
+      # @raise [Error] on the first bad section
+      def validate!
+        results
+        error_budget
+        score_weights
+      end
 
       # @param path [String] read when it exists; an absent file is an empty lineup
-      # @param direct_providers [Array<String>] the provider names `direct:` may use
-      def initialize(path, direct_providers: Providers::Registry.default.direct_names)
+      # @param providers [Providers::Registry] whose names the file's sections may use
+      def initialize(path, providers: Providers::Registry.default)
         @path = path
-        @direct_providers = direct_providers
+        @providers = providers
         @data = File.exist?(path) ? (YAML.safe_load_file(path) || {}) : {}
       end
 
@@ -79,8 +117,8 @@ module LLM
 
       # A tracker's own section of the file, e.g. `logfire: { include_replies: true }`.
       #
-      # @param name [String]
-      # @return [Hash]
+      # @param name [String] the tracker's name, which is also its section's key
+      # @return [Hash{String => Object}] empty when the file has no such mapping
       def tracker_options(name) = (@data[name.to_s].is_a?(Hash) ? @data[name.to_s] : {}).transform_keys(&:to_s)
 
       # @return [Array<String>] the `require:` key, one path or a list
@@ -112,87 +150,60 @@ module LLM
       # @return [Hash{String => Object}] the built-in defaults under the file's own
       def defaults = BUILTIN.merge(@data.fetch("defaults", nil) || {})
 
-      # Where completed runs are saved and how they are presented. Paths are
-      # rooted at the caller's working directory so the same config works from
-      # a project root or when invoked by a task runner.
-      #
       # @return [Results]
       # @raise [Error] when a setting has the wrong type or an unknown display
-      def results
-        section = @data.fetch("results", {})
-        raise Error, "#{path}: results must be a mapping" unless section.is_a?(Hash)
-
-        autosave = section.fetch("autosave", true)
-        unless [true, false].include?(autosave)
-          raise Error, "#{path}: results.autosave must be true or false"
-        end
-
-        display = section.fetch("display", "tui").to_s
-        unless RESULT_DISPLAYS.include?(display)
-          raise Error, "#{path}: results.display must be one of #{RESULT_DISPLAYS.join(", ")}"
-        end
+      def results = Results.from_hash(@data.fetch("results", {}), path:)
 
-        destination = section.fetch("destination", "./evals/results").to_s
-        temp_dir = section.fetch("temp_dir", "./tmp").to_s
-        Results.new(autosave:,
-          destination: File.expand_path(destination, Dir.pwd),
-          temp_dir: File.expand_path(temp_dir, Dir.pwd),
-          display_mode: display)
-      end
-
-      # @return [Hash{String => Object}] `weighted_score.weights`; see {WeightedScore}
+      # @return [Hash{String => Object}] `weighted_score.weights`; see {Scoring::WeightedScore}
       # @raise [Error] when either level is not a mapping
-      def score_weights
-        section = @data["weighted_score"]
-        return {} if section.nil?
-
-        weights = section["weights"] if section.is_a?(Hash)
-        raise Error, "#{path}: weighted_score needs a weights: mapping of column to number" unless weights.is_a?(Hash)
+      def score_weights = ScoreWeights.from_hash(@data["weighted_score"], path:)
 
-        weights.transform_keys(&:to_s)
+      # When to give up on a run; see {Actions::ErrorBudget}.
+      #
+      # @return [Actions::ErrorBudget, nil] nil when the file sets no limit
+      # @raise [Error] on a limit that is missing or out of range
+      def error_budget
+        section = @data["errors"]
+        ErrorLimits.from_hash(section, path:).budget unless section.nil?
       end
 
-      # Every vendor/model/effort/temperature combination the file lists,
-      # `vendors:` first, then `direct:`.
+      # Every vendor/model/effort/temperature combination the file lists, in
+      # the file's order.
       #
       # @param overrides [Hash{String => Object}] values from the command line; nil entries are ignored
-      # @return [Array<Contender>]
-      # @raise [Error] on a key under `direct:` that names no provider
+      # @return [Array<Models::Contender>]
+      # @raise [Error] on a section or entry the file format does not have
       def contenders(overrides = {})
-        vendors = (@data["vendors"] || {}).flat_map do |vendor, entries|
-          Array(entries).flat_map { expand("#{vendor}/#{it.fetch("model")}", it, overrides) }
-        end
-        direct = (@data["direct"] || {}).flat_map do |name, entries|
-          unless direct_providers.include?(name.to_s)
-            raise Error, "#{path}: unknown provider #{name.inspect} under direct:; known: #{direct_providers.join(", ")}"
-          end
-
-          Array(entries).flat_map { expand("#{name}/#{it.fetch("model")}", it, overrides, provider: name.to_sym) }
+        builder = ContenderBuilder.new(defaults:, overrides:)
+        listing.entries.flat_map do |entry|
+          builder.call(entry.id, entry.settings, provider: entry.provider.direct? ? entry.provider.name.to_sym : nil)
         end
-        vendors + direct
       end
 
+      # The file's `rate_limit`s, ready for calls. The same one every time, since
+      # the limits only hold if every model of a run shares them.
+      #
+      # @return [Models::Throttle]
+      # @raise [Error] on a limit that is missing a number, out of range or set twice
+      def throttle = @throttle ||= Models::Throttle.new(listing.limits)
+
       # The models given on the command line, at the file's default settings.
       #
       # @param models [Array<String>] model ids
-      # @param overrides [Hash{String => Object}]
-      # @return [Array<Contender>]
+      # @param overrides [Hash{String => Object}] values from the command line; nil entries are ignored
+      # @return [Array<Models::Contender>]
       def contenders_for(models, overrides = {})
-        models.flat_map { expand(it, {}, overrides) }
+        builder = ContenderBuilder.new(defaults:, overrides:)
+        models.flat_map { builder.call(it) }
       end
 
       private
 
-      def expand(model, entry, overrides, provider: nil)
-        settings = defaults.merge(entry).merge(overrides.compact)
-        Array(settings["effort"]).product(Array(settings["temperature"])).map do |effort, temperature|
-          Contender.new(model: model,
-            effort: effort.to_s,
-            temperature: temperature,
-            max_tokens: Integer(settings["max_tokens"]),
-            provider: provider)
-        end
-      end
+      # The provider sections, read once on first use, so a bad file fails only the commands that read them.
+      #
+      # @return [Listing]
+      # @raise [Error] on a section, entry or limit the file format does not have
+      def listing = @listing ||= Listing.new(@data, path:, providers:)
     end
   end
 end

data/lib/llm/evals/datasets/case.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    module Datasets
+      # One eval case, as {Discovery} found it on disk: the text the model is
+      # given, the answer a perfect reply would produce, and where it came from.
+      #
+      # @!attribute [r] id
+      #   @return [String] unique within its dataset, e.g. "clear/003"
+      # @!attribute [r] dataset
+      #   @return [String] the folder under datasets/, e.g. "colors"
+      # @!attribute [r] variant
+      #   @return [String] the folder under cases/, e.g. "clear" or "vague"
+      # @!attribute [r] input
+      #   @return [String] input.txt, stripped
+      # @!attribute [r] expected
+      #   @return [Object, nil] expected.json parsed; nil when missing or unparsable
+      # @!attribute [r] path
+      #   @return [String] the case's directory
+      # @!attribute [r] metadata
+      #   @return [Hash{Symbol => Object}] number, difficulty, and any `expected_error`
+      Case = Data.define(:id, :dataset, :variant, :input, :expected, :path, :metadata) do
+        # Absent for a case that sits directly under its variant's folder.
+        #
+        # @return [Integer, nil] the NNN/ level as a number, when the case has one
+        def number = metadata[:number]
+
+        # A metadata.json may override this, so it need not equal {#number}.
+        #
+        # @return [Integer] the number, else 0; what the tables sort and show difficulty by
+        def difficulty = metadata.fetch(:difficulty, number || 0)
+
+        # What reports group by, since the same id can recur in another dataset.
+        #
+        # @return [String] "dataset/id", unique across the suite
+        def key = "#{dataset}/#{id}"
+
+        # A false here is what {Actions::Verify} reports through `metadata[:expected_error]`.
+        #
+        # @return [Boolean] whether expected.json was there and parsed
+        def expected? = !expected.nil?
+      end
+    end
+  end
+end

data/lib/llm/evals/datasets/discovery.rb

@@ -0,0 +1,143 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    module Datasets
+      # Finds the cases under a root with a glob and decodes each path into a
+      # {Case}. The default layout is
+      #
+      #   datasets/<dataset>/cases/<variant>/[NNN/]input.txt
+      #   datasets/<dataset>/cases/<variant>/[NNN/]expected.json
+      #
+      # The folder after `datasets/` is the dataset, the folder holding the
+      # input is the variant, unless it is a number, in which case it is the
+      # case number and its parent is the variant.
+      class Discovery
+        # File holding a case's expected answer, beside its input.
+        EXPECTED = "expected.json"
+        # Optional file of extra case metadata, beside its input.
+        METADATA = "metadata.json"
+
+        # Where to look: `root` is absolute and `glob` is relative to it.
+        #
+        # @return [String]
+        attr_reader :root, :glob
+
+        # Reads nothing yet; {#all} does that, once.
+        #
+        # @param root [String] an absolute directory
+        # @param glob [String] relative to root
+        def initialize(root:, glob: Runtime::Configuration::DEFAULT_GLOB)
+          @root = root
+          @glob = glob
+        end
+
+        # Derived from the cases found, so a folder with no input file does not count.
+        #
+        # @return [Array<String>] every dataset that has at least one case, sorted
+        def datasets = all.map(&:dataset).uniq.sort
+
+        # {#cases}, for a caller that cannot go on without any.
+        #
+        # @param dataset [String]
+        # @param only [Array<String>, nil] case id prefixes
+        # @return [Array<Case>]
+        # @raise [Error] when nothing matches
+        def cases!(dataset:, only: [])
+          cases(dataset:, only: Array(only)).tap do |found|
+            raise Error, "no cases match #{Array(only).join(", ")} in #{dataset}" if found.empty?
+          end
+        end
+
+        # A filtered view of {#all}. Entries of `only` may themselves be comma-separated,
+        # and each matches the start of a case's id or its "dataset/id" key.
+        #
+        # @param dataset [String, nil] one dataset; nil for all of them
+        # @param only [Array<String>] id prefixes to keep ("clear", "clear/00"); empty keeps all
+        # @return [Array<Case>] sorted by dataset, variant and number
+        # @raise [Error] when the dataset does not exist
+        # @example
+        #   discovery.cases(dataset: "colors", only: ["clear/00"]).map(&:key)
+        #   # => ["colors/clear/001", "colors/clear/002"]
+        def cases(dataset: nil, only: [])
+          found = all
+          if dataset
+            raise Error, "no dataset #{dataset.inspect}; have #{datasets.join(", ")}" unless datasets.include?(dataset)
+
+            found = found.select { it.dataset == dataset }
+          end
+          only = only.flat_map { it.split(",") }.map(&:strip).reject(&:empty?)
+          found = found.select { |kase| only.any? { kase.id.start_with?(it) || kase.key.start_with?(it) } } unless only.empty?
+          found
+        end
+
+        # Every case under the root, read on the first call and kept for this instance's life.
+        #
+        # @return [Array<Case>]
+        # @raise [JSON::ParserError] when a case's metadata.json is malformed
+        def all
+          @all ||= Dir[File.join(root, glob)].map { decode(it) }
+        end
+
+        # Builds the case at one input file.
+        #
+        # @param input_path [String]
+        # @return [Case]
+        # @raise [JSON::ParserError] when the case's metadata.json is malformed
+        def decode(input_path)
+          dir = File.dirname(input_path)
+          parts = Pathname(dir).relative_path_from(Pathname(root)).each_filename.to_a
+          number = parts.last.match?(/\A\d+\z/) ? parts.last : nil
+          variant = number ? parts[-2] : parts.last
+          expected, problem = read_expected(File.join(dir, EXPECTED))
+          Case.new(
+            id:       [variant, number].compact.join("/"),
+            dataset:  dataset_of(parts),
+            variant:  variant,
+            input:    File.read(input_path).strip,
+            expected: expected,
+            path:     dir,
+            metadata: metadata(dir, number, problem)
+          )
+        end
+
+        private
+
+        # The folder after `datasets/`, else the first folder, so a glob that
+        # does not say `datasets` still groups by its top level.
+        #
+        # @param parts [Array<String>] the case directory's path segments, relative to root
+        # @return [String] the dataset's name
+        def dataset_of(parts)
+          index = parts.index("datasets")
+          index && parts[index + 1] ? parts[index + 1] : parts.first
+        end
+
+        # Reports a missing or malformed file as a problem instead of raising, so the
+        # case still loads and {Actions::Verify} can name what is wrong with it.
+        #
+        # @param path [String] where the case's expected.json should be
+        # @return [Array(Object, nil), Array(nil, String)] the parsed answer and nil, or nil and the problem
+        def read_expected(path)
+          return [nil, "#{EXPECTED} is missing"] unless File.exist?(path)
+
+          [JSON.parse(File.read(path)), nil]
+        rescue JSON::ParserError => e
+          [nil, "#{EXPECTED} is not valid JSON: #{e.message.lines.first&.strip}"]
+        end
+
+        # Derives the number and difficulty, then lets metadata.json override them.
+        #
+        # @param dir [String] the case's directory
+        # @param number [String, nil] the NNN/ folder name, when the case has one
+        # @param problem [String, nil] what is wrong with expected.json, kept as `expected_error`
+        # @return [Hash{Symbol => Object}]
+        # @raise [JSON::ParserError] when metadata.json is malformed; unlike expected.json, not caught here
+        def metadata(dir, number, problem)
+          extra = File.exist?(File.join(dir, METADATA)) ? JSON.parse(File.read(File.join(dir, METADATA))).transform_keys(&:to_sym) : {}
+          { number: number&.to_i, difficulty: number.to_i, expected_error: problem }.compact.merge(extra)
+        end
+      end
+    end
+  end
+end

data/lib/llm/evals/datasets.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    # The cases a suite evaluates, and how they are found on disk.
+    module Datasets
+    end
+  end
+end

data/lib/llm/evals/launcher.rb

@@ -5,6 +5,16 @@ module LLM
     # Starts the CLI. The executable and the specs both build one, each with
     # its own argv and streams, and it exits only through kernel.exit.
     class Launcher
+      # Commands whose output is piped or which say nothing about a suite.
+      UNANNOUNCED = %w[version -V --version completion].freeze
+      # Help prints the box after itself, and everything else before.
+      HELP = %w[help -h --help].freeze
+
+      # @param argv [Array<String>] the arguments, which may hold `--helper` and `--no-search` for {Runtime::Helper}
+      # @param stdin [IO] nothing reads it yet
+      # @param stdout [IO]
+      # @param stderr [IO]
+      # @param kernel [#exit] replaced in the specs, so a test run survives the exit
       def initialize(argv, stdin = $stdin, stdout = $stdout, stderr = $stderr, kernel = Kernel)
         @argv = argv
         @stdin = stdin
@@ -13,6 +23,9 @@ module LLM
         @kernel = kernel
       end
 
+      # Runs the command, and exits with its status.
+      #
+      # @return [void]
       def execute!
         @kernel.exit(run)
       end
@@ -20,10 +33,18 @@ module LLM
       private
 
       # dry-cli exits by itself after help or a usage error; its status is kept.
+      # @return [Integer] the exit status: the command's own, or 1 on an error, which prints
+      #   as one line unless -v asked for its backtrace
       def run
-        argv, helper = Helper.extract(@argv)
-        Helper.load!(Helper.locate(helper))
-        Dry::CLI.new(CLI).call(arguments: argv, out: @stdout, err: @stderr)
+        argv, helper = Runtime::Helper.extract(@argv)
+        argv, search = Runtime::Helper.extract_search(argv)
+        search &&= !Runtime::Helper.config_flag?(argv)
+        found = Runtime::Helper.locate(helper, search: search)
+        LLM::Evals.configuration.located_from(Dir.pwd, search: search)
+        in_home(found) do
+          Runtime::Helper.load!(found)
+          dispatch(argv)
+        end
         0
       rescue SystemExit => e
         e.status
@@ -32,6 +53,39 @@ module LLM
         @stderr.puts(e.backtrace) if @argv.intersect?(%w[-v --verbose])
         1
       end
+
+      # Runs where the search found the suite; the caller's directory comes back afterwards.
+      #
+      # @param found [Runtime::Helper::Found]
+      # @yield the work to run there
+      # @return [Object] what the block returned
+      def in_home(found, &)
+        found.home ? Dir.chdir(found.home, &) : yield
+      end
+
+      # Runs the command with the configuration's box first, or after the help.
+      #
+      # @param argv [Array<String>] the arguments dry-cli will read
+      # @return [void]
+      def dispatch(argv)
+        help = argv.empty? || argv.intersect?(HELP)
+        announce(argv) unless help
+        Dry::CLI.new(CLI).call(arguments: argv, out: @stdout, err: @stderr)
+      ensure
+        announce(argv) if help
+      end
+
+      # Says on stderr which configuration the run uses, or that there is none.
+      #
+      # @param argv [Array<String>] the arguments dry-cli will read
+      # @return [void]
+      def announce(argv)
+        return if UNANNOUNCED.include?(argv.first) || Runtime::Helper.config_flag?(argv)
+
+        @stdout.flush
+        color = @stderr.tty? && !ENV.key?("NO_COLOR")
+        @stderr.print("\n", CLI.dataset_conf_label(configuration: LLM::Evals.configuration, color: color))
+      end
     end
   end
 end

data/lib/llm/evals/models/contender.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    module Models
+      # One model at one setting: what a single experiment runs. Built by
+      # {Config} from the lineup file, or from `-m` on the command line.
+      #
+      # `provider` is nil for an entry whose id picks its provider, as OpenRouter's
+      # do, and the provider's name for a direct provider's entry. It travels with the
+      # contender rather than being read back out of the id, so `typesafe/jev-router`
+      # (Jev on OpenRouter) and a direct `typesafe/jev-latest` can never be confused.
+      #
+      # @!attribute [r] model
+      #   @return [String] the model id, as the lineup names it
+      # @!attribute [r] effort
+      #   @return [String, nil] reasoning effort; "default" or nil sends none
+      # @!attribute [r] temperature
+      #   @return [Numeric, String, nil] sampling temperature; "default" or nil sends none
+      # @!attribute [r] max_tokens
+      #   @return [Integer, nil] output token cap; nil sends none
+      # @!attribute [r] provider
+      #   @return [Symbol, nil] a direct provider's name; nil when the id picks the provider
+      Contender = Data.define(:model, :effort, :temperature, :max_tokens, :provider) do
+        # Gives everything but the model a default, so `-m` needs to name only that.
+        #
+        # @param model [String] the model id, as the lineup names it
+        # @param effort [String, nil] reasoning effort
+        # @param temperature [Numeric, String, nil] sampling temperature
+        # @param max_tokens [Integer, nil] output token cap
+        # @param provider [Symbol, nil] a direct provider's name
+        def initialize(model:, effort: "default", temperature: "default", max_tokens: nil, provider: nil) = super
+
+        # @return [Boolean] whether the entry came from a direct provider's section
+        def direct? = !provider.nil?
+
+        # @return [String] unique across the lineup: "typesafe:typesafe/jev-latest" for a direct entry
+        def key = direct? ? "#{provider}:#{model}" : model
+
+        # The settings as a {Model} takes them: "default" means nil, send nothing.
+        #
+        # @return [Hash{Symbol => Object}]
+        def settings
+          { effort:      effort.to_s == "default" || effort.nil? ? nil : effort.to_s,
+            temperature: temperature.to_s == "default" || temperature.nil? ? nil : Float(temperature),
+            max_tokens:  max_tokens.nil? ? nil : Integer(max_tokens) }
+        end
+      end
+    end
+  end
+end

data/lib/llm/evals/models/model.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    module Models
+      # One model at one setting, reached through its provider. Adapters call
+      # {#complete}; the settings an experiment is named by are read here.
+      class Model
+        # The settings every provider understands; a provider may accept more.
+        SETTINGS = %i[effort temperature max_tokens].freeze
+
+        attr_reader :id, :provider, :settings
+
+        # @!attribute [r] id
+        #   The model id as the lineup names it, with any provider prefix.
+        #   @return [String]
+
+        # @!attribute [r] provider
+        #   What answers the calls.
+        #   @return [Providers::Base]
+
+        # @!attribute [r] settings
+        #   Frozen; effort, temperature and max_tokens, and whatever else a provider accepts.
+        #   @return [Hash{Symbol => Object}]
+
+        # @param id [String] the model id as the lineup names it, e.g. "anthropic/claude-haiku-5.5"
+        # @param provider [Providers::Base] what answers {#complete}
+        # @param settings [Hash{Symbol => Object}] effort, temperature, max_tokens; nil leaves the model's default
+        # @param throttle [Throttle] what every call passes through; share one to share its limits
+        def initialize(id:, provider:, settings: {}, throttle: Throttle.new)
+          @id = id
+          @provider = provider
+          @settings = settings.transform_keys(&:to_sym).freeze
+          @throttle = throttle
+        end
+
+        # @return [String, nil] reasoning effort; nil sends none
+        def effort = settings[:effort]
+
+        # @return [Float, nil] sampling temperature; nil sends none
+        def temperature = settings[:temperature]
+
+        # @return [Integer, nil] output token cap
+        def max_tokens = settings[:max_tokens]
+
+        # @return [String] the id without the provider's prefix, as the provider's API knows it
+        def name = provider.model_name(id)
+
+        # @return [String] "provider:id", unique where two providers render the same id
+        def key = "#{provider.name}:#{id}"
+
+        # One call. Transport failures raise; they are the run's errors, not the model's.
+        #
+        # @param system [String, nil] the system prompt
+        # @param user [String] the user prompt
+        # @param json [Boolean] ask the provider for a JSON object reply where it supports that
+        # @return [Results::ProviderResponse]
+        def complete(user:, system: nil, json: true)
+          @throttle.call(self) { provider.complete(self, system: system, user: user, json: json) }
+        end
+
+        # One request of typed questions, for a provider of request type `typesafe`.
+        #
+        # @param state [Object] what the questions are about, as the suite's adapter shaped it
+        # @param questions [Hash{String => Hash}] by name: type, instructions, criteria
+        # @return [Results::ProviderResponse]
+        def ask(state:, questions:)
+          @throttle.call(self) { provider.ask(self, state: state, questions: questions) }
+        end
+
+        # @return [String] the effort as reports show it; "default" sends none
+        def effort_label = effort || "default"
+
+        # @return [Float, String] the temperature as reports show it; "default" sends none
+        def temperature_label = temperature.nil? ? "default" : temperature
+
+        # @return [String] e.g. "effort=low t=0.2"
+        def label = "effort=#{effort_label} t=#{temperature_label}"
+
+        # How an experiment's name starts.
+        #
+        # @return [String] e.g. "anthropic/claude-haiku-5.5 effort=low t=0.2"
+        def to_s = "#{id} #{label}"
+      end
+    end
+  end
+end