llm-evals 0.4.0 → 0.5.0

data/lib/llm/evals/runtime.rb

@@ -1,74 +1,9 @@
 # frozen_string_literal: true
 
-require "dotenv"
-
 module LLM
   module Evals
-    # One-time process setup: secrets from the suite's env files, a check that
-    # every key and gem the run needs is there, the suite's own hooks, and
-    # the providers' clients.
-    class Runtime
-      attr_reader :env_files, :hooks, :app_name
-
-      # @param env_files [Array<String>] absolute paths; those that exist load
-      # @param hooks [Array<Proc>] the suite's `before_run` blocks
-      # @param app_name [String] what a provider tells its API the caller is
-      def initialize(env_files: [], hooks: [], app_name: "llm-evals")
-        @env_files = env_files
-        @hooks = hooks
-        @app_name = app_name
-      end
-
-      # Loads the env files that exist, without overriding what is already
-      # set, so an exported variable beats the file.
-      #
-      # @return [void]
-      def load_env
-        files = env_files.select { File.exist?(it) }
-        Dotenv.load(*files) unless files.empty?
-      end
-
-      # Fails before anything is called when a key the run needs is neither
-      # exported nor in an env file.
-      #
-      # @param names [Array<String>]
-      # @return [void]
-      # @raise [Error] naming every missing variable
-      def require_env!(names)
-        missing = names.uniq.select { ENV[it].to_s.strip.empty? }
-        return if missing.empty?
-
-        found = env_files.select { File.exist?(it) }
-        where = if found.empty?
-                  env_files.empty? ? "no env files are configured" : "there is no #{env_files.join(" or ")}"
-                else
-                  "checked the environment, then #{found.join(", ")}"
-                end
-        raise Error,
-          "missing #{missing.join(", ")}: export them or add them to an env file (#{where}); " \
-          "an exported blank value wins over the file"
-      end
-
-      # @param providers [Array<Providers::Base>] the providers the run's models use
-      # @param tracker [Class<Tracker>, nil] the remote tracker chosen, whose gems and key are checked too
-      # @return [void]
-      # @raise [Error] when a needed variable or gem is missing
-      def setup!(providers: [], tracker: nil)
-        load_env
-        require_env!(providers.flat_map(&:env_keys))
-        tracker&.requirements!(ENV)
-        hooks.each(&:call) # after load_env: a hook usually re-reads ENV
-        providers.each { it.setup!(app_name: app_name) }
-      end
-
-      # Opens a URL in the desktop browser.
-      #
-      # @param url [String]
-      # @return [Boolean] whether the opener ran
-      def browse(url)
-        opener = RUBY_PLATFORM.include?("darwin") ? "open" : "xdg-open"
-        system(opener, url, out: File::NULL, err: File::NULL) || false
-      end
+    # How a suite is loaded and set up: its settings, its eval helper, its environment.
+    module Runtime
     end
   end
 end

data/lib/llm/evals/scoring/metric.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    module Scoring
+      # One number a {Scorer} reports per case. `aggregate` is how the run
+      # combines them (only :mean today), skipping nil, so a metric a case does
+      # not apply to never drags its mean down with a free 0 or up with a free 1.
+      # The `primary` metric ranks the leaderboard and picks the best value.
+      #
+      # @!attribute [r] name
+      #   @return [String] the column's key
+      # @!attribute [r] description
+      #   @return [String] the glossary's line
+      # @!attribute [r] aggregate
+      #   @return [Symbol] one of {AGGREGATES}
+      # @!attribute [r] primary
+      #   @return [Boolean] whether it ranks the leaderboard
+      class Metric < Data.define(:name, :description, :aggregate, :primary)
+        # The ways {#combine} can be asked to combine values.
+        AGGREGATES = %i[mean].freeze
+
+        # Turns the name into a String, so a metric can be declared with a Symbol.
+        #
+        # @param name [String, Symbol] the column's key
+        # @param description [String] the glossary's line
+        # @param aggregate [Symbol] one of {AGGREGATES}
+        # @param primary [Boolean] whether it ranks the leaderboard
+        # @raise [ArgumentError] on an aggregate outside {AGGREGATES}
+        def initialize(name:, description: "", aggregate: :mean, primary: false)
+          raise ArgumentError, "aggregate must be one of #{AGGREGATES.join(", ")}" unless AGGREGATES.include?(aggregate)
+
+          super(name: name.to_s, description: description, aggregate: aggregate, primary: primary)
+        end
+
+        # Averages what the cases reported, skipping nil.
+        #
+        # @param values [Array<Float, nil>] one per case
+        # @return [Float, nil] nil when no case reported the metric
+        def combine(values)
+          present = values.compact
+          present.sum.fdiv(present.size) unless present.empty?
+        end
+      end
+    end
+  end
+end

data/lib/llm/evals/scoring/schema_validator.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    module Scoring
+      # Checks a Hash of answers against a field map, for scorers and verifiers
+      # whose expected values are typed and closed. Nothing here is specific to
+      # what the fields mean.
+      #
+      # The map is `{ "name" => { type:, enum:, nullable: } }`; `type` is one of
+      # {TYPES}, `enum` lists the allowed values for enum and multi_enum, and a
+      # nullable field accepts nil (the text never answered it).
+      class SchemaValidator
+        # The types a field may declare.
+        TYPES = %i[boolean integer number string enum multi_enum].freeze
+
+        # The map as given, with String names, and each spec normalized to type, enum and nullable.
+        #
+        # @return [Hash{String => Hash{Symbol => Object}}]
+        attr_reader :fields
+
+        # @param fields [Hash{String, Symbol => Hash}]
+        # @raise [ArgumentError] on a type outside {TYPES}, since a typo there would pass everything
+        def initialize(fields)
+          @fields = fields.to_h { |name, spec| [name.to_s, normalize(name, spec)] }.freeze
+        end
+
+        # @return [Array<String>] the field names
+        def names = fields.keys
+
+        # @param values [Hash] String or Symbol keys
+        # @return [Array<String>] every problem found; empty when the values fit the map
+        def problems(values)
+          return ["#{values.inspect} is not a Hash"] unless values.is_a?(Hash)
+
+          actual = values.transform_keys(&:to_s)
+          missing = names - actual.keys
+          extra = actual.keys - names
+          problems = []
+          problems << "missing #{missing.join(", ")}" if missing.any?
+          problems << "unknown fields #{extra.join(", ")}" if extra.any?
+          fields.each do |name, spec|
+            next unless actual.key?(name)
+
+            problem = value_problem(spec, actual[name])
+            problems << "#{name}: #{problem}" if problem
+          end
+          problems
+        end
+
+        # @param values [Hash] String or Symbol keys
+        # @return [Boolean] whether {#problems} finds none
+        def valid?(values) = problems(values).empty?
+
+        # @param spec [Hash] one field's normalized spec
+        # @param value [Object]
+        # @return [String, nil] what is wrong with the value, or nil
+        def value_problem(spec, value)
+          return (spec[:nullable] ? nil : "is null") if value.nil?
+
+          case spec[:type]
+          when :boolean then "#{value.inspect} is not a boolean" unless [true, false].include?(value)
+          when :integer then "#{value.inspect} is not an integer" unless value.is_a?(Integer)
+          when :number then "#{value.inspect} is not a number" unless value.is_a?(Numeric)
+          when :string then "#{value.inspect} is not a string" unless value.is_a?(String)
+          when :enum then "#{value.inspect} is not one of #{spec[:enum].join("|")}" unless spec[:enum].include?(value)
+          when :multi_enum then multi_enum_problem(spec, value)
+          end
+        end
+
+        private
+
+        # Checks that a multi_enum is a list holding only allowed values.
+        #
+        # @param spec [Hash] one field's normalized spec
+        # @param value [Object] the field's non-nil value
+        # @return [String, nil] what is wrong with the value, or nil
+        def multi_enum_problem(spec, value)
+          return "#{value.inspect} is not an array" unless value.is_a?(Array)
+
+          stray = value - spec[:enum]
+          "#{stray.join(", ")} not in #{spec[:enum].join("|")}" if stray.any?
+        end
+
+        # Accepts `values:` as a synonym for `enum:`, and makes allowed values Strings.
+        #
+        # @param name [String, Symbol] the field, for the error
+        # @param spec [Hash] the field's spec, with String or Symbol keys
+        # @return [Hash{Symbol => Object}] with type, enum and nullable
+        # @raise [ArgumentError] on a type outside {TYPES}
+        # @raise [KeyError] when the spec has no type
+        def normalize(name, spec)
+          spec = spec.transform_keys(&:to_sym)
+          type = spec.fetch(:type).to_sym
+          raise ArgumentError, "#{name}: unknown type #{type.inspect}, expected one of #{TYPES.join(", ")}" unless TYPES.include?(type)
+
+          { type: type, enum: Array(spec[:enum] || spec[:values]).map(&:to_s), nullable: spec.fetch(:nullable, false) }
+        end
+      end
+    end
+  end
+end

data/lib/llm/evals/scoring/scorer.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    module Scoring
+      # What a suite implements to score one reply. Pure and deterministic: it
+      # reads the case, the reply and the adapter's context, and calls nothing.
+      #
+      # @abstract Subclass and override {#metrics} and {#call}.
+      class Scorer
+        # The metrics every {Results::Score} this scorer returns carries. The CLI builds
+        # its columns, its glossary and its Braintrust scorers from this list,
+        # so a metric missing here is a metric nobody sees.
+        #
+        # @return [Array<Metric>]
+        # @raise [NotImplementedError] until a subclass overrides it
+        def metrics
+          raise NotImplementedError, "#{self.class}#metrics must return an Array of LLM::Evals::Scoring::Metric"
+        end
+
+        # @return [Metric] the one flagged primary, else the first
+        def primary = metrics.find(&:primary) || metrics.first
+
+        # @param kase [Datasets::Case]
+        # @param reply [Results::Reply] never one with `error` set; the runner scores those 0 itself
+        # @param context [Hash] what {Actions::Adapter#context} returned for the case
+        # @return [Results::Score]
+        # @raise [NotImplementedError] until a subclass overrides it
+        def call(kase, reply, context:)
+          raise NotImplementedError, "#{self.class}#call(kase, reply, context:) must return an LLM::Evals::Results::Score"
+        end
+
+        # Offline checks of one case against what the scorer will demand of it,
+        # so a typo in expected.json is caught before it is billed as a model's mistake.
+        #
+        # @param kase [Datasets::Case]
+        # @param context [Hash] what {Actions::Adapter#context} returned for the case
+        # @return [Array<String>] problems; empty when the case is sound
+        def verify(kase, context:) = [] # rubocop:disable Lint/UnusedMethodArgument -- the signature is the contract
+
+        # One short cell describing the case in `llm-evals cases`, e.g. "5/8 answered".
+        #
+        # @param _kase [Datasets::Case] the case to describe
+        # @return [String, nil] nil for the generic fallback
+        def summary(_kase) = nil
+      end
+    end
+  end
+end

data/lib/llm/evals/scoring/weighted_score.rb

@@ -0,0 +1,148 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    module Scoring
+      # One number from the leaderboard's columns, weighted as the lineup file
+      # says, for a suite to state what matters most to it. A positive weight
+      # means higher is better; a negative one, lower is better:
+      #
+      #   weighted_score:
+      #     weights:
+      #       score: 0.4
+      #       hallucinated_rate: -0.2
+      #       cost: -0.2
+      #
+      #   weighted_score = Σ(|weight| × goodness) / Σ|weight|
+      #
+      # Goodness runs from 0 to 1. Metrics are used as they are, or as 1 - value
+      # under a negative weight. {MAGNITUDES}
+      # are placed on a log scale between the lowest and highest in the group, so
+      # that each doubling costs the same: on a linear scale, one model a hundred
+      # times the price would squash all the others together. Where the whole
+      # group is equal, each gets 1.
+      #
+      # Errors come off the total, when weighted: 20 points for one, 25 for two, 30
+      # for three, and so on, down to 0. A failed case has no scores, so no other
+      # column notices it, and a single failure must show. A clean contender gets
+      # the whole of the weight on errors.
+      #
+      # Magnitudes are relative, so a weighted score compares within one group
+      # only: another model in the lineup moves everyone's.
+      class WeightedScore
+        # Columns compared on a log scale.
+        MAGNITUDES = %w[time cost latency p95].freeze
+
+        # Columns that add up over a contender's datasets; the rest average, which
+        # for p95 is an approximation good enough to rank by.
+        SUMMED = %w[errors time cost].freeze
+
+        # The gem's columns that can be weighted, beside the metrics. `temp` is a
+        # setting, not a result, and tokens are already in the cost.
+        COLUMNS = ["score", "errors", *MAGNITUDES].freeze
+
+        # What errors take off the total: {ERROR_BASE} + {ERROR_STEP} per error.
+        ERROR_BASE = 0.15
+        # The second part of the errors' penalty; see {ERROR_BASE}.
+        ERROR_STEP = 0.05
+
+        # The weights as the lineup gives them, without those that are zero.
+        #
+        # @return [Hash{String => Numeric}] by column
+        attr_reader :weights
+
+        # @param weights [Hash{String => Numeric}]
+        # @param metrics [Array<String>] the scorer's metric names
+        # @raise [Error] on a weight that is not a number, an unknown column, or no weight at all
+        def initialize(weights, metrics:)
+          known = (metrics + COLUMNS).uniq
+          unknown = weights.keys - known
+          raise Error, "weighted_score: unknown columns #{unknown.join(", ")}; known: #{known.join(", ")}" if unknown.any?
+
+          invalid = weights.reject { |_name, weight| weight.is_a?(Numeric) }.keys
+          raise Error, "weighted_score: weights must be numbers: #{invalid.join(", ")}" if invalid.any?
+
+          raise Error, "weighted_score: errors must have a negative weight, since fewer is better" if weights.fetch("errors", -1).positive?
+
+          @weights = weights.reject { |_name, weight| weight.zero? }
+          raise Error, "weighted_score: every weight is zero" if @weights.empty?
+        end
+
+        # A column no row has does not count. A row missing a column the others
+        # have gets nil: it was not measured on everything they were.
+        #
+        # @param rows [Array<Hash{String => Numeric, nil}>] one per contender, compared with each other
+        # @return [Array<Float, nil>] in the order given
+        def call(rows)
+          present = weights.select { |name, _weight| rows.any? { it[name] } }
+          total = present.values.sum(&:abs)
+          rows.map do |row|
+            next if present.empty? || present.keys.any? { row[it].nil? }
+
+            score = present.sum { |name, weight| weight.abs * goodness(name, row[name], weight, rows) } / total
+            [score - penalty(row["errors"], present), 0.0].max
+          end
+        end
+
+        # What each weight really counts for, since weights need not add up to 1.
+        #
+        # @return [Hash{String => Float}] by column, each from 0 to 1, largest first
+        def shares
+          total = weights.values.sum(&:abs).to_f
+          weights.sort_by { |_name, weight| -weight.abs }.to_h { |name, weight| [name, weight.abs / total] }
+        end
+
+        # @return [String] the formula, with this lineup's weights, for the glossary
+        def description
+          "Σ(|weight| × goodness) / Σ|weight|, from 0% to 100%, and higher is better. A negative weight means that column is better low. " \
+            "Errors come off the total: 20 points for one, then 5 more for each further error. " \
+            "#{MAGNITUDES.join(", ")} are on a log scale within the group, so compare wscore within one table only. " \
+            "Weights: #{weights.map { |name, weight| "#{name} #{weight}" }.join(", ")}."
+        end
+
+        private
+
+        # What errors take off the total, when errors carry a weight.
+        #
+        # @param errors [Numeric, nil] the row's error count; only nil when errors are not weighted
+        # @param present [Hash{String => Numeric}] the weights of the columns some row has
+        # @return [Float] 0.0 for a clean row
+        def penalty(errors, present)
+          return 0.0 unless present.key?("errors") && errors.positive?
+
+          ERROR_BASE + (ERROR_STEP * errors)
+        end
+
+        # How good one value is, from 0 to 1, with its weight's sign applied.
+        #
+        # @param name [String] the column
+        # @param value [Numeric] the row's value
+        # @param weight [Numeric] the column's weight, not zero
+        # @param rows [Array<Hash{String => Numeric, nil}>] the group it is compared with
+        # @return [Float] 1.0 for errors, which {#penalty} accounts for, and where the group is equal
+        def goodness(name, value, weight, rows)
+          return 1.0 if name == "errors"
+
+          position = MAGNITUDES.include?(name) ? position(value, rows.filter_map { it[name] }) : value.to_f
+          return 1.0 if position.nil?
+
+          weight.negative? ? 1.0 - position : position
+        end
+
+        # A zero has no logarithm, so it stands at half the lowest above zero.
+        #
+        # @param value [Numeric] the value to place
+        # @param values [Array<Numeric>] the group's values, the value among them
+        # @return [Float, nil] from 0 at the lowest to 1 at the highest; nil when all are equal
+        def position(value, values)
+          highest = values.max
+          return nil if values.min == highest
+
+          lowest = values.select(&:positive?).min
+          lowest /= 2.0 if values.any?(&:zero?)
+          (Math.log([value, lowest].max) - Math.log(lowest)) / (Math.log(highest) - Math.log(lowest))
+        end
+      end
+    end
+  end
+end

data/lib/llm/evals/scoring.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    # How a reply is judged: the suite's scorer, its metrics, and how columns are weighted.
+    module Scoring
+    end
+  end
+end

data/lib/llm/evals/stats/call_stats.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    module Stats
+      # How many calls a run made and how fast they came back, per provider, per
+      # vendor behind it, and per model at every setting. Latency counts only
+      # calls that worked, since a refusal returns quickly and would flatter the
+      # slowest model. Requests per second are over the whole run, so the rows
+      # compare with each other.
+      class CallStats
+        # The share {Stats.percentile} is asked for, shared with {Actions::Experiment::Summary#latency_p95}.
+        P95 = 0.95
+
+        # One line of the report: a provider, a vendor behind it, or one model at one setting.
+        #
+        # @!attribute [r] scope
+        #   @return [Symbol] :provider, :vendor or :model
+        # @!attribute [r] name
+        #   @return [String] the provider's name, or "provider/vendor" or "provider/model"
+        # @!attribute [r] requests
+        #   @return [Integer] every call made, including failures
+        # @!attribute [r] errors
+        #   @return [Integer] calls that raised or that the model failed
+        # @!attribute [r] per_second
+        #   @return [Float, nil] over the whole run; nil when its length is unknown or zero
+        # @!attribute [r] latency
+        #   @return [Float, nil] mean seconds of the calls that worked; nil with none
+        # @!attribute [r] p95
+        #   @return [Float, nil] 95th percentile seconds of the calls that worked; nil with none
+        Row = Data.define(:scope, :name, :requests, :errors, :per_second, :latency, :p95)
+
+        # Reads the run's length out of its recorded options, however they were stored.
+        #
+        # @param summaries [Array<Actions::Experiment::Summary>]
+        # @param run_options [Hash] as a run recorded them, with symbol keys or, read back from JSON, string keys
+        # @return [CallStats]
+        def self.for_run(summaries, run_options)
+          new(summaries, seconds: run_options.dig(:effective, :seconds) || run_options.dig("effective", "seconds"))
+        end
+
+        # Computes nothing until {#rows}.
+        #
+        # @param summaries [Array<Actions::Experiment::Summary>]
+        # @param seconds [Numeric, nil] how long the run took; nil leaves requests per second empty
+        def initialize(summaries, seconds:)
+          @summaries = summaries
+          @seconds = seconds
+        end
+
+        # A direct provider has no vendor rows, since it serves only its own models.
+        #
+        # @return [Array<Row>] each provider, then its vendors, each followed by its models
+        def rows
+          @summaries.group_by { it.model.provider }.flat_map do |provider, by_provider|
+            [row(:provider, provider.name, by_provider)] + by_provider.group_by { vendor(it) }.flat_map do |name, by_vendor|
+              models = by_vendor.group_by { it.model.to_s }.map { |model, group| row(:model, "#{provider.name}/#{model}", group) }
+              provider.direct? ? models : [row(:vendor, "#{provider.name}/#{name}", by_vendor), *models]
+            end
+          end
+        end
+
+        private
+
+        # The vendor is the id's first segment, "anthropic" in "anthropic/claude-haiku-5.5".
+        #
+        # @param summary [Actions::Experiment::Summary]
+        # @return [String]
+        def vendor(summary) = summary.model.id.split("/").first
+
+        # Totals one group of summaries into a row.
+        #
+        # @param scope [Symbol] :provider, :vendor or :model
+        # @param name [String] the row's label
+        # @param summaries [Array<Actions::Experiment::Summary>] the group
+        # @return [Row]
+        def row(scope, name, summaries)
+          results = summaries.flat_map(&:results)
+          worked = summaries.flat_map(&:worked_durations)
+          Row.new(scope:,
+            name:,
+            requests:   results.size,
+            errors:     results.count(&:errored?),
+            per_second: @seconds&.positive? ? results.size.fdiv(@seconds) : nil,
+            latency:    Stats.mean(worked),
+            p95:        Stats.percentile(worked, P95))
+        end
+      end
+    end
+  end
+end

data/lib/llm/evals/stats/usage.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    module Stats
+      # What one call cost, as the provider counted it. Summed over a run.
+      #
+      # @!attribute [r] input_tokens
+      #   @return [Integer]
+      # @!attribute [r] output_tokens
+      #   @return [Integer]
+      # @!attribute [r] cost_usd
+      #   @return [Float] in US dollars
+      Usage = Data.define(:input_tokens, :output_tokens, :cost_usd) do
+        # The start of a sum, and the usage of a reply that never reached a provider.
+        #
+        # @return [Usage] zero of everything
+        def self.zero = new(input_tokens: 0, output_tokens: 0, cost_usd: 0.0)
+
+        # Reads a Hash with Symbol or String keys, missing keys as zero.
+        #
+        # @param hash [Hash, Usage, nil]
+        # @return [Usage]
+        def self.from(hash)
+          return hash if hash.is_a?(Usage)
+          return zero if hash.nil?
+
+          h = hash.transform_keys(&:to_sym)
+          new(input_tokens: h.fetch(:input_tokens, 0).to_i,
+            output_tokens: h.fetch(:output_tokens, 0).to_i,
+            cost_usd: h.fetch(:cost_usd, 0.0).to_f)
+        end
+
+        # Adds field by field, which is how a run's usage is totalled.
+        #
+        # @param other [Usage]
+        # @return [Usage]
+        def +(other)
+          with(input_tokens: input_tokens + other.input_tokens,
+            output_tokens: output_tokens + other.output_tokens,
+            cost_usd: cost_usd + other.cost_usd)
+        end
+      end
+    end
+  end
+end

data/lib/llm/evals/stats.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    # The arithmetic reports and summaries share.
+    module Stats
+      module_function
+
+      # Nearest-rank, so the value is one a call really took.
+      #
+      # @param values [Array<Numeric>]
+      # @param share [Float] from 0 to 1
+      # @return [Numeric, nil] nil with no values
+      # @example
+      #   Stats.percentile([1, 2, 3, 4], 0.5) # => 2
+      def percentile(values, share)
+        return if values.empty?
+
+        values.sort[[(share * values.size).ceil - 1, 0].max]
+      end
+
+      # Plain average, sparing callers a division by zero on an empty list.
+      #
+      # @param values [Array<Numeric>]
+      # @return [Float, nil] nil with no values
+      def mean(values) = values.empty? ? nil : values.sum.fdiv(values.size)
+    end
+  end
+end

data/lib/llm/evals/testing/echo_adapter.rb

@@ -7,21 +7,26 @@ module LLM
       # prompt, parses the reply as a JSON object, and reports a reply that
       # is not one as the model's failure. The toy suite runs on it, and a
       # new suite can start from it.
-      class EchoAdapter < Adapter
+      class EchoAdapter < Actions::Adapter
+        # The system prompt sent with every case.
         SYSTEM = "Answer with a single JSON object and nothing else."
+        # Matches the Markdown code fences some models wrap JSON in, for stripping.
         FENCE = /\A```(?:json)?\s*|\s*```\z/
 
-        # @param kase [Case]
-        # @param model [Model]
-        # @return [Reply]
+        # Overrides {Actions::Adapter#call}. Failures to parse become the reply's
+        # error; transport errors raise, as they do from {Models::Model#complete}.
+        #
+        # @param kase [Datasets::Case] the case whose input is the prompt
+        # @param model [Models::Model] the model to call
+        # @return [Results::Reply]
         def call(kase, model:)
-          completion = model.complete(system: SYSTEM, user: kase.input, json: true)
-          output = JSON.parse(completion.text.strip.gsub(FENCE, ""))
-          return Reply.new(error: "reply is not a JSON object", raw: completion.text, usage: completion.usage) unless output.is_a?(Hash)
+          provider_response = model.complete(system: SYSTEM, user: kase.input, json: true)
+          output = JSON.parse(provider_response.text.strip.gsub(FENCE, ""))
+          return Results::Reply.new(error: "reply is not a JSON object", raw: provider_response.text, usage: provider_response.usage) unless output.is_a?(Hash)
 
-          Reply.new(output: output, raw: completion.text, usage: completion.usage)
+          Results::Reply.new(output: output, raw: provider_response.text, usage: provider_response.usage)
         rescue JSON::ParserError
-          Reply.new(error: "reply is not JSON", raw: completion.text, usage: completion.usage)
+          Results::Reply.new(error: "reply is not JSON", raw: provider_response.text, usage: provider_response.usage)
         end
       end
     end

data/lib/llm/evals/testing/fake_model.rb

@@ -3,12 +3,14 @@
 module LLM
   module Evals
     module Testing
-      # A {Model} on a {FakeProvider} of its own, for specs that run an
-      # adapter or an {Experiment} without the CLI.
-      class FakeModel < Model
-        # @param id [String]
-        # @param settings [Hash]
-        # @param provider_options [Hash] passed to {FakeProvider#initialize}
+      # A {Models::Model} on a {FakeProvider} of its own, for specs that run an
+      # adapter or an {Actions::Experiment} without the CLI.
+      class FakeModel < Models::Model
+        # Builds a fresh {FakeProvider}, so its recorded calls belong to this model.
+        #
+        # @param id [String] the model id
+        # @param settings [Hash{Symbol => Object}] effort, temperature, max_tokens
+        # @param provider_options [Hash{Symbol => Object}] passed to {FakeProvider#initialize}
         def initialize(id: "fake/echo", settings: { effort: nil, temperature: 0.2, max_tokens: 4096 }, **provider_options)
           super(id: id, provider: FakeProvider.new(**provider_options), settings: settings)
         end