llm-evals 0.4.0 → 0.5.0

data/lib/llm/evals/actions/error_budget.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    module Actions
+      # How many failed calls a run tolerates, from `errors.abort_after`: at least
+      # `total` errors, and, when `percent` is set, at least that share of the
+      # calls finished so far. Shared by every experiment's threads, and once
+      # spent it stays spent, so a late success cannot restart an aborted run.
+      class ErrorBudget
+        attr_reader :total, :percent, :errors, :calls
+
+        # @!attribute [r] total
+        #   The fewest errors at which the run may abort.
+        #   @return [Integer]
+
+        # @!attribute [r] percent
+        #   The share of finished calls that must also have failed, from 0 to 100.
+        #   @return [Numeric, nil]
+
+        # @!attribute [r] errors
+        #   Failed calls so far, counted across every experiment sharing this budget.
+        #   @return [Integer]
+
+        # @!attribute [r] calls
+        #   Finished calls so far, whether or not they failed.
+        #   @return [Integer]
+
+        # @param total [Integer] errors before the run may abort
+        # @param percent [Numeric, nil] the share of finished calls that must have failed
+        def initialize(total:, percent: nil)
+          @total = total
+          @percent = percent
+          @errors = 0
+          @calls = 0
+          @spent = false
+          @lock = Mutex.new
+        end
+
+        # Counts one finished call, and marks the budget spent once both limits are met.
+        #
+        # @param result [Results::RunResult] the finished call
+        # @return [void]
+        def record(result)
+          @lock.synchronize do
+            @calls += 1
+            @errors += 1 if result.errored?
+            @spent ||= @errors >= total && (percent.nil? || @errors * 100.0 / @calls >= percent)
+          end
+        end
+
+        # @return [Boolean] whether the run must stop
+        def spent? = @lock.synchronize { @spent }
+
+        # @return [String] why the run stopped
+        def reason
+          limit = percent ? "#{total} errors at #{percent}%" : "#{total} errors"
+          "run aborted: #{errors} of #{calls} finished calls failed (errors.abort_after: #{limit}); partial results were kept"
+        end
+      end
+    end
+  end
+end

data/lib/llm/evals/actions/error_diagnosis.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    module Actions
+      # What a streak of failed calls has in common: the messages, and how much of
+      # the run they touch. One failing model and a failing key call for different
+      # answers, so this names which it is.
+      #
+      # @!attribute [r] failures
+      #   @return [Array<Results::RunResult>] the failed calls
+      # @!attribute [r] runs
+      #   @return [Array<Experiment>] the whole run, which "all" is measured against
+      ErrorDiagnosis = Data.define(:failures, :runs) do
+        # The distinct messages, most frequent first.
+        #
+        # @return [Array<Array(String, Integer)>] each message with its count
+        def messages
+          failures.map { it.error || it.reply.error }.map(&:to_s).tally.sort_by { |message, count| [-count, message] }
+        end
+
+        # @return [Scope] the models that failed
+        def models = scope(failures.map { it.experiment.model.id }, runs.map { it.model.id })
+
+        # @return [Scope] the datasets that failed
+        def datasets = scope(failures.map { it.case.dataset }, runs.map(&:dataset))
+
+        # @return [Scope] the cases that failed
+        def cases = scope(failures.map { it.case.key }, runs.flat_map(&:cases).map(&:key))
+
+        private
+
+        # @param failed [Array<String>] one name per failure
+        # @param planned [Array<String>] one name per planned item
+        # @return [Scope]
+        def scope(failed, planned) = ErrorDiagnosis::Scope.new(failed: failed.uniq.sort, total: planned.uniq.size)
+      end
+
+      # How much of one dimension of the run has failed.
+      #
+      # @!attribute [r] failed
+      #   @return [Array<String>] the names that failed, sorted
+      # @!attribute [r] total
+      #   @return [Integer] how many the run planned
+      ErrorDiagnosis::Scope = Data.define(:failed, :total) do
+        # @return [Boolean] whether every planned one failed
+        def all? = failed.size >= total
+      end
+    end
+  end
+end

data/lib/llm/evals/actions/error_streak.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    module Actions
+      # Notices when every call has been failing, and holds the run while someone
+      # decides what to do. A bad key or a dead endpoint fails everything, and
+      # without this the run spends its whole matrix finding that out. Unlike
+      # {ErrorBudget}, which the lineup opts into and which only stops, this is
+      # always on and can resume. Shared by every experiment's threads.
+      class ErrorStreak
+        # Failures in a row, with no success between them, that pause the run.
+        LIMIT = 100
+
+        # @param limit [Integer] failures in a row that pause the run
+        def initialize(limit: LIMIT)
+          @limit = limit
+          @failures = []
+          @paused = false
+          @stopped = false
+          @lock = Mutex.new
+          @resumed = ConditionVariable.new
+        end
+
+        # Counts one finished call. The call that completes a streak is the one
+        # that asks; the other workers wait in {#hold}. Calls still in flight
+        # while it asks do not count, so one question cannot lead straight to another.
+        #
+        # @param result [Results::RunResult] the finished call
+        # @yieldparam failures [Array<Results::RunResult>] the streak, oldest first
+        # @yieldreturn [Boolean] whether the run should carry on
+        # @return [void]
+        def record(result)
+          failures = @lock.synchronize { tally(result) }
+          settle { yield failures } if failures
+        end
+
+        # Blocks while the run is paused. Workers call it before starting a case.
+        #
+        # @return [void]
+        def hold
+          @lock.synchronize { @resumed.wait(@lock) while @paused }
+        end
+
+        # @return [Boolean] whether the answer was to stop
+        def stopped? = @lock.synchronize { @stopped }
+
+        # @return [String] why the run stopped
+        def reason = "run stopped after #{@limit} failed calls in a row; partial results were kept"
+
+        private
+
+        # @param result [Results::RunResult]
+        # @return [Array<Results::RunResult>, nil] the streak, once it reaches the limit
+        def tally(result)
+          return if @paused || @stopped
+
+          result.errored? ? @failures << result : @failures.clear
+          return if @failures.size < @limit
+
+          @paused = true
+          @failures.dup.tap { @failures.clear }
+        end
+
+        # Releases the workers whatever happens: an exception or an interrupt
+        # while asking counts as a stop, since nobody said to carry on.
+        #
+        # @yieldreturn [Boolean] whether the run should carry on
+        # @return [void]
+        def settle
+          carry_on = false
+          carry_on = yield
+        ensure
+          @lock.synchronize do
+            @paused = false
+            @stopped = !carry_on
+            @resumed.broadcast
+          end
+        end
+      end
+    end
+  end
+end

data/lib/llm/evals/actions/experiment.rb

@@ -0,0 +1,251 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    module Actions
+      # One model at one setting over every case of one dataset, run on a
+      # thread pool the gem owns, plus the per-case detail the terminal report
+      # needs. Trackers only watch: {Trackers::Local} always, and at most one
+      # remote tracker, each told when the experiment starts, when a case
+      # finishes, and what the summary was.
+      class Experiment
+        # What one finished experiment reports back. `means` holds one entry
+        # per metric plus "score", the mean of each case's overall value;
+        # `usage` is summed over the cases; `permalink` is the remote tracker's
+        # page, when it has one.
+        #
+        # @!attribute [r] model
+        #   @return [Models::Model] what ran
+        # @!attribute [r] dataset
+        #   @return [String] the dataset's name
+        # @!attribute [r] name
+        #   @return [String] the experiment's {Experiment#name}
+        # @!attribute [r] means
+        #   @return [Hash{String => Float, nil}] one mean per metric, plus "score"
+        # @!attribute [r] results
+        #   @return [Array<Results::RunResult>] in the experiment's case order
+        # @!attribute [r] errors
+        #   @return [Array<String>] one "case: message" per call that raised
+        # @!attribute [r] duration
+        #   @return [Float] seconds, less an estimate of the time spent on rate limits
+        # @!attribute [r] permalink
+        #   @return [String, nil] the remote tracker's page
+        # @!attribute [r] usage
+        #   @return [Stats::Usage] summed over the cases
+        Summary = Data.define(:model, :dataset, :name, :means, :results, :errors, :duration, :permalink, :usage) do
+          # @return [Float, nil] the mean overall score
+          def score = means["score"]
+
+          # @param metric [Scoring::Metric, String]
+          # @return [Float, nil]
+          def mean(metric) = means[metric.respond_to?(:name) ? metric.name : metric.to_s]
+
+          # @return [Float, nil] the mean seconds of the calls that worked
+          def latency = Stats.mean(worked_durations)
+
+          # @return [Float, nil] the 95th percentile seconds of the calls that worked
+          def latency_p95 = Stats.percentile(worked_durations, Stats::CallStats::P95)
+
+          # Each working call's own seconds, which {#latency} and {#latency_p95} read.
+          #
+          # @return [Array<Float>] errored calls are left out
+          def worked_durations = results.reject(&:errored?).map(&:duration)
+
+          # The same experiment over one variant's cases. Its duration is the sum
+          # of those cases' own, since they overlapped with the others.
+          #
+          # @param variant [String]
+          # @param metrics [Array<Scoring::Metric>] the scorer's
+          # @return [Summary]
+          def for_variant(variant, metrics)
+            chosen = results.select { it.case.variant == variant }
+            with(means:    Experiment.means(metrics, chosen),
+              results:  chosen,
+              errors:   chosen.select(&:failed?).map { "#{it.case.key}: #{it.error}" },
+              duration: chosen.sum(&:duration),
+              usage:    Experiment.usage(chosen))
+          end
+        end
+
+        # Averages what the results scored; a failed case has no score and is skipped.
+        #
+        # @param metrics [Array<Scoring::Metric>]
+        # @param results [Array<Results::RunResult>]
+        # @return [Hash{String => Float, nil}] one mean per metric, plus "score"
+        def self.means(metrics, results)
+          scored = results.filter_map(&:score)
+          means = metrics.to_h { |metric| [metric.name, metric.combine(scored.map { it.metrics[metric.name] })] }
+          means.merge("score" => scored.empty? ? nil : scored.sum(&:value).fdiv(scored.size))
+        end
+
+        # Tokens and cost across the results; a raised case has no reply and adds nothing.
+        #
+        # @param results [Array<Results::RunResult>]
+        # @return [Stats::Usage] summed over the results
+        def self.usage(results) = results.filter_map(&:reply).map(&:usage).reduce(Stats::Usage.zero, :+)
+
+        attr_reader :model, :adapter, :scorer, :cases, :dataset, :parallelism, :trackers
+
+        # @!attribute [r] model
+        #   The model at the setting under test.
+        #   @return [Models::Model]
+
+        # @!attribute [r] adapter
+        #   Runs each case.
+        #   @return [Adapter]
+
+        # @!attribute [r] scorer
+        #   Scores each reply.
+        #   @return [Scoring::Scorer]
+
+        # @!attribute [r] cases
+        #   Sorted by variant, number and id.
+        #   @return [Array<Datasets::Case>]
+
+        # @!attribute [r] dataset
+        #   The name of the dataset the first case came from.
+        #   @return [String]
+
+        # @!attribute [r] parallelism
+        #   Cases in flight at once, never below 1.
+        #   @return [Integer]
+
+        # @!attribute [r] trackers
+        #   Only the enabled ones of those given, in the order given.
+        #   @return [Array<Trackers::Base>]
+
+        # @param model [Models::Model]
+        # @param cases [Array<Datasets::Case>] all of one dataset
+        # @param adapter [Adapter]
+        # @param scorer [Scoring::Scorer]
+        # @param parallelism [Integer] cases in flight at once
+        # @param trackers [Array<Trackers::Base>] told about the run, in this order
+        # @raise [ArgumentError] when there are no cases
+        def initialize(model:, cases:, adapter:, scorer:, parallelism: 4, trackers: [])
+          raise ArgumentError, "an experiment needs at least one case" if cases.empty?
+
+          @model = model
+          @cases = cases.sort_by { [it.variant, it.number || 0, it.id] }
+          @dataset = cases.first.dataset
+          @adapter = adapter
+          @scorer = scorer
+          @parallelism = [parallelism.to_i, 1].max
+          @trackers = trackers.select(&:enabled?)
+          @results = {}
+          @lock = Mutex.new
+          @waited = 0.0
+        end
+
+        # How reports and trackers tell this experiment from the others of a run.
+        #
+        # @return [String] e.g. "anthropic/claude-haiku-5.5 effort=low t=0.0 [clear]"
+        def name = "#{model} [#{dataset}]"
+
+        # Runs every case, then asks the trackers for a page to link the summary to.
+        #
+        # @param on_case [#call, nil] called with each finished case's {Results::RunResult}
+        # @param stop [#call, nil] asked before each case; truthy leaves the rest unrun
+        # @return [Summary]
+        def run(on_case: nil, stop: nil)
+          started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          adapter.prepare(dataset)
+          trackers.each { it.start(self) }
+          errors = run_locally(on_case, stop)
+          # An approximation: it assumes waits fell evenly across the workers. Workers
+          # that wait while others run did not hold up the run for that long.
+          summary = summarize(errors, nil, Process.clock_gettime(Process::CLOCK_MONOTONIC) - started - (@waited / workers))
+          urls = trackers.map { it.finish(self, summary)&.fetch(:url, nil) }
+          summary.with(permalink: urls.compact.first)
+        end
+
+        # Runs one case: the adapter, then the scorer. A reply the model failed
+        # (`Reply#error`) scores 0 without troubling the scorer; an exception is
+        # the transport's and is left to the caller, which counts it as an error.
+        #
+        # @param kase [Datasets::Case]
+        # @return [Results::RunResult]
+        # @raise [Error] when the adapter returns anything but a {Results::Reply}
+        def evaluate(kase)
+          started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          reply, waited = Models::Throttle.measure { adapter.call(kase, model: model) }
+          @lock.synchronize { @waited += waited }
+          raise Error, "#{adapter.class}#call returned #{reply.class}, not an LLM::Evals::Results::Reply" unless reply.is_a?(Results::Reply)
+
+          score = reply.error? ? Results::Score.failed(scorer.metrics, reply.error) : scorer.call(kase, reply, context: adapter.context(kase))
+          Results::RunResult.new(experiment: self,
+            case: kase,
+            reply: reply,
+            score: score,
+            duration: Process.clock_gettime(Process::CLOCK_MONOTONIC) - started - waited)
+        end
+
+        private
+
+        # Threads worth starting: more than there are cases would only sit idle.
+        #
+        # @return [Integer]
+        def workers = [parallelism, cases.size].min
+
+        # Remembers a result and tells the trackers, under one lock, so a
+        # tracker sees results one at a time and in the order they are kept.
+        #
+        # @param result [Results::RunResult] the finished case
+        # @return [Results::RunResult] the same result
+        def remember(result)
+          @lock.synchronize do
+            @results[result.case.key] = result
+            trackers.each { it.record(result) }
+          end
+          result
+        end
+
+        # A fixed pool of threads over a queue: `parallelism` cases in flight.
+        #
+        # @param on_case [#call, nil] called with each finished case's {Results::RunResult}
+        # @param stop [#call, nil] asked before each case; truthy leaves the rest unrun
+        # @return [Array<String>] one "case: message" per case that raised
+        def run_locally(on_case, stop)
+          queue = Queue.new
+          cases.each { queue << it }
+          queue.close
+          errors = Queue.new
+          Array.new(workers) do
+            Thread.new do
+              while !stop&.call && (kase = queue.pop)
+                result = begin
+                  remember evaluate(kase)
+                rescue StandardError => e
+                  errors << "#{kase.key}: #{e.class}: #{e.message}"
+                  remember Results::RunResult.new(experiment: self, case: kase, error: "#{e.class}: #{e.message}")
+                end
+                on_case&.call(result)
+              end
+            end
+          end.each(&:join)
+          Array.new(errors.size) { errors.pop }
+        end
+
+        # Freezes what has been remembered so far into a {Summary}, in case order.
+        #
+        # @param errors [Array<String>, nil] the messages {#run_locally} collected
+        # @param permalink [String, nil] the remote tracker's page, when already known
+        # @param duration [Float] seconds to report
+        # @return [Summary]
+        def summarize(errors, permalink, duration)
+          results = @lock.synchronize { @results.values }.sort_by { cases.index(it.case) }
+          Summary.new(
+            model:     model,
+            dataset:   dataset,
+            name:      name,
+            means:     self.class.means(scorer.metrics, results),
+            results:   results,
+            errors:    Array(errors),
+            duration:  duration,
+            permalink: permalink,
+            usage:     self.class.usage(results)
+          )
+        end
+      end
+    end
+  end
+end

data/lib/llm/evals/actions/verify.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    module Actions
+      # Offline consistency checks over every case, so a typo in the suite is
+      # caught before it is billed as a model's mistake. The gem checks what
+      # every suite has: an input and a parsable expected answer. The adapter
+      # checks each dataset's shared files, and the scorer each case against
+      # what it will demand of it.
+      class Verify
+        # @param configuration [Runtime::Configuration] the suite to check
+        def initialize(configuration = LLM::Evals.configuration)
+          @configuration = configuration
+        end
+
+        # Asks the adapter about each dataset, then checks every case in it.
+        #
+        # @return [Hash{String => Array<String>}] problems by "dataset/id"; empty when all is sound
+        def problems
+          discovery = @configuration.discovery
+          failures = {}
+          discovery.datasets.each do |dataset|
+            Array(adapter.verify(dataset)).each { (failures[dataset.to_s] ||= []) << it }
+            discovery.cases(dataset: dataset).each do |kase|
+              list = case_problems(kase)
+              failures[kase.key] = list unless list.empty?
+            end
+          end
+          failures
+        end
+
+        # Checks one case; an exception from the scorer or adapter is reported as a problem.
+        #
+        # @param kase [Datasets::Case] the case to check
+        # @return [Array<String>] problems; empty when the case is sound
+        def case_problems(kase)
+          problems = []
+          problems << "input is empty" if kase.input.empty?
+          problems << kase.metadata[:expected_error] if kase.metadata[:expected_error]
+          return problems unless kase.expected?
+
+          problems.concat(Array(scorer.verify(kase, context: adapter.context(kase))))
+        rescue StandardError => e
+          problems << "#{e.class}: #{e.message}"
+        end
+
+        private
+
+        # The suite's adapter.
+        #
+        # @return [Actions::Adapter, nil] nil when the suite has not registered one
+        def adapter = @configuration.adapter
+        # The suite's scorer.
+        #
+        # @return [Scoring::Scorer, nil] nil when the suite has not registered one
+        def scorer = @configuration.scorer
+      end
+    end
+  end
+end

data/lib/llm/evals/actions.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    # What the gem does with a suite: runs experiments through its adapter, and verifies it.
+    module Actions
+    end
+  end
+end

data/lib/llm/evals/cli/cases.rb

@@ -9,6 +9,11 @@ module LLM
 
         option :dataset, type: :array, aliases: ["-d"], default: [], desc: "Only these datasets, comma-separated"
 
+        # Prints one row per case, then whatever the lineup cannot ask.
+        #
+        # @param dataset [Array<String>] as given to -d, possibly comma-separated; empty means every dataset
+        # @return [void]
+        # @raise [Error] when no case is found
         def call(dataset: [], **)
           scorer = configuration.scorer
           names = dataset.flat_map { it.split(",") }
@@ -26,6 +31,8 @@ module LLM
 
         # Names, on STDERR, every lineup entry the suite cannot ask, so the
         # mismatch is visible before anyone reaches for `run`.
+        #
+        # @return [void]
         def refusals
           config = configuration.lineup
           _, refused = configuration.providers.admit(config.contenders, request_types: config.request_types)
@@ -34,6 +41,9 @@ module LLM
 
         # The generic cell when the scorer has nothing to say: how many fields
         # the expected answer holds, or what kind of thing it is.
+        #
+        # @param kase [Datasets::Case]
+        # @return [String]
         def summary(kase)
           case kase.expected
           when Hash then "#{kase.expected.size} fields"