llm-evals 0.4.0 → 0.5.0

data/lib/llm/evals/testing/fake_provider.rb

@@ -11,16 +11,24 @@ module LLM
       # `wrong` maps a case to fields to answer wrongly. `answers`, a callable
       # taking the user prompt, replaces the lookup altogether.
       class FakeProvider < Providers::Base
+        # The namespace of the ids this provider claims.
         PREFIX = "fake/"
-        USAGE = Usage.new(input_tokens: 100, output_tokens: 10, cost_usd: 0.001)
+        # What every call reports having cost, so totals are predictable in a spec.
+        USAGE = Stats::Usage.new(input_tokens: 100, output_tokens: 10, cost_usd: 0.001)
+        # The reply a `junk` case gets.
         JUNK = "Sure! Here is my answer, in prose rather than JSON."
 
+        # Every {#complete} call so far, with `model:`, `system:`, `user:` and `json:`, for a spec to inspect.
+        #
+        # @return [Array<Hash{Symbol => Object}>]
         attr_reader :calls
 
+        # With no arguments, every case gets its expected JSON.
+        #
         # @param junk [Array<String>] case ids or keys to answer with prose
         # @param wrong [Hash{String => Hash}] case id or key => fields to override
         # @param answers [#call, nil] user prompt => reply text; replaces the lookup
-        # @param discovery [Discovery, nil] where to look cases up; the configured suite's when nil
+        # @param discovery [Datasets::Discovery, nil] where to look cases up; the configured suite's when nil
         def initialize(junk: [], wrong: {}, answers: nil, discovery: nil)
           super()
           @junk = junk
@@ -31,26 +39,46 @@ module LLM
           @lock = Mutex.new
         end
 
+        # Overrides {Base#name}.
+        #
+        # @return [String] "fake"
         def name = "fake"
+        # Overrides {Base#request_type}.
+        #
+        # @return [Symbol] `:prompt`
         def request_type = :prompt
+        # Overrides {Base#prefix}.
+        #
+        # @return [String] {PREFIX}
         def prefix = PREFIX
+        # Overrides {Base#env_keys}: nothing is called, so nothing is needed.
+        #
+        # @return [Array<String>] always empty
         def env_keys = []
 
+        # Overrides {Base#catalog} to say so outright.
+        #
         # @return [nil] any id passes verification
         def catalog = nil
 
-        # @param model [Model]
-        # @param system [String, nil]
-        # @param user [String]
-        # @param json [Boolean]
-        # @return [Completion]
+        # Records the call under a lock, then answers it. Overrides {Base#complete}.
+        #
+        # @param model [Models::Model] the model called
+        # @param system [String, nil] the system prompt, recorded only
+        # @param user [String] the user prompt, which selects the case
+        # @param json [Boolean] recorded only
+        # @return [Results::ProviderResponse]
         def complete(model, system:, user:, json: true)
           @lock.synchronize { @calls << { model: model.id, system: system, user: user, json: json } }
-          Completion.new(text: @answers ? @answers.call(user) : answer(user), usage: USAGE)
+          Results::ProviderResponse.new(text: @answers ? @answers.call(user) : answer(user), usage: USAGE)
         end
 
         private
 
+        # Looks the case up by its input. A prompt matching no case gets "{}".
+        #
+        # @param user [String] the user prompt
+        # @return [String] prose for a `junk` case; otherwise JSON, with `wrong` merged in
         def answer(user)
           kase = discovery.all.find { it.input == user }
           return "{}" if kase.nil?
@@ -60,6 +88,9 @@ module LLM
           JSON.generate(kase.expected.is_a?(Hash) ? kase.expected.merge(wrong) : kase.expected)
         end
 
+        # The injected discovery, or the configured suite's when none was given.
+        #
+        # @return [Datasets::Discovery]
         def discovery = @discovery || LLM::Evals.configuration.discovery
       end
     end

data/lib/llm/evals/testing.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    # Doubles a suite can run without calling anything.
+    module Testing
+    end
+  end
+end

data/lib/llm/evals/trackers/base.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    module Trackers
+      # Where a run's results go besides the terminal. The gem always owns the
+      # run loop; a tracker only receives callbacks: {#start} once per
+      # experiment, {#record} once per finished case, {#finish} once with the
+      # summary. The local JSON log ({Trackers::Local}) always runs; at most
+      # one remote tracker runs beside it, chosen by `--tracker`, the
+      # `tracker:` key, or the environment (see {Trackers::Registry#choose}).
+      #
+      # A tracker for another framework (W&B Weave, an OTLP backend) subclasses
+      # this in a consumer or a plugin gem and is registered with `c.tracker`.
+      #
+      # @abstract Override the three callbacks; declare {.env_key} and {.gems}.
+      class Base
+        class << self
+          # The class name is the default, so a subclass needs no declaration.
+          #
+          # @return [String] the name `--tracker` and `tracker:` use; the class name, dasherized, by default
+          def tracker_name = (name || "Tracker").split("::").last.gsub(/([a-z\d])([A-Z])/, '\1-\2').downcase
+
+          # Nil means the environment can never select this tracker by itself.
+          #
+          # @return [String, nil] the environment variable whose presence selects this tracker when nothing else does
+          def env_key = nil
+
+          # Loaded by {.requirements!} rather than at require time.
+          #
+          # @return [Array<String>] the optional gems this tracker requires when chosen
+          def gems = []
+
+          # A blank value counts as absent.
+          #
+          # @param env [#[]] the environment, ENV by default
+          # @return [Boolean] whether the environment names this tracker
+          def detect?(env = ENV) = !env_key.nil? && !env[env_key].to_s.strip.empty?
+
+          # Loads the gems and checks the key, with an error that names both, so
+          # a chosen tracker that cannot run says so before any model is called.
+          #
+          # @param env [#[]] the environment, ENV by default
+          # @return [void]
+          # @raise [MissingDependency] when a gem is absent
+          # @raise [Error] when the key is not set
+          def requirements!(env = ENV)
+            gems.each do |gem_name|
+              LLM::Evals.optional!(gem_name.tr("-", "/"), gem_name, needed_for: "the #{tracker_name} tracker")
+            end
+            return if env_key.nil? || detect?(env)
+
+            raise Error, "the #{tracker_name} tracker needs #{env_key} in the environment (gems: #{gems.join(", ")})"
+          end
+        end
+
+        # What {#initialize} was given, for subclasses to read.
+        #
+        # @return [Hash{Symbol => Object}]
+        attr_reader :context
+
+        # Keeps the context whole, so a subclass reads only what it wants.
+        #
+        # @param context [Hash] `suite:`, `project:`, `project_id:` and `options:` (the tracker's own key in the config file)
+        def initialize(**context)
+          @context = context
+        end
+
+        # Delegates to {.tracker_name}, so an instance and its class agree.
+        #
+        # @return [String]
+        def name = self.class.tracker_name
+
+        # Nothing in this class turns itself off; a subclass may.
+        #
+        # @return [Boolean] false turns every callback into a no-op
+        def enabled? = true
+
+        # Overridden by each tracker to name its destination.
+        #
+        # @return [String] one line for `--verbose`: where this tracker sends things, and why
+        def describe = name
+
+        # Called once per experiment, before any case runs.
+        #
+        # @param _experiment [Actions::Experiment] the experiment starting
+        # @return [void]
+        def start(_experiment) = nil
+
+        # Called once per finished case, one at a time under the experiment's lock.
+        #
+        # @param _result [Results::RunResult] the finished case
+        # @return [void]
+        def record(_result) = nil
+
+        # Called once per experiment, after the last case.
+        #
+        # @param _experiment [Actions::Experiment] the experiment finishing
+        # @param _summary [Actions::Experiment::Summary] its totals
+        # @return [Hash] `url:` to the experiment when one is known, else nil
+        def finish(_experiment, _summary) = { url: nil }
+
+        # Empty when the config file has no section for this tracker.
+        #
+        # @return [Hash] the tracker's own section of the config file
+        def options = context.fetch(:options, nil) || {}
+      end
+    end
+  end
+end

data/lib/llm/evals/trackers/braintrust.rb

@@ -7,17 +7,38 @@ module LLM
       # suite's project, through the braintrust gem's own OpenTelemetry
       # exporter, the way `Braintrust::Eval.run` would have, except that the
       # loop stays ours. The gem is required only when this tracker is chosen.
-      class Braintrust < Tracker
+      class Braintrust < Base
+        # A set id skips the project lookup in {#start}.
+        #
+        # @param env [#[]]
+        # @return [String, nil] the project BRAINTRUST_PROJECT_ID names
+        def self.project_id(env = ENV) = env["BRAINTRUST_PROJECT_ID"].to_s[/\S+/]
+
+        # Overrides {Base.env_key}.
+        #
+        # @return [String] "BRAINTRUST_API_KEY"
         def self.env_key = "BRAINTRUST_API_KEY"
+        # Overrides {Base.gems}.
+        #
+        # @return [Array<String>] just the braintrust gem
         def self.gems = ["braintrust"]
 
         attr_reader :experiment_id, :permalink
 
+        # @!attribute [r] experiment_id
+        #   @return [String, nil] the Braintrust experiment; nil until {#start}
+
+        # @!attribute [r] permalink
+        #   @return [String, nil] the experiment's page; nil until {#start}
+
+        # Overrides {Base#describe}.
+        #
+        # @return [String] the destination project, by id when there is one
         def describe = "braintrust: project #{context[:project_id] || context[:project].inspect}"
 
         # Logs in, finds or creates the project, creates the experiment.
         #
-        # @param experiment [Experiment]
+        # @param experiment [Actions::Experiment]
         # @return [void]
         def start(experiment)
           ::Braintrust.init(blocking_login: true, auto_instrument: false)
@@ -41,7 +62,7 @@ module LLM
         # runner emits: input, expected, output, scores and metadata as JSON
         # attributes, parented to the experiment.
         #
-        # @param result [RunResult]
+        # @param result [Results::RunResult]
         # @return [void]
         def record(result)
           span = @tracer.start_root_span("eval")
@@ -63,6 +84,8 @@ module LLM
 
         # Flushes the spans and hands back the experiment's page.
         #
+        # @param _experiment [Actions::Experiment] unused
+        # @param _summary [Actions::Experiment::Summary] unused
         # @return [Hash] `url:` the experiment permalink
         def finish(_experiment, _summary)
           provider = OpenTelemetry.tracer_provider
@@ -72,13 +95,27 @@ module LLM
 
         private
 
+        # Sets one span attribute as JSON, the form Braintrust reads.
+        #
+        # @param span [OpenTelemetry::Trace::Span] the span to annotate
+        # @param key [String] the attribute name
+        # @param value [Object] any JSON-encodable value
+        # @return [void]
         def json(span, key, value) = span.set_attribute(key, JSON.dump(value))
 
+        # What the experiment was run with, as Braintrust stores it.
+        #
+        # @param experiment [Actions::Experiment] the experiment starting
+        # @return [Hash{Symbol => Object}]
         def metadata(experiment)
           { model: experiment.model.id, provider: experiment.model.provider.name, dataset: experiment.dataset,
             **experiment.model.settings, **experiment.adapter.metadata }
         end
 
+        # What a case span carries beyond its scores; a failed case has no score or reply to report.
+        #
+        # @param result [Results::RunResult] the finished case
+        # @return [Hash{Symbol => Object}]
         def case_metadata(result)
           base = { case: result.case.key, dataset: result.case.dataset, variant: result.case.variant, difficulty: result.case.difficulty }
           return base.merge(error: result.error) if result.failed?

data/lib/llm/evals/trackers/local.rb

@@ -8,11 +8,19 @@ module LLM
       # Always on: every experiment's results and summary land in one JSON
       # file under `log/`, so a run is never lost to a closed terminal or a
       # remote tracker that was never configured.
-      class Local < Tracker
+      class Local < Base
         attr_reader :dir, :path
 
+        # @!attribute [r] dir
+        #   @return [String] where the files go
+
+        # @!attribute [r] path
+        #   @return [String, nil] the file for this experiment; nil until {#start}
+
+        # Holds results in memory and writes nothing until {#finish}.
+        #
         # @param dir [String] where the files go; `log/llm-evals` at the repository root by default
-        # @param context [Hash] see {Tracker#initialize}
+        # @param context [Hash] see {Base#initialize}
         def initialize(dir: File.join(Dry::CLI::UI::Invocation.root(Dir.pwd), "log", "llm-evals"), **context)
           super(**context)
           @dir = dir
@@ -20,17 +28,32 @@ module LLM
           @lock = Mutex.new
         end
 
+        # Overrides {Base#describe}.
+        #
+        # @return [String] the directory
         def describe = "local: #{dir}"
 
+        # Fixes the file name from the start time, so {#finish} needs no clock.
+        #
+        # @param experiment [Actions::Experiment] the experiment starting
+        # @return [void]
         def start(experiment)
           @started_at = Time.now
           @path = File.join(dir, "#{slug(experiment.name)}.#{@started_at.strftime("%Y-%m-%d.%H%M%S")}.json")
         end
 
+        # Keeps the result for {#finish}'s document.
+        #
+        # @param result [Results::RunResult] the finished case
+        # @return [void]
         def record(result)
           @lock.synchronize { @results << result }
         end
 
+        # Creates the directory and writes the whole document in one go.
+        #
+        # @param experiment [Actions::Experiment] the experiment finishing
+        # @param summary [Actions::Experiment::Summary] its totals
         # @return [Hash] `url:` nil, `path:` the file written
         def finish(experiment, summary)
           FileUtils.mkdir_p(dir)
@@ -40,8 +63,17 @@ module LLM
 
         private
 
+        # Makes a name safe for a file name.
+        #
+        # @param text [String] the experiment's name
+        # @return [String]
         def slug(text) = text.gsub(/[^A-Za-z0-9.-]+/, "_").squeeze("_").delete_prefix("_").delete_suffix("_")
 
+        # The file's whole contents, with results back in case order rather than finishing order.
+        #
+        # @param experiment [Actions::Experiment] the experiment finishing
+        # @param summary [Actions::Experiment::Summary] its totals
+        # @return [Hash{Symbol => Object}]
         def document(experiment, summary)
           {
             suite:      context[:suite],
@@ -59,6 +91,10 @@ module LLM
           }
         end
 
+        # One case's entry; the score and reply are nil when the case raised.
+        #
+        # @param result [Results::RunResult] the finished case
+        # @return [Hash{Symbol => Object}]
         def row(result)
           {
             case:        result.case.key,

data/lib/llm/evals/trackers/logfire.rb

@@ -14,27 +14,45 @@ module LLM
       # already set, the SDK's own reading of it wins and {#describe} says so.
       # The raw reply is not sent unless the config file opts in with
       # `logfire: { include_replies: true }`.
-      class Logfire < Tracker
+      class Logfire < Base
         # Logfire's US region, per their docs on 2026-10-15; other regions
         # exist, so LOGFIRE_BASE_URL overrides it.
         DEFAULT_BASE_URL = "https://logfire-us.pydantic.dev"
+        # The environment variable holding the write token; also what selects this tracker.
         TOKEN = "LOGFIRE_TOKEN"
+        # The environment variable that overrides {DEFAULT_BASE_URL}.
         BASE_URL = "LOGFIRE_BASE_URL"
+        # The standard OTLP endpoint variable, which beats ours when set.
         OTEL_ENDPOINT = "OTEL_EXPORTER_OTLP_ENDPOINT"
+        # The standard OTLP headers variable, which beats ours when set.
         OTEL_HEADERS = "OTEL_EXPORTER_OTLP_HEADERS"
 
         # Bytes of the diagnostics JSON kept on a case span; the rest is cut,
         # since an attribute is not the place for a whole transcript.
         DIAGNOSTICS_LIMIT = 4096
 
+        # Overrides {Base.env_key}.
+        #
+        # @return [String] {TOKEN}
         def self.env_key = TOKEN
+        # Overrides {Base.gems}.
+        #
+        # @return [Array<String>] the OpenTelemetry SDK and its OTLP exporter
         def self.gems = %w[opentelemetry-sdk opentelemetry-exporter-otlp]
 
         attr_reader :endpoint, :source
 
+        # @!attribute [r] endpoint
+        #   @return [String, nil] where traces go; nil leaves the SDK's default
+
+        # @!attribute [r] source
+        #   @return [String] which variables the credentials came from, for {#describe}
+
+        # Reads the settings now; the OpenTelemetry objects wait for {#start}.
+        #
         # @param exporter [#export, nil] an exporter to use instead of OTLP, for tests
-        # @param env [Hash]
-        # @param context [Hash] see {Tracker#initialize}
+        # @param env [#[]] the environment, ENV by default
+        # @param context [Hash] see {Base#initialize}
         def initialize(exporter: nil, env: ENV, **context)
           super(**context)
           @env = env
@@ -43,17 +61,31 @@ module LLM
           @cases = []
         end
 
+        # Overrides {Base#describe}.
+        #
+        # @return [String] the endpoint and where the credentials came from
         def describe = "logfire: endpoint #{endpoint || "(SDK default)"}, credentials from #{source}"
 
+        # Accepts the config's key as a string or a symbol.
+        #
         # @return [Boolean] whether raw replies ride along on the case spans
         def include_replies? = options["include_replies"] == true || options[:include_replies] == true
 
+        # Builds the provider and opens the experiment span.
+        #
+        # @param experiment [Actions::Experiment] the experiment starting
+        # @return [void]
         def start(experiment)
           @provider = provider
           @tracer = @provider.tracer("llm-evals", LLM::Evals::VERSION)
           @span = @tracer.start_span("experiment #{experiment.name}", attributes: experiment_attributes(experiment))
         end
 
+        # Adds a finished child span, back-dated by the case's duration, since the
+        # callback arrives only once the case is over.
+        #
+        # @param result [Results::RunResult] the finished case
+        # @return [void]
         def record(result)
           now = Time.now
           span = @tracer.start_span("case #{result.case.key}",
@@ -70,6 +102,8 @@ module LLM
         # Closes the experiment span with the summary and flushes. No URL:
         # Logfire's trace pages are not something the token lets us build.
         #
+        # @param _experiment [Actions::Experiment] unused
+        # @param summary [Actions::Experiment::Summary] its totals
         # @return [Hash] `url:` nil
         def finish(_experiment, summary)
           summary.means.each { |metric, mean| @span.set_attribute("evals.mean.#{metric}", mean) unless mean.nil? }
@@ -87,6 +121,10 @@ module LLM
         # Which credentials win: the standard OTEL_* variables when either is
         # set, else our LOGFIRE_* pair. Returns the endpoint and headers to
         # hand the exporter (nil leaves the SDK's own reading in place).
+        #
+        # @param env [#[]] the environment
+        # @return [Array(String, Hash{String => String}, String)] endpoint, headers and a
+        #   description of the source; endpoint and headers may be nil
         def settings(env)
           if [OTEL_ENDPOINT, OTEL_HEADERS].any? { !env[it].to_s.empty? }
             [env[OTEL_ENDPOINT], nil, "#{OTEL_ENDPOINT}/#{OTEL_HEADERS} (set in the environment; #{TOKEN} ignored)"]
@@ -98,6 +136,8 @@ module LLM
 
         # A tracer provider of our own, so the global one, which a consumer
         # may have configured for its application, is left alone.
+        #
+        # @return [OpenTelemetry::SDK::Trace::TracerProvider]
         def provider
           exporter = @exporter || OpenTelemetry::Exporter::OTLP::Exporter.new(**{ endpoint: endpoint, headers: @headers }.compact)
           processor = if @exporter
@@ -109,6 +149,10 @@ module LLM
           OpenTelemetry::SDK::Trace::TracerProvider.new(resource: resource).tap { it.add_span_processor(processor) }
         end
 
+        # The attributes of the experiment span.
+        #
+        # @param experiment [Actions::Experiment] the experiment starting
+        # @return [Hash{String => String, Numeric}]
         def experiment_attributes(experiment)
           model = experiment.model
           { "evals.suite" => context[:suite].to_s, "evals.project" => context[:project].to_s, "evals.experiment" => experiment.name,
@@ -117,6 +161,11 @@ module LLM
             "evals.cases" => experiment.cases.size }
         end
 
+        # The attributes of a case span. A failed case has no score or reply, so it
+        # stops at the error.
+        #
+        # @param result [Results::RunResult] the finished case
+        # @return [Hash{String => String, Numeric}]
         def case_attributes(result)
           kase = result.case
           attributes = { "evals.case" => kase.id, "evals.dataset" => kase.dataset, "evals.variant" => kase.variant,
@@ -136,6 +185,10 @@ module LLM
           attributes
         end
 
+        # Caps an attribute at {DIAGNOSTICS_LIMIT} bytes.
+        #
+        # @param text [String] the JSON or reply text
+        # @return [String] the text, cut and marked with "..." when over the limit
         def truncate(text) = text.bytesize > DIAGNOSTICS_LIMIT ? "#{text.byteslice(0, DIAGNOSTICS_LIMIT)}..." : text
       end
     end

data/lib/llm/evals/trackers/registry.rb

@@ -15,26 +15,38 @@ module LLM
         # @return [Registry] Braintrust and Logfire
         def self.default = new([Braintrust, Logfire])
 
-        # @param classes [Array<Class<Tracker>>]
+        # The array is copied, so registering later does not touch the caller's.
+        #
+        # @param classes [Array<Class<Base>>]
         def initialize(classes = [])
           @classes = classes.dup
         end
 
-        # @param klass [Class<Tracker>] replaces one registered under the same name
-        # @return [Class<Tracker>]
+        # Later registration wins, which is how a suite swaps a built-in tracker.
+        #
+        # @param klass [Class<Base>] replaces one registered under the same name
+        # @return [Class<Base>] the class given
         def register(klass)
           @classes.reject! { it.tracker_name == klass.tracker_name }
           @classes << klass
           klass
         end
 
+        # Walks the classes in registration order.
+        #
+        # @yieldparam klass [Class<Base>] each tracker class
+        # @return [Array<Class<Base>>, Enumerator] the list; an Enumerator without a block
         def each(&) = @classes.each(&)
 
+        # For {#named}'s error message.
+        #
         # @return [Array<String>]
         def names = map(&:tracker_name)
 
-        # @param name [String, Symbol]
-        # @return [Class<Tracker>]
+        # Finds a tracker class by its {Base.tracker_name}.
+        #
+        # @param name [String, Symbol] the name as given
+        # @return [Class<Base>]
         # @raise [Error] naming the known trackers
         def named(name)
           find { it.tracker_name == name.to_s } || raise(Error, "no tracker named #{name.to_s.inspect}; known: #{names.join(", ")}")
@@ -47,9 +59,9 @@ module LLM
         #
         # @param flag [String, nil] what `--tracker` was given
         # @param config [String, nil] the config file's `tracker:`
-        # @param env [Hash]
+        # @param env [#[]] the environment, ENV by default
         # @param local [Boolean] `--local`
-        # @return [Array(Class<Tracker>, String), nil] the class and why it was chosen; nil for none
+        # @return [Array(Class<Base>, String), nil] the class and why it was chosen; nil for none
         # @raise [Error] on an unknown name or an ambiguous environment
         def choose(flag: nil, config: nil, env: ENV, local: false)
           return nil if local
@@ -69,8 +81,18 @@ module LLM
 
         private
 
+        # Whether a flag or config value actually says something; blank counts as unset.
+        #
+        # @param value [String, nil] the value given
+        # @return [Boolean]
         def present?(value) = !value.nil? && !value.to_s.strip.empty?
 
+        # Resolves a named choice, where the {NONE} spellings mean no remote tracker.
+        #
+        # @param name [String] the name as given
+        # @param from [String] where it came from, for `--verbose`
+        # @return [Array(Class<Base>, String), nil] the class and the reason; nil for none
+        # @raise [Error] on an unknown name
         def choice(name, from)
           return nil if NONE.include?(name.to_s)
 

data/lib/llm/evals/trackers.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    # Where results go besides the screen: the local log and at most one remote service.
+    module Trackers
+    end
+  end
+end

data/lib/llm/evals/version.rb

@@ -2,6 +2,8 @@
 
 module LLM
   module Evals
-    VERSION = "0.4.0"
+    # The gem's version, which `evals version` prints.
+    VERSION = "0.5.0"
+    RELEASED_ON = "2026-11-01"
   end
 end

data/lib/llm/evals.rb

@@ -8,14 +8,16 @@ require "dry/cli/ui"
 require "dry/cli/autocomplete/command"
 require "zeitwerk"
 
+# Home of the gem; everything it defines sits under {Evals}.
 module LLM
   # A generic evaluation runner with a one-way boundary. The consumer suite
-  # registers an {Adapter}, which runs one {Case} against a {Model} and
-  # returns a {Reply} without scoring it, and a {Scorer}, which turns a reply
-  # into a {Score} without calling anything. Everything else, from discovery
+  # registers an {Actions::Adapter}, which runs one {Datasets::Case} against a {Models::Model} and
+  # returns a {Results::Reply} without scoring it, and a {Scoring::Scorer}, which turns a reply
+  # into a {Results::Score} without calling anything. Everything else, from discovery
   # to the leaderboard, is this gem's, and none of it knows what is being
   # evaluated.
   module Evals
+    # The base of every error the gem raises on purpose.
     class Error < StandardError; end
 
     # Raised by a command that needs an optional gem the bundle does not hold.
@@ -24,9 +26,15 @@ module LLM
     class << self
       # The suite's settings, filled in by {.configure} from `evals/eval_helper.rb`.
       #
-      # @return [Configuration]
+      # @return [Runtime::Configuration]
       def configuration
-        @configuration ||= Configuration.new
+        @configuration ||= Runtime::Configuration.new
+      end
+
+      # The root of the gem, which holds the default config and the CLI.
+      # @return [String]
+      def root
+        @root ||= File.expand_path("../..", __dir__)
       end
 
       # Builds the suite's configuration. Called once from the eval helper.
@@ -40,8 +48,8 @@ module LLM
       #     c.before_run { MyGem.reset_configuration! }
       #   end
       #
-      # @yieldparam configuration [Configuration]
-      # @return [Configuration]
+      # @yieldparam configuration [Runtime::Configuration]
+      # @return [Runtime::Configuration]
       def configure
         yield configuration
         configuration
@@ -54,6 +62,8 @@ module LLM
         @configuration = nil
       end
 
+      # A suite needs both halves of the boundary, so a helper that set only one does not count.
+      #
       # @return [Boolean] whether a helper has registered an adapter and a scorer
       def configured? = configuration.complete?