agent-evals 0.1.1

data/lib/agent/evals/score.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Agent
+  module Evals
+    # What a {Scorer} says about one reply. `value` is the case's overall
+    # score in 0..1; `metrics` holds one entry per {Metric} the scorer
+    # declares, nil where the case does not apply; `outcomes` and
+    # `diagnostics` are free-form, keyed by field, and only ever displayed.
+    Score = Data.define(:value, :metrics, :outcomes, :diagnostics) do
+      def initialize(value:, metrics: {}, outcomes: {}, diagnostics: {})
+        raise ArgumentError, "score value #{value.inspect} is not within 0..1" unless value.is_a?(Numeric) && (0..1).cover?(value)
+
+        super(value: value.to_f, metrics: metrics.transform_keys(&:to_s), outcomes: outcomes, diagnostics: diagnostics)
+      end
+
+      # The score of a reply the model failed: 0 everywhere, the failure as a diagnostic.
+      #
+      # @param metrics [Array<Metric>]
+      # @param error [String]
+      # @return [Score]
+      def self.failed(metrics, error)
+        new(value: 0.0, metrics: metrics.to_h { [it.name, 0.0] }, diagnostics: { "error" => error })
+      end
+    end
+  end
+end

data/lib/agent/evals/scorer.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Agent
+  module Evals
+    # What a suite implements to score one reply. Pure and deterministic: it
+    # reads the case, the reply and the adapter's context, and calls nothing.
+    #
+    # @abstract Subclass and override {#metrics} and {#call}.
+    class Scorer
+      # The metrics every {Score} this scorer returns carries. The CLI builds
+      # its columns, its glossary and its Braintrust scorers from this list,
+      # so a metric missing here is a metric nobody sees.
+      #
+      # @return [Array<Metric>]
+      def metrics
+        raise NotImplementedError, "#{self.class}#metrics must return an Array of Agent::Evals::Metric"
+      end
+
+      # @return [Metric] the one flagged primary, else the first
+      def primary = metrics.find(&:primary) || metrics.first
+
+      # @param kase [Case]
+      # @param reply [Reply] never one with `error` set; the runner scores those 0 itself
+      # @param context [Hash] what {Adapter#context} returned for the case
+      # @return [Score]
+      def call(kase, reply, context:)
+        raise NotImplementedError, "#{self.class}#call(kase, reply, context:) must return an Agent::Evals::Score"
+      end
+
+      # Offline checks of one case against what the scorer will demand of it,
+      # so a typo in expected.json is caught before it is billed as a model's mistake.
+      #
+      # @param kase [Case]
+      # @param context [Hash] what {Adapter#context} returned for the case
+      # @return [Array<String>] problems; empty when the case is sound
+      def verify(kase, context:) = [] # rubocop:disable Lint/UnusedMethodArgument -- the signature is the contract
+
+      # One short cell describing the case in `agent-evals cases`, e.g. "5/8 answered".
+      #
+      # @param kase [Case]
+      # @return [String, nil] nil for the generic fallback
+      def summary(_kase) = nil
+    end
+  end
+end

data/lib/agent/evals/testing/echo_adapter.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Agent
+  module Evals
+    module Testing
+      # The smallest honest adapter: sends the case's input as the user
+      # prompt, parses the reply as a JSON object, and reports a reply that
+      # is not one as the model's failure. The toy suite runs on it, and a
+      # new suite can start from it.
+      class EchoAdapter < Adapter
+        SYSTEM = "Answer with a single JSON object and nothing else."
+        FENCE = /\A```(?:json)?\s*|\s*```\z/
+
+        # @param kase [Case]
+        # @param model [Model]
+        # @return [Reply]
+        def call(kase, model:)
+          completion = model.complete(system: SYSTEM, user: kase.input, json: true)
+          output = JSON.parse(completion.text.strip.gsub(FENCE, ""))
+          return Reply.new(error: "reply is not a JSON object", raw: completion.text, usage: completion.usage) unless output.is_a?(Hash)
+
+          Reply.new(output: output, raw: completion.text, usage: completion.usage)
+        rescue JSON::ParserError
+          Reply.new(error: "reply is not JSON", raw: completion.text, usage: completion.usage)
+        end
+      end
+    end
+  end
+end

data/lib/agent/evals/testing/fake_model.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Agent
+  module Evals
+    module Testing
+      # A {Model} on a {FakeProvider} of its own, for specs that run an
+      # adapter or an {Experiment} without the CLI.
+      class FakeModel < Model
+        # @param id [String]
+        # @param settings [Hash]
+        # @param provider_options [Hash] passed to {FakeProvider#initialize}
+        def initialize(id: "fake/echo", settings: { effort: nil, temperature: 0.2, max_tokens: 4096 }, **provider_options)
+          super(id: id, provider: FakeProvider.new(**provider_options), settings: settings)
+        end
+      end
+    end
+  end
+end

data/lib/agent/evals/testing/fake_provider.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module Agent
+  module Evals
+    module Testing
+      # Serves every "fake/..." model without a network: a call answers with
+      # the expected JSON of the case whose input matches the user prompt, so
+      # a suite's own plumbing can be run end to end through the executable.
+      #
+      # `junk` names cases (by id or key) that get a non-JSON reply, and
+      # `wrong` maps a case to fields to answer wrongly. `answers`, a callable
+      # taking the user prompt, replaces the lookup altogether.
+      class FakeProvider < Providers::Base
+        PREFIX = "fake/"
+        USAGE = Usage.new(input_tokens: 100, output_tokens: 10, cost_usd: 0.001)
+        JUNK = "Sure! Here is my answer, in prose rather than JSON."
+
+        attr_reader :calls
+
+        # @param junk [Array<String>] case ids or keys to answer with prose
+        # @param wrong [Hash{String => Hash}] case id or key => fields to override
+        # @param answers [#call, nil] user prompt => reply text; replaces the lookup
+        # @param discovery [Discovery, nil] where to look cases up; the configured suite's when nil
+        def initialize(junk: [], wrong: {}, answers: nil, discovery: nil)
+          super()
+          @junk = junk
+          @wrong = wrong
+          @answers = answers
+          @discovery = discovery
+          @calls = []
+          @lock = Mutex.new
+        end
+
+        def name = "fake"
+        def request_type = :prompt
+        def prefix = PREFIX
+        def env_keys = []
+
+        # @return [nil] any id passes verification
+        def catalog = nil
+
+        # @param model [Model]
+        # @param system [String, nil]
+        # @param user [String]
+        # @param json [Boolean]
+        # @return [Completion]
+        def complete(model, system:, user:, json: true)
+          @lock.synchronize { @calls << { model: model.id, system: system, user: user, json: json } }
+          Completion.new(text: @answers ? @answers.call(user) : answer(user), usage: USAGE)
+        end
+
+        private
+
+        def answer(user)
+          kase = discovery.all.find { it.input == user }
+          return "{}" if kase.nil?
+          return JUNK if @junk.include?(kase.id) || @junk.include?(kase.key)
+
+          wrong = @wrong[kase.id] || @wrong[kase.key] || {}
+          JSON.generate(kase.expected.is_a?(Hash) ? kase.expected.merge(wrong) : kase.expected)
+        end
+
+        def discovery = @discovery || Agent::Evals.configuration.discovery
+      end
+    end
+  end
+end

data/lib/agent/evals/tracker.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module Agent
+  module Evals
+    # Where a run's results go besides the terminal. The gem always owns the
+    # run loop; a tracker only receives callbacks: {#start} once per
+    # experiment, {#record} once per finished case, {#finish} once with the
+    # summary. The local JSON log ({Trackers::Local}) always runs; at most
+    # one remote tracker runs beside it, chosen by `--tracker`, the
+    # `tracker:` key, or the environment (see {Trackers::Registry#choose}).
+    #
+    # A tracker for another framework (W&B Weave, an OTLP backend) subclasses
+    # this in a consumer or a plugin gem and is registered with `c.tracker`.
+    #
+    # @abstract Override the three callbacks; declare {.env_key} and {.gems}.
+    class Tracker
+      class << self
+        # @return [String] the name `--tracker` and `tracker:` use; the class name, dasherized, by default
+        def tracker_name = (name || "Tracker").split("::").last.gsub(/([a-z\d])([A-Z])/, '\1-\2').downcase
+
+        # @return [String, nil] the environment variable whose presence selects this tracker when nothing else does
+        def env_key = nil
+
+        # @return [Array<String>] the optional gems this tracker requires when chosen
+        def gems = []
+
+        # @param env [Hash] the environment
+        # @return [Boolean] whether the environment names this tracker
+        def detect?(env = ENV) = !env_key.nil? && !env[env_key].to_s.strip.empty?
+
+        # Loads the gems and checks the key, with an error that names both, so
+        # a chosen tracker that cannot run says so before any model is called.
+        #
+        # @param env [Hash]
+        # @return [void]
+        # @raise [MissingDependency, Error]
+        def requirements!(env = ENV)
+          gems.each do |gem_name|
+            Agent::Evals.optional!(gem_name.tr("-", "/"), gem_name, needed_for: "the #{tracker_name} tracker")
+          end
+          return if env_key.nil? || detect?(env)
+
+          raise Error, "the #{tracker_name} tracker needs #{env_key} in the environment (gems: #{gems.join(", ")})"
+        end
+      end
+
+      attr_reader :context
+
+      # @param context [Hash] `suite:`, `project:`, `project_id:` and `options:` (the tracker's own key in the config file)
+      def initialize(**context)
+        @context = context
+      end
+
+      # @return [String]
+      def name = self.class.tracker_name
+
+      # @return [Boolean] false turns every callback into a no-op
+      def enabled? = true
+
+      # @return [String] one line for `--verbose`: where this tracker sends things, and why
+      def describe = name
+
+      # @param experiment [Experiment]
+      # @return [void]
+      def start(_experiment) = nil
+
+      # @param result [RunResult]
+      # @return [void]
+      def record(_result) = nil
+
+      # @param experiment [Experiment]
+      # @param summary [Experiment::Summary]
+      # @return [Hash] `url:` to the experiment when one is known, else nil
+      def finish(_experiment, _summary) = { url: nil }
+
+      # @return [Hash] the tracker's own section of the config file
+      def options = context.fetch(:options, nil) || {}
+    end
+  end
+end

data/lib/agent/evals/trackers/braintrust.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+module Agent
+  module Evals
+    module Trackers
+      # Records each case as a Braintrust eval row in an experiment of the
+      # suite's project, through the braintrust gem's own OpenTelemetry
+      # exporter, the way `Braintrust::Eval.run` would have, except that the
+      # loop stays ours. The gem is required only when this tracker is chosen.
+      class Braintrust < Tracker
+        def self.env_key = "BRAINTRUST_API_KEY"
+        def self.gems = ["braintrust"]
+
+        attr_reader :experiment_id, :permalink
+
+        def describe = "braintrust: project #{context[:project_id] || context[:project].inspect}"
+
+        # Logs in, finds or creates the project, creates the experiment.
+        #
+        # @param experiment [Experiment]
+        # @return [void]
+        def start(experiment)
+          ::Braintrust.init(blocking_login: true, auto_instrument: false)
+          @state = ::Braintrust.current_state
+          @state.login
+          project_id = context[:project_id] || ::Braintrust::API::Internal::Projects.new(@state).create(name: context[:project]).fetch("id")
+          @experiment_id = ::Braintrust::API::Internal::Experiments.new(@state).create(
+            name:       experiment.name,
+            project_id: project_id,
+            ensure_new: true,
+            tags:       [context[:suite], experiment.model.id, "dataset-#{experiment.dataset}"].compact,
+            metadata:   metadata(experiment)
+          ).fetch("id")
+          @permalink = @state.object_permalink(object_type: "experiment", object_id: experiment_id)
+          # Braintrust.init installs its exporter on the global provider, which
+          # is where the gem's own runner takes its tracer from too.
+          @tracer = OpenTelemetry.tracer_provider.tracer("agent-evals")
+        end
+
+        # One root "eval" span per case, with the shape Braintrust's own
+        # runner emits: input, expected, output, scores and metadata as JSON
+        # attributes, parented to the experiment.
+        #
+        # @param result [RunResult]
+        # @return [void]
+        def record(result)
+          span = @tracer.start_root_span("eval")
+          span.set_attribute("braintrust.parent", "experiment_id:#{experiment_id}")
+          json(span, "braintrust.span_attributes", { type: "eval", name: result.experiment.name })
+          json(span, "braintrust.input_json", { case: result.case.key, text: result.case.input })
+          json(span, "braintrust.expected_json", result.case.expected) unless result.case.expected.nil?
+          json(span, "braintrust.metadata", case_metadata(result))
+          span.set_attribute("braintrust.tags", ["variant-#{result.case.variant}", "dataset-#{result.case.dataset}"])
+          if result.failed?
+            span.status = OpenTelemetry::Trace::Status.error(result.error)
+          else
+            json(span, "braintrust.output_json", result.reply.error? ? { "_error" => result.reply.error } : result.reply.output)
+            json(span, "braintrust.scores", result.score.metrics.compact)
+          end
+        ensure
+          span&.finish
+        end
+
+        # Flushes the spans and hands back the experiment's page.
+        #
+        # @return [Hash] `url:` the experiment permalink
+        def finish(_experiment, _summary)
+          provider = OpenTelemetry.tracer_provider
+          provider.force_flush if provider.respond_to?(:force_flush)
+          { url: permalink }
+        end
+
+        private
+
+        def json(span, key, value) = span.set_attribute(key, JSON.dump(value))
+
+        def metadata(experiment)
+          { model: experiment.model.id, provider: experiment.model.provider.name, dataset: experiment.dataset,
+            **experiment.model.settings, **experiment.adapter.metadata }
+        end
+
+        def case_metadata(result)
+          base = { case: result.case.key, dataset: result.case.dataset, variant: result.case.variant, difficulty: result.case.difficulty }
+          return base.merge(error: result.error) if result.failed?
+
+          base.merge(diagnostics: result.score.diagnostics, outcomes: result.score.outcomes, usage: result.reply.usage.to_h)
+        end
+      end
+    end
+  end
+end

data/lib/agent/evals/trackers/local.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+require "fileutils"
+
+module Agent
+  module Evals
+    module Trackers
+      # Always on: every experiment's results and summary land in one JSON
+      # file under `log/`, so a run is never lost to a closed terminal or a
+      # remote tracker that was never configured.
+      class Local < Tracker
+        attr_reader :dir, :path
+
+        # @param dir [String] where the files go; `log/agent-evals` at the repository root by default
+        # @param context [Hash] see {Tracker#initialize}
+        def initialize(dir: File.join(Dry::CLI::UI::Invocation.root(Dir.pwd), "log", "agent-evals"), **context)
+          super(**context)
+          @dir = dir
+          @results = []
+          @lock = Mutex.new
+        end
+
+        def describe = "local: #{dir}"
+
+        def start(experiment)
+          @started_at = Time.now
+          @path = File.join(dir, "#{slug(experiment.name)}.#{@started_at.strftime("%Y-%m-%d.%H%M%S")}.json")
+        end
+
+        def record(result)
+          @lock.synchronize { @results << result }
+        end
+
+        # @return [Hash] `url:` nil, `path:` the file written
+        def finish(experiment, summary)
+          FileUtils.mkdir_p(dir)
+          File.write(path, JSON.pretty_generate(document(experiment, summary)))
+          { url: nil, path: path }
+        end
+
+        private
+
+        def slug(text) = text.gsub(/[^A-Za-z0-9.-]+/, "_").squeeze("_").delete_prefix("_").delete_suffix("_")
+
+        def document(experiment, summary)
+          {
+            suite:      context[:suite],
+            experiment: experiment.name,
+            model:      experiment.model.id,
+            provider:   experiment.model.provider.name,
+            settings:   experiment.model.settings,
+            dataset:    experiment.dataset,
+            started_at: @started_at.iso8601,
+            duration:   summary.duration,
+            means:      summary.means,
+            errors:     summary.errors,
+            usage:      summary.usage.to_h,
+            results:    @lock.synchronize { @results.sort_by { experiment.cases.index(it.case) }.map { row(it) } }
+          }
+        end
+
+        def row(result)
+          {
+            case:        result.case.key,
+            duration:    result.duration,
+            error:       result.error,
+            score:       result.score&.value,
+            metrics:     result.score&.metrics,
+            outcomes:    result.score&.outcomes,
+            diagnostics: result.score&.diagnostics,
+            reply_error: result.reply&.error,
+            usage:       result.reply&.usage&.to_h
+          }
+        end
+      end
+    end
+  end
+end

data/lib/agent/evals/trackers/logfire.rb

@@ -0,0 +1,143 @@
+# frozen_string_literal: true
+
+module Agent
+  module Evals
+    module Trackers
+      # Pydantic Logfire, which has no Ruby gem of its own: it is an OTLP
+      # backend for the standard OpenTelemetry Ruby SDK (pydantic.dev/logfire/ruby,
+      # read 2026-10-15). Traces only; Logfire's metrics and logs SDKs are
+      # pre-1.0. One span per experiment, one child span per case.
+      #
+      # The write token comes from `LOGFIRE_TOKEN`, our name for what their
+      # page passes as a raw `Authorization` header. When the standard
+      # `OTEL_EXPORTER_OTLP_ENDPOINT` or `OTEL_EXPORTER_OTLP_HEADERS` is
+      # already set, the SDK's own reading of it wins and {#describe} says so.
+      # The raw reply is not sent unless the config file opts in with
+      # `logfire: { include_replies: true }`.
+      class Logfire < Tracker
+        # Logfire's US region, per their docs on 2026-10-15; other regions
+        # exist, so LOGFIRE_BASE_URL overrides it.
+        DEFAULT_BASE_URL = "https://logfire-us.pydantic.dev"
+        TOKEN = "LOGFIRE_TOKEN"
+        BASE_URL = "LOGFIRE_BASE_URL"
+        OTEL_ENDPOINT = "OTEL_EXPORTER_OTLP_ENDPOINT"
+        OTEL_HEADERS = "OTEL_EXPORTER_OTLP_HEADERS"
+
+        # Bytes of the diagnostics JSON kept on a case span; the rest is cut,
+        # since an attribute is not the place for a whole transcript.
+        DIAGNOSTICS_LIMIT = 4096
+
+        def self.env_key = TOKEN
+        def self.gems = %w[opentelemetry-sdk opentelemetry-exporter-otlp]
+
+        attr_reader :endpoint, :source
+
+        # @param exporter [#export, nil] an exporter to use instead of OTLP, for tests
+        # @param env [Hash]
+        # @param context [Hash] see {Tracker#initialize}
+        def initialize(exporter: nil, env: ENV, **context)
+          super(**context)
+          @env = env
+          @exporter = exporter
+          @endpoint, @headers, @source = settings(env)
+          @cases = []
+        end
+
+        def describe = "logfire: endpoint #{endpoint || "(SDK default)"}, credentials from #{source}"
+
+        # @return [Boolean] whether raw replies ride along on the case spans
+        def include_replies? = options["include_replies"] == true || options[:include_replies] == true
+
+        def start(experiment)
+          @provider = provider
+          @tracer = @provider.tracer("agent-evals", Agent::Evals::VERSION)
+          @span = @tracer.start_span("experiment #{experiment.name}", attributes: experiment_attributes(experiment))
+        end
+
+        def record(result)
+          now = Time.now
+          span = @tracer.start_span("case #{result.case.key}",
+            with_parent:     OpenTelemetry::Trace.context_with_span(@span),
+            start_timestamp: now - result.duration,
+            attributes:      case_attributes(result))
+          if result.failed?
+            span.record_exception(Error.new(result.error))
+            span.status = OpenTelemetry::Trace::Status.error(result.error)
+          end
+          span.finish(end_timestamp: now)
+        end
+
+        # Closes the experiment span with the summary and flushes. No URL:
+        # Logfire's trace pages are not something the token lets us build.
+        #
+        # @return [Hash] `url:` nil
+        def finish(_experiment, summary)
+          summary.means.each { |metric, mean| @span.set_attribute("evals.mean.#{metric}", mean) unless mean.nil? }
+          @span.set_attribute("evals.errors", summary.errors.size)
+          @span.set_attribute("evals.duration_seconds", summary.duration)
+          @span.set_attribute("evals.usage.cost_usd", summary.usage.cost_usd)
+          @span.finish
+          @provider.force_flush
+          @provider.shutdown unless @exporter # an injected exporter is the caller's to read afterwards
+          { url: nil }
+        end
+
+        private
+
+        # Which credentials win: the standard OTEL_* variables when either is
+        # set, else our LOGFIRE_* pair. Returns the endpoint and headers to
+        # hand the exporter (nil leaves the SDK's own reading in place).
+        def settings(env)
+          if [OTEL_ENDPOINT, OTEL_HEADERS].any? { !env[it].to_s.empty? }
+            [env[OTEL_ENDPOINT], nil, "#{OTEL_ENDPOINT}/#{OTEL_HEADERS} (set in the environment; #{TOKEN} ignored)"]
+          else
+            base = env[BASE_URL].to_s.empty? ? DEFAULT_BASE_URL : env[BASE_URL]
+            ["#{base.chomp("/")}/v1/traces", { "Authorization" => env[TOKEN].to_s }, "#{TOKEN}#{" and #{BASE_URL}" unless env[BASE_URL].to_s.empty?}"]
+          end
+        end
+
+        # A tracer provider of our own, so the global one, which a consumer
+        # may have configured for its application, is left alone.
+        def provider
+          exporter = @exporter || OpenTelemetry::Exporter::OTLP::Exporter.new(**{ endpoint: endpoint, headers: @headers }.compact)
+          processor = if @exporter
+                        OpenTelemetry::SDK::Trace::Export::SimpleSpanProcessor.new(exporter)
+                      else
+                        OpenTelemetry::SDK::Trace::Export::BatchSpanProcessor.new(exporter)
+                      end
+          resource = OpenTelemetry::SDK::Resources::Resource.create("service.name" => context[:suite] || "agent-evals")
+          OpenTelemetry::SDK::Trace::TracerProvider.new(resource: resource).tap { it.add_span_processor(processor) }
+        end
+
+        def experiment_attributes(experiment)
+          model = experiment.model
+          { "evals.suite" => context[:suite].to_s, "evals.project" => context[:project].to_s, "evals.experiment" => experiment.name,
+            "evals.model" => model.id, "evals.provider" => model.provider.name, "evals.dataset" => experiment.dataset,
+            "evals.effort" => model.effort.to_s, "evals.temperature" => model.temperature.to_f, "evals.max_tokens" => model.max_tokens.to_i,
+            "evals.cases" => experiment.cases.size }
+        end
+
+        def case_attributes(result)
+          kase = result.case
+          attributes = { "evals.case" => kase.id, "evals.dataset" => kase.dataset, "evals.variant" => kase.variant,
+                         "evals.duration_seconds" => result.duration }
+          return attributes.merge("evals.error" => result.error) if result.failed?
+
+          score = result.score
+          attributes["evals.score"] = score.value
+          score.metrics.each { |metric, value| attributes["evals.metric.#{metric}"] = value unless value.nil? }
+          score.outcomes.group_by { |_, v| v.to_s }.each { |outcome, pairs| attributes["evals.outcome.#{outcome}"] = pairs.size }
+          attributes["evals.diagnostics"] = truncate(JSON.generate(score.diagnostics))
+          attributes["evals.usage.input_tokens"] = result.reply.usage.input_tokens
+          attributes["evals.usage.output_tokens"] = result.reply.usage.output_tokens
+          attributes["evals.usage.cost_usd"] = result.reply.usage.cost_usd
+          attributes["evals.reply_error"] = result.reply.error if result.reply.error?
+          attributes["evals.reply"] = truncate(result.reply.raw.to_s) if include_replies?
+          attributes
+        end
+
+        def truncate(text) = text.bytesize > DIAGNOSTICS_LIMIT ? "#{text.byteslice(0, DIAGNOSTICS_LIMIT)}..." : text
+      end
+    end
+  end
+end

data/lib/agent/evals/trackers/registry.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+module Agent
+  module Evals
+    module Trackers
+      # The tracker classes a run may choose from, by name, and the choice
+      # itself: exactly one remote tracker, or none. A consumer or a plugin
+      # gem registers its own with `c.tracker`.
+      class Registry
+        include Enumerable
+
+        # Spellings of "no remote tracker" on the command line and in the config.
+        NONE = %w[none local off].freeze
+
+        # @return [Registry] Braintrust and Logfire
+        def self.default = new([Braintrust, Logfire])
+
+        # @param classes [Array<Class<Tracker>>]
+        def initialize(classes = [])
+          @classes = classes.dup
+        end
+
+        # @param klass [Class<Tracker>] replaces one registered under the same name
+        # @return [Class<Tracker>]
+        def register(klass)
+          @classes.reject! { it.tracker_name == klass.tracker_name }
+          @classes << klass
+          klass
+        end
+
+        def each(&) = @classes.each(&)
+
+        # @return [Array<String>]
+        def names = map(&:tracker_name)
+
+        # @param name [String, Symbol]
+        # @return [Class<Tracker>]
+        # @raise [Error] naming the known trackers
+        def named(name)
+          find { it.tracker_name == name.to_s } || raise(Error, "no tracker named #{name.to_s.inspect}; known: #{names.join(", ")}")
+        end
+
+        # The one remote tracker to run, in this precedence: `--tracker`,
+        # then the config's `tracker:`, then the environment, where a single
+        # tracker's key being set selects it and two or more is an ambiguity
+        # to be settled by naming one. `--local` is none.
+        #
+        # @param flag [String, nil] what `--tracker` was given
+        # @param config [String, nil] the config file's `tracker:`
+        # @param env [Hash]
+        # @param local [Boolean] `--local`
+        # @return [Array(Class<Tracker>, String), nil] the class and why it was chosen; nil for none
+        # @raise [Error] on an unknown name or an ambiguous environment
+        def choose(flag: nil, config: nil, env: ENV, local: false)
+          return nil if local
+          return choice(flag, "--tracker") if present?(flag)
+          return choice(config, "tracker: in the config file") if present?(config)
+
+          detected = @classes.select { it.detect?(env) }
+          case detected.size
+          when 0 then nil
+          when 1 then [detected.first, "#{detected.first.env_key} is set (pass --tracker or set tracker: to choose explicitly)"]
+          else
+            raise Error,
+              "more than one tracker is configured in the environment (#{detected.map(&:env_key).join(", ")}); " \
+              "choose one with --tracker #{detected.map(&:tracker_name).join("|")} or tracker: in the config file"
+          end
+        end
+
+        private
+
+        def present?(value) = !value.nil? && !value.to_s.strip.empty?
+
+        def choice(name, from)
+          return nil if NONE.include?(name.to_s)
+
+          [named(name), from]
+        end
+      end
+    end
+  end
+end