agent-evals 0.1.1

data/lib/agent/evals/config.rb

@@ -0,0 +1,146 @@
+# frozen_string_literal: true
+
+require "yaml"
+
+module Agent
+  module Evals
+    # The suite's `.agent-eval.config.yml`: which models to compare, at which
+    # settings, and which kinds of request the suite can produce.
+    #
+    #   project: my-evals            # Braintrust project name
+    #   request_types: [prompt]      # required: what the adapter sends; a provider must match
+    #   tracker: braintrust          # optional: the one remote tracker; else the environment decides
+    #   logfire:                     # optional: a tracker's own settings
+    #     include_replies: true
+    #   require: my_gem/evals        # optional: a path (or list) to require, for a suite with no eval_helper.rb
+    #   adapter: MyGem::Evals::Adapter   # optional: constant names, instantiated with no arguments
+    #   scorer: MyGem::Evals::Scorer
+    #   defaults:
+    #     effort: default
+    #     temperature: 0.2
+    #     max_tokens: 4096
+    #     parallelism: 4
+    #   vendors:                     # the OpenRouter list; every entry takes a prompt
+    #     anthropic:
+    #       - model: claude-haiku-5.5
+    #         effort: [default, low]  # a list expands into one contender per value
+    #   direct:                      # a vendor's own API, one request type each
+    #     typesafe:
+    #       - model: jev-latest
+    #
+    # Each entry becomes "<vendor>/<model>", the id a provider recognises. A
+    # key under `direct:` names a provider; its entries carry that provider.
+    class Config
+      # The request types a suite may declare. `prompt` is a chat completion:
+      # system and user text in, text out. `typesafe` is TypeSafe's typed
+      # question API, whose request shape the first such suite will define.
+      REQUEST_TYPES = %w[prompt typesafe].freeze
+
+      BUILTIN = {
+        "effort" => "default", "temperature" => 0.2, "max_tokens" => 4096, "parallelism" => 4
+      }.freeze
+
+      attr_reader :path, :direct_providers
+
+      # @param path [String] read when it exists; an absent file is an empty lineup
+      # @param direct_providers [Array<String>] the provider names `direct:` may use
+      def initialize(path, direct_providers: Providers::Registry.default.direct_names)
+        @path = path
+        @direct_providers = direct_providers
+        @data = File.exist?(path) ? (YAML.safe_load_file(path) || {}) : {}
+      end
+
+      # @return [String, nil] the Braintrust project name
+      def project = @data["project"]
+
+      # The remote tracker the file chooses; a list of more than one is an
+      # error, since exactly one remote tracker runs, or none.
+      #
+      # @return [String, nil]
+      # @raise [Error] on a list of several
+      def tracker
+        listed = Array(@data["tracker"]).map(&:to_s)
+        raise Error, "#{path}: tracker: names #{listed.join(", ")}; exactly one remote tracker runs, or none" if listed.size > 1
+
+        listed.first
+      end
+
+      # A tracker's own section of the file, e.g. `logfire: { include_replies: true }`.
+      #
+      # @param name [String]
+      # @return [Hash]
+      def tracker_options(name) = (@data[name.to_s].is_a?(Hash) ? @data[name.to_s] : {}).transform_keys(&:to_s)
+
+      # @return [Array<String>] the `require:` key, one path or a list
+      def requires = Array(@data["require"]).map(&:to_s)
+
+      # @return [String, nil] the `adapter:` constant name
+      def adapter_name = @data["adapter"]&.to_s
+
+      # @return [String, nil] the `scorer:` constant name
+      def scorer_name = @data["scorer"]&.to_s
+
+      # The kinds of request the suite's adapter produces. A provider whose
+      # request type is not listed cannot serve this suite, whatever model it
+      # names, so the list is required: without it every contender would be
+      # admitted on faith and fail at the first call.
+      #
+      # @return [Array<Symbol>]
+      # @raise [Error] when the key is missing, empty, or names an unknown type
+      def request_types
+        listed = Array(@data["request_types"]).map(&:to_s)
+        raise Error, "#{path}: request_types is required; list one or more of #{REQUEST_TYPES.join(", ")}" if listed.empty?
+
+        unknown = listed - REQUEST_TYPES
+        raise Error, "#{path}: unknown request_types #{unknown.join(", ")}; known: #{REQUEST_TYPES.join(", ")}" if unknown.any?
+
+        listed.map(&:to_sym)
+      end
+
+      # @return [Hash{String => Object}] the built-in defaults under the file's own
+      def defaults = BUILTIN.merge(@data.fetch("defaults", nil) || {})
+
+      # Every vendor/model/effort/temperature combination the file lists,
+      # `vendors:` first, then `direct:`.
+      #
+      # @param overrides [Hash{String => Object}] values from the command line; nil entries are ignored
+      # @return [Array<Contender>]
+      # @raise [Error] on a key under `direct:` that names no provider
+      def contenders(overrides = {})
+        vendors = (@data["vendors"] || {}).flat_map do |vendor, entries|
+          Array(entries).flat_map { expand("#{vendor}/#{it.fetch("model")}", it, overrides) }
+        end
+        direct = (@data["direct"] || {}).flat_map do |name, entries|
+          unless direct_providers.include?(name.to_s)
+            raise Error, "#{path}: unknown provider #{name.inspect} under direct:; known: #{direct_providers.join(", ")}"
+          end
+
+          Array(entries).flat_map { expand("#{name}/#{it.fetch("model")}", it, overrides, provider: name.to_sym) }
+        end
+        vendors + direct
+      end
+
+      # The models given on the command line, at the file's default settings.
+      #
+      # @param models [Array<String>] model ids
+      # @param overrides [Hash{String => Object}]
+      # @return [Array<Contender>]
+      def contenders_for(models, overrides = {})
+        models.flat_map { expand(it, {}, overrides) }
+      end
+
+      private
+
+      def expand(model, entry, overrides, provider: nil)
+        settings = defaults.merge(entry).merge(overrides.compact)
+        Array(settings["effort"]).product(Array(settings["temperature"])).map do |effort, temperature|
+          Contender.new(model: model,
+            effort: effort.to_s,
+            temperature: temperature,
+            max_tokens: Integer(settings["max_tokens"]),
+            provider: provider)
+        end
+      end
+    end
+  end
+end

data/lib/agent/evals/configuration.rb

@@ -0,0 +1,172 @@
+# frozen_string_literal: true
+
+module Agent
+  module Evals
+    # Everything a suite tells the gem, through {Agent::Evals.configure}.
+    # Paths are relative to `root`, the directory the eval helper lives in.
+    class Configuration
+      DEFAULT_GLOB = "datasets/*/cases/**/input.txt"
+      DEFAULT_CONFIG = ".agent-eval.config.yml"
+
+      attr_writer :adapter, :scorer
+      attr_reader :name, :root, :env_files, :cases_glob, :hooks, :providers, :trackers, :description
+
+      def initialize
+        @name = "agent-evals"
+        @root = Dir.pwd
+        @env_files = []
+        @cases_glob = DEFAULT_GLOB
+        @config_path = DEFAULT_CONFIG
+        @hooks = []
+        @providers = Providers::Registry.default
+        @trackers = Trackers::Registry.default
+        @description = "Run a suite of cases against one or more models and score the replies."
+        @project = nil
+      end
+
+      # Names the suite. The name titles the help and tags the experiments.
+      #
+      # @param name [String]
+      # @param root [String] where .agent-eval.config.yml and datasets/ live; the helper's directory
+      # @param env_files [Array<String>] dotenv files to load, relative to root; the first that exists wins nothing,
+      #   all that exist load, and an exported variable beats them all
+      # @param description [String, nil] one line for the help's header
+      # @return [void]
+      def suite(name, root:, env_files: [], description: nil)
+        @name = name
+        @root = File.expand_path(root)
+        @env_files = env_files.map { File.expand_path(it, @root) }
+        @description = description if description
+      end
+
+      # Where the cases are: a glob relative to root ending in the input file.
+      # Each match is a case; the expected answer is `expected.json` beside it.
+      #
+      # @param glob [String]
+      # @return [void]
+      def cases(glob)
+        @cases_glob = glob
+      end
+
+      # @param object [Adapter]
+      # @return [void]
+
+      # @param object [Adapter, nil] setting; reading when nil
+      # @return [Adapter]
+      def adapter(object = nil)
+        @adapter = object unless object.nil?
+        @adapter
+      end
+
+      # @param object [Scorer, nil] setting; reading when nil
+      # @return [Scorer]
+      def scorer(object = nil)
+        @scorer = object unless object.nil?
+        @scorer
+      end
+
+      # @param object [Scorer]
+      # @return [void]
+
+      # Runs once, after the env files are loaded and before any model is
+      # called. The place to reset a library that read ENV at require time.
+      #
+      # @yield
+      # @return [void]
+      def before_run(&block)
+        @hooks << block
+      end
+
+      # Adds a provider, ahead of the defaults, so a suite's own transport
+      # wins the ids it claims.
+      #
+      # @param provider [Providers::Base]
+      # @return [void]
+      def provider(provider)
+        @providers.register(provider)
+      end
+
+      # Adds a tracker class, so a run can choose it by name with `--tracker`
+      # or `tracker:`, or have its env key select it.
+      #
+      # @param klass [Class<Tracker>]
+      # @return [void]
+      def tracker(klass)
+        @trackers.register(klass)
+      end
+
+      # @param path [String, nil] setting, relative to root; reading when nil
+      # @return [String] the absolute path of the lineup file, `.agent-eval.config.yml` at the root by default
+      def config(path = nil)
+        @config_path = path unless path.nil?
+        File.expand_path(@config_path, root)
+      end
+
+      # @param name [String, nil] setting; reading when nil
+      # @return [String, nil] the Braintrust project, when .agent-eval.config.yml does not name one
+      def project(name = nil)
+        @project = name unless name.nil?
+        @project
+      end
+
+      # @return [Discovery] the cases, as the glob finds them
+      def discovery = Discovery.new(root: root, glob: cases_glob)
+
+      # @return [Config] the lineup file
+      def lineup = Config.new(config, direct_providers: providers.direct_names)
+
+      # @return [Runtime]
+      def runtime = Runtime.new(env_files: env_files, hooks: hooks, app_name: name)
+
+      # @return [Boolean] whether an adapter and a scorer are registered
+      def complete? = !adapter.nil? && !scorer.nil?
+
+      # Fails a command that needs the suite when no helper was loaded, with
+      # the fix in the message, instead of a NoMethodError on nil.
+      #
+      # @return [self]
+      # @raise [Error]
+      def require_suite!
+        resolve_from_config!
+        return self if complete?
+
+        raise Error,
+          "no suite is configured: register an adapter and a scorer in evals/eval_helper.rb, or set " \
+          "adapter: and scorer: (with require:) in #{config}; --helper PATH or AGENT_EVALS_HELPER names another helper"
+      end
+
+      # Fills whatever the helper left unset from the config file's `require:`,
+      # `adapter:` and `scorer:` keys, so a suite can be config-only: its Ruby
+      # lives in its own gem and `evals/` holds the file and the datasets. The
+      # helper, when there is one, was loaded first and wins.
+      #
+      # @return [void]
+      # @raise [Error] naming the key and the value when a require fails or a constant is unknown
+      def resolve_from_config!
+        return if complete? || !File.exist?(config)
+
+        file = lineup
+        file.requires.each do |feature|
+          require feature
+        rescue LoadError => e
+          raise Error, "#{config}: require: #{feature} could not be loaded (#{e.message}); is its gem in the bundle?"
+        end
+        @adapter ||= instantiate("adapter", file.adapter_name)
+        @scorer ||= instantiate("scorer", file.scorer_name) # rubocop:disable Naming/MemoizedInstanceVariableName
+      end
+
+      private
+
+      # @return [Object, nil] a new instance of the named constant; nil when no name is set
+      def instantiate(key, name)
+        return nil if name.nil? || name.empty?
+
+        Object.const_get(name).new
+      rescue NameError, ArgumentError => e
+        raise Error,
+          "#{config}: #{key}: #{name} could not be instantiated (#{e.class}: #{e.message}); " \
+          "name a constant the require: key loads, taking no arguments"
+      end
+    end
+  end
+end

data/lib/agent/evals/contender.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Agent
+  module Evals
+    # One model at one setting: what a single experiment runs. Built by
+    # {Config} from the lineup file, or from `-m` on the command line.
+    #
+    # `provider` is nil for a `vendors:` entry, which OpenRouter serves, and
+    # the provider's name for a `direct:` entry. It travels with the
+    # contender rather than being read back out of the id, so `typesafe/jev-router`
+    # (Jev on OpenRouter) and a direct `typesafe/jev-latest` can never be confused.
+    Contender = Data.define(:model, :effort, :temperature, :max_tokens, :provider) do
+      def initialize(model:, effort: "default", temperature: "default", max_tokens: nil, provider: nil) = super
+
+      # @return [Boolean] whether the entry came from `direct:`
+      def direct? = !provider.nil?
+
+      # @return [String] unique across the lineup: "typesafe:typesafe/jev-latest" for a direct entry
+      def key = direct? ? "#{provider}:#{model}" : model
+
+      # The settings as a {Model} takes them: "default" means nil, send nothing.
+      #
+      # @return [Hash{Symbol => Object}]
+      def settings
+        { effort:      effort.to_s == "default" || effort.nil? ? nil : effort.to_s,
+          temperature: temperature.to_s == "default" || temperature.nil? ? nil : Float(temperature),
+          max_tokens:  max_tokens.nil? ? nil : Integer(max_tokens) }
+      end
+    end
+  end
+end

data/lib/agent/evals/discovery.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+module Agent
+  module Evals
+    # Finds the cases under a root with a glob and decodes each path into a
+    # {Case}. The default layout is
+    #
+    #   datasets/<dataset>/cases/<variant>/[NNN/]input.txt
+    #   datasets/<dataset>/cases/<variant>/[NNN/]expected.json
+    #
+    # The folder after `datasets/` is the dataset, the folder holding the
+    # input is the variant, unless it is a number, in which case it is the
+    # case number and its parent is the variant.
+    class Discovery
+      EXPECTED = "expected.json"
+      METADATA = "metadata.json"
+
+      attr_reader :root, :glob
+
+      # @param root [String] an absolute directory
+      # @param glob [String] relative to root
+      def initialize(root:, glob: Configuration::DEFAULT_GLOB)
+        @root = root
+        @glob = glob
+      end
+
+      # @return [Array<String>] every dataset that has at least one case, sorted
+      def datasets = all.map(&:dataset).uniq.sort
+
+      # @param dataset [String, nil] one dataset; nil for all of them
+      # @param only [Array<String>] id prefixes to keep ("clear", "clear/00"); empty keeps all
+      # @return [Array<Case>] sorted by dataset, variant and number
+      # @raise [Error] when the dataset does not exist
+      def cases(dataset: nil, only: [])
+        found = all
+        if dataset
+          raise Error, "no dataset #{dataset.inspect}; have #{datasets.join(", ")}" unless datasets.include?(dataset)
+
+          found = found.select { it.dataset == dataset }
+        end
+        only = only.flat_map { it.split(",") }.map(&:strip).reject(&:empty?)
+        found = found.select { |kase| only.any? { kase.id.start_with?(it) || kase.key.start_with?(it) } } unless only.empty?
+        found
+      end
+
+      # @return [Array<Case>]
+      def all
+        @all ||= Dir[File.join(root, glob)].map { decode(it) }
+      end
+
+      # Builds the case at one input file.
+      #
+      # @param input_path [String]
+      # @return [Case]
+      def decode(input_path)
+        dir = File.dirname(input_path)
+        parts = Pathname(dir).relative_path_from(Pathname(root)).each_filename.to_a
+        number = parts.last.match?(/\A\d+\z/) ? parts.last : nil
+        variant = number ? parts[-2] : parts.last
+        expected, problem = read_expected(File.join(dir, EXPECTED))
+        Case.new(
+          id:       [variant, number].compact.join("/"),
+          dataset:  dataset_of(parts),
+          variant:  variant,
+          input:    File.read(input_path).strip,
+          expected: expected,
+          path:     dir,
+          metadata: metadata(dir, number, problem)
+        )
+      end
+
+      private
+
+      # The folder after `datasets/`, else the first folder, so a glob that
+      # does not say `datasets` still groups by its top level.
+      def dataset_of(parts)
+        index = parts.index("datasets")
+        index && parts[index + 1] ? parts[index + 1] : parts.first
+      end
+
+      def read_expected(path)
+        return [nil, "#{EXPECTED} is missing"] unless File.exist?(path)
+
+        [JSON.parse(File.read(path)), nil]
+      rescue JSON::ParserError => e
+        [nil, "#{EXPECTED} is not valid JSON: #{e.message.lines.first&.strip}"]
+      end
+
+      def metadata(dir, number, problem)
+        extra = File.exist?(File.join(dir, METADATA)) ? JSON.parse(File.read(File.join(dir, METADATA))).transform_keys(&:to_sym) : {}
+        { number: number&.to_i, difficulty: number.to_i, expected_error: problem }.compact.merge(extra)
+      end
+    end
+  end
+end

data/lib/agent/evals/experiment.rb

@@ -0,0 +1,133 @@
+# frozen_string_literal: true
+
+module Agent
+  module Evals
+    # One model at one setting over every case of one dataset, run on a
+    # thread pool the gem owns, plus the per-case detail the terminal report
+    # needs. Trackers only watch: {Trackers::Local} always, and at most one
+    # remote tracker, each told when the experiment starts, when a case
+    # finishes, and what the summary was.
+    class Experiment
+      # What one finished experiment reports back. `means` holds one entry
+      # per metric plus "score", the mean of each case's overall value;
+      # `usage` is summed over the cases; `permalink` is the remote tracker's
+      # page, when it has one.
+      Summary = Data.define(:model, :dataset, :name, :means, :results, :errors, :duration, :permalink, :usage) do
+        # @return [Float, nil] the mean overall score
+        def score = means["score"]
+
+        # @param metric [Metric, String]
+        # @return [Float, nil]
+        def mean(metric) = means[metric.respond_to?(:name) ? metric.name : metric.to_s]
+      end
+
+      attr_reader :model, :adapter, :scorer, :cases, :dataset, :parallelism, :trackers
+
+      # @param model [Model]
+      # @param cases [Array<Case>] all of one dataset
+      # @param adapter [Adapter]
+      # @param scorer [Scorer]
+      # @param parallelism [Integer] cases in flight at once
+      # @param trackers [Array<Tracker>] told about the run, in this order
+      def initialize(model:, cases:, adapter:, scorer:, parallelism: 4, trackers: [])
+        raise ArgumentError, "an experiment needs at least one case" if cases.empty?
+
+        @model = model
+        @cases = cases.sort_by { [it.variant, it.number || 0, it.id] }
+        @dataset = cases.first.dataset
+        @adapter = adapter
+        @scorer = scorer
+        @parallelism = [parallelism.to_i, 1].max
+        @trackers = trackers.select(&:enabled?)
+        @results = {}
+        @lock = Mutex.new
+      end
+
+      # @return [String] e.g. "anthropic/claude-haiku-5.5 effort=low t=0.0 [clear]"
+      def name = "#{model} [#{dataset}]"
+
+      # @param on_case [#call, nil] called once per finished case
+      # @return [Summary]
+      def run(on_case: nil)
+        started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        adapter.prepare(dataset)
+        trackers.each { it.start(self) }
+        errors = run_locally(on_case)
+        summary = summarize(errors, nil, Process.clock_gettime(Process::CLOCK_MONOTONIC) - started)
+        urls = trackers.map { it.finish(self, summary)&.fetch(:url, nil) }
+        summary.with(permalink: urls.compact.first)
+      end
+
+      # Runs one case: the adapter, then the scorer. A reply the model failed
+      # (`Reply#error`) scores 0 without troubling the scorer; an exception is
+      # the transport's and is left to the caller, which counts it as an error.
+      #
+      # @param kase [Case]
+      # @return [RunResult]
+      def evaluate(kase)
+        started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        reply = adapter.call(kase, model: model)
+        raise Error, "#{adapter.class}#call returned #{reply.class}, not an Agent::Evals::Reply" unless reply.is_a?(Reply)
+
+        score = reply.error? ? Score.failed(scorer.metrics, reply.error) : scorer.call(kase, reply, context: adapter.context(kase))
+        RunResult.new(experiment: self,
+          case: kase,
+          reply: reply,
+          score: score,
+          duration: Process.clock_gettime(Process::CLOCK_MONOTONIC) - started)
+      end
+
+      private
+
+      # Remembers a result and tells the trackers, under one lock, so a
+      # tracker sees results one at a time and in the order they are kept.
+      def remember(result)
+        @lock.synchronize do
+          @results[result.case.key] = result
+          trackers.each { it.record(result) }
+        end
+        result
+      end
+
+      # A fixed pool of threads over a queue: `parallelism` cases in flight.
+      def run_locally(on_case)
+        queue = Queue.new
+        cases.each { queue << it }
+        queue.close
+        errors = Queue.new
+        Array.new([parallelism, cases.size].min) do
+          Thread.new do
+            while (kase = queue.pop)
+              begin
+                remember evaluate(kase)
+              rescue StandardError => e
+                remember RunResult.new(experiment: self, case: kase, error: "#{e.class}: #{e.message}")
+                errors << "#{kase.key}: #{e.class}: #{e.message}"
+              end
+              on_case&.call
+            end
+          end
+        end.each(&:join)
+        Array.new(errors.size) { errors.pop }
+      end
+
+      def summarize(errors, permalink, duration)
+        results = @lock.synchronize { @results.values }.sort_by { cases.index(it.case) }
+        scored = results.filter_map(&:score)
+        means = scorer.metrics.to_h { |metric| [metric.name, metric.combine(scored.map { it.metrics[metric.name] })] }
+        means["score"] = scored.empty? ? nil : scored.sum(&:value).fdiv(scored.size)
+        Summary.new(
+          model:     model,
+          dataset:   dataset,
+          name:      name,
+          means:     means,
+          results:   results,
+          errors:    Array(errors),
+          duration:  duration,
+          permalink: permalink,
+          usage:     results.filter_map(&:reply).map(&:usage).reduce(Usage.zero, :+)
+        )
+      end
+    end
+  end
+end

data/lib/agent/evals/helper.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+module Agent
+  module Evals
+    # Finds and loads the suite: `evals/eval_helper.rb`, the file that calls
+    # {Agent::Evals.configure}, or, for a config-only suite, the directory
+    # holding `evals/.agent-eval.config.yml`. The executable looks before
+    # dispatching, the way rspec finds spec_helper, so the suite's own
+    # directory needs no wrapper script.
+    module Helper
+      FILE = File.join("evals", "eval_helper.rb")
+      ENV_VAR = "AGENT_EVALS_HELPER"
+      FLAG = "--helper"
+
+      # What {.locate} found: a helper to load, a config file to read, or neither.
+      Found = Data.define(:helper, :config) do
+        def initialize(helper: nil, config: nil) = super
+
+        # @return [Boolean] whether there is anything to load
+        def any? = !helper.nil? || !config.nil?
+
+        # @return [Boolean]
+        def none? = !any?
+      end
+
+      module_function
+
+      # Pulls `--helper PATH` (or `--helper=PATH`) out of argv, so dry-cli
+      # never sees a flag no command declares.
+      #
+      # @param argv [Array<String>]
+      # @return [Array(Array<String>, String, nil)] the remaining arguments and the path, if given
+      def extract(argv)
+        rest = []
+        path = nil
+        skip = false
+        argv.each_with_index do |arg, index|
+          next skip = false if skip
+
+          if arg == FLAG
+            path = argv[index + 1]
+            skip = true
+          elsif arg.start_with?("#{FLAG}=")
+            path = arg.delete_prefix("#{FLAG}=")
+          else
+            rest << arg
+          end
+        end
+        [rest, path]
+      end
+
+      # The suite to load: the flag, then the variable, then, from the
+      # directory upward, the nearest `evals/eval_helper.rb` or, failing
+      # that, the nearest `evals/.agent-eval.config.yml` (also found from
+      # inside `evals/` itself).
+      #
+      # @param explicit [String, nil] what `--helper` was given
+      # @param from [String] the directory to search up from
+      # @param env [Hash] the environment
+      # @return [Found] absolute paths; empty when there is none
+      def locate(explicit = nil, from: Dir.pwd, env: ENV)
+        candidate = explicit || env[ENV_VAR]
+        return Found.new(helper: File.expand_path(candidate, from)) if candidate && !candidate.empty?
+
+        Pathname(from).expand_path.ascend do |dir|
+          helper = dir.join(FILE)
+          config = [dir.join("evals", Configuration::DEFAULT_CONFIG), dir.join(Configuration::DEFAULT_CONFIG)].find(&:file?)
+          return Found.new(helper: helper.to_s, config: config&.to_s) if helper.file?
+          return Found.new(config: config.to_s) if config
+        end
+        Found.new
+      end
+
+      # Loads the helper, once, or roots the configuration at a config-only
+      # suite's directory so {Configuration#resolve_from_config!} can read it.
+      # A helper is loaded first and wins; the config fills what it left unset.
+      #
+      # @param found [Found, String, nil] nothing is loaded for nil or an empty Found
+      # @param configuration [Configuration]
+      # @return [Boolean] whether anything was loaded
+      # @raise [Error] when the helper does not exist
+      def load!(found, configuration: Agent::Evals.configuration)
+        found = Found.new(helper: found) if found.is_a?(String)
+        return false if found.nil? || found.none?
+
+        if found.helper
+          raise Error, "no eval helper at #{found.helper}" unless File.file?(found.helper)
+
+          require File.expand_path(found.helper)
+        else
+          root = File.dirname(found.config)
+          configuration.suite(File.basename(File.dirname(root)), root: root) unless configuration.complete?
+        end
+        true
+      end
+    end
+  end
+end