llm-evals 0.4.0 → 0.5.0

data/lib/llm/evals/results/warnings.rb

@@ -0,0 +1,149 @@
+# frozen_string_literal: true
+
+require "logger"
+
+module LLM
+  module Evals
+    module Results
+      # Everything a run would otherwise print as a warning while its progress
+      # bars redraw: cases that failed, and what Braintrust, RubyLLM, OpenTelemetry
+      # (whose exporters report failures on $stdout) and Ruby itself log. Printed among the bars, each one breaks the redraw; collected
+      # here, they are saved to one file, and the run prints only how many.
+      #
+      # It quacks as an IO, so a Logger, and $stderr, can write to it. Threads
+      # share it.
+      class Warnings
+        # Starts empty, with the lock every reader and writer below shares.
+        def initialize
+          @lines = []
+          @lock = Mutex.new
+        end
+
+        # Keeps one warning, dropping trailing newlines and anything blank.
+        #
+        # @param text [#to_s]
+        # @return [self]
+        def record(text)
+          text = text.to_s.chomp
+          @lock.synchronize { @lines << text } unless text.strip.empty?
+          self
+        end
+
+        # IO#write, for a Logger or $stderr writing here. Each argument is one warning.
+        #
+        # @param texts [Array<#to_s>]
+        # @return [Integer] the bytes "written", as IO#write does
+        def write(*texts)
+          texts.each { record(it) }
+          texts.sum { it.to_s.bytesize }
+        end
+
+        # IO#puts, for code holding this as $stderr.
+        #
+        # @param texts [Array<#to_s>] one warning each
+        # @return [nil]
+        def puts(*texts) = texts.each { record(it) } && nil
+        # IO#print, for code holding this as $stderr.
+        #
+        # @param texts [Array<#to_s>] one warning each
+        # @return [nil]
+        def print(*texts) = write(*texts) && nil
+        # IO#<<, chainable as on an IO.
+        #
+        # @param text [#to_s]
+        # @return [self]
+        def <<(text) = record(text)
+        # IO#flush; nothing is buffered, so there is nothing to do.
+        #
+        # @return [self]
+        def flush = self
+        # IO#sync; records are always kept at once.
+        #
+        # @return [Boolean] always true
+        def sync = true
+        # IO#sync=; ignored, since {#sync} cannot be anything but true.
+        #
+        # @param _value [Boolean]
+        # @return [void]
+        def sync=(_value); end
+        # IO#close; ignored, so a Logger closing its device cannot lose the warnings.
+        #
+        # @return [nil]
+        def close = nil
+        # IO#tty?; false tells a writer not to emit terminal control codes here.
+        #
+        # @return [Boolean] always false
+        def tty? = false
+
+        # Safe to call while other threads are still recording.
+        #
+        # @return [Integer]
+        def count = @lock.synchronize { @lines.size }
+
+        # A copy, so a caller cannot change what was recorded.
+        #
+        # @return [Array<String>]
+        def to_a = @lock.synchronize { @lines.dup }
+
+        # Lets a caller skip saving an empty file.
+        #
+        # @return [Boolean] whether nothing was recorded
+        def none? = count.zero?
+
+        # Writes the warnings to a file, separated by blank lines, replacing what was there.
+        #
+        # @param path [String]
+        # @return [String] the path
+        def save(path)
+          File.write(path, "#{to_a.join("\n\n")}\n")
+          path
+        end
+
+        # Runs the block with $stderr, Ruby's own warnings, and the loggers of
+        # whichever optional gems are loaded writing here, and puts them back after.
+        #
+        # @example
+        #   warnings.capture { warn "deprecated" }
+        #   warnings.to_a # => ["deprecated"]
+        # @yield runs with the output redirected; everything is restored afterwards, even if it raises
+        # @return [Object] what the block returns
+        def capture
+          logger = Logger.new(self, level: Logger::WARN)
+          stderr = $stderr
+          kernel_warn = Kernel.instance_method(:warn)
+          warnings = self
+          restore = redirect_loggers(logger)
+          $stderr = self
+          Kernel.define_method(:warn) { |*messages, **_options| warnings.write(messages.join("\n")) }
+          Kernel.send(:private, :warn)
+          yield
+        ensure
+          Kernel.define_method(:warn, kernel_warn) if kernel_warn
+          Kernel.send(:private, :warn)
+          restore&.call
+          $stderr = stderr
+        end
+
+        private
+
+        # Points the optional gems' loggers at the one given, for the gems that are loaded.
+        #
+        # @param logger [Logger] writes here
+        # @return [Proc] puts the loggers back
+        def redirect_loggers(logger)
+          braintrust = defined?(Braintrust::Log) ? Braintrust::Log.logger : nil
+          ruby_llm = defined?(RubyLLM) ? RubyLLM.instance_variable_get(:@logger) : nil
+          open_telemetry = defined?(OpenTelemetry) ? OpenTelemetry.logger : nil
+          Braintrust::Log.logger = logger if defined?(Braintrust::Log)
+          RubyLLM.instance_variable_set(:@logger, logger) if defined?(RubyLLM)
+          OpenTelemetry.logger = logger if defined?(OpenTelemetry)
+          lambda do
+            Braintrust::Log.logger = braintrust if defined?(Braintrust::Log)
+            RubyLLM.instance_variable_set(:@logger, ruby_llm) if defined?(RubyLLM)
+            OpenTelemetry.logger = open_telemetry if defined?(OpenTelemetry)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/llm/evals/results.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    # What comes back: a provider's response, an adapter's reply, a score, and what a run held back.
+    module Results
+    end
+  end
+end

data/lib/llm/evals/runtime/configuration.rb

@@ -0,0 +1,295 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    module Runtime
+      # Everything a suite tells the gem, through {LLM::Evals.configure}.
+      # Paths are relative to `root`, the directory the eval helper lives in.
+      class Configuration
+        # Where cases are looked for, relative to the root, when a suite does not say.
+        DEFAULT_GLOB = "datasets/*/cases/**/input.txt"
+        # The suite's name until it calls {#suite}.
+        DEFAULT_NAME = "llm-evals"
+        # The lineup file's name, also searched for by {Helper.locate}.
+        DEFAULT_CONFIG = ".llm-evals.config.yml"
+
+        attr_writer :adapter, :scorer
+        attr_reader :name,
+          :root,
+          :env_files,
+          :cases_glob,
+          :hooks,
+          :providers,
+          :trackers,
+          :description,
+          :config_path,
+          :started_in
+
+        # @!attribute [w] adapter
+        #   Registers the adapter, as {#adapter} does with an argument.
+        #   @param value [Actions::Adapter, nil]
+        #   @return [Actions::Adapter, nil]
+
+        # @!attribute [w] scorer
+        #   Registers the scorer, as {#scorer} does with an argument.
+        #   @param value [Scoring::Scorer, nil]
+        #   @return [Scoring::Scorer, nil]
+
+        # @!attribute [r] name
+        #   Titles the help and tags the experiments.
+        #   @return [String]
+
+        # @!attribute [r] root
+        #   The absolute directory other paths are relative to; the helper's directory.
+        #   @return [String]
+
+        # @!attribute [r] env_files
+        #   Absolute paths of the dotenv files to load, whether or not they exist.
+        #   @return [Array<String>]
+
+        # @!attribute [r] cases_glob
+        #   Finds each case's input file, relative to the root.
+        #   @return [String]
+
+        # @!attribute [r] hooks
+        #   The {#before_run} blocks, in the order given.
+        #   @return [Array<Proc>]
+
+        # @!attribute [r] providers
+        #   The defaults, plus any added through {#provider}.
+        #   @return [Providers::Registry]
+
+        # @!attribute [r] trackers
+        #   The defaults, plus any added through {#tracker}.
+        #   @return [Trackers::Registry]
+
+        # @!attribute [r] description
+        #   One line for the help's header.
+        #   @return [String]
+
+        # @!attribute [r] config_path
+        #   The lineup file as set, which {#config} resolves against the root.
+        #   @return [String]
+
+        # @!attribute [r] started_in
+        #   The directory the command was typed in, which the search may have left.
+        #   @return [String]
+
+        # Starts under the default name at the working directory, with no adapter or scorer.
+        def initialize
+          @name        = DEFAULT_NAME
+          @root        = Dir.pwd
+          @env_files   = []
+          @cases_glob  = DEFAULT_GLOB
+          @config_path = DEFAULT_CONFIG
+          @hooks       = []
+          @providers   = Providers::Registry.default
+          @trackers    = Trackers::Registry.default
+          @description = "Run a suite of cases against one or more models and score the replies."
+          @project     = nil
+          @started_in  = Dir.pwd
+          @search      = false
+        end
+
+        # Names the suite. The name titles the help and tags the experiments.
+        #
+        # @param name [String]
+        # @param root [String] where .llm-evals.config.yml and datasets/ live; the helper's directory
+        # @param env_files [Array<String>] dotenv files to load, relative to root; the first that exists wins nothing,
+        #   all that exist load, and an exported variable beats them all
+        # @param description [String, nil] one line for the help's header
+        # @return [void]
+        def suite(name, root:, env_files: [], description: nil)
+          @name        = name
+          @root        = File.expand_path(root)
+          @env_files   = env_files.map { File.expand_path(it, @root) }
+          @description = description if description
+        end
+
+        # @param path [String, nil] setting, relative to root; reading when nil
+        # @return [String] the absolute path of the lineup file, `.llm-evals.config.yml` at the root by default
+        def config(path = nil)
+          @config_path = path unless path.nil?
+          File.expand_path(@config_path, root)
+        end
+
+        # How many directories below the working directory the search goes.
+        MAX_SEARCH_DEPTH = 4
+
+        # Directories that hold other people's files or generated ones, never a suite.
+        SEARCH_SKIPS = %w[node_modules vendor tmp log pkg coverage].freeze
+
+        # The nearest `.llm-evals.config.yml` below a directory: the shallowest,
+        # then the first by name, so that the same tree always gives the same file.
+        # Hidden directories and {SEARCH_SKIPS} are not entered.
+        #
+        # @param from [String] the directory to search below
+        # @param max_depth [Integer] how many directories down to look
+        # @return [String, nil] the file's path, or nil when there is none
+        def self.search_config_file(from: Dir.pwd, max_depth: MAX_SEARCH_DEPTH)
+          level = [from]
+          max_depth.times do
+            level = level.flat_map { search_subdirectories(it) }
+            found = level.map { File.join(it, DEFAULT_CONFIG) }.find { File.file?(it) }
+            return found if found
+          end
+          nil
+        end
+
+        # @param dir [String]
+        # @return [Array<String>] the directories to enter, by name; none when unreadable
+        def self.search_subdirectories(dir)
+          Dir.children(dir).sort
+             .reject { it.start_with?(".") || SEARCH_SKIPS.include?(it) }
+             .map { File.join(dir, it) }
+             .select { File.directory?(it) && !File.symlink?(it) }
+        rescue SystemCallError
+          []
+        end
+        private_class_method :search_subdirectories
+
+        # Records where the command started, since the launcher may change directory to run it.
+        #
+        # @param dir [String] the directory the command was typed in
+        # @param search [Boolean] whether a search was allowed to run
+        # @return [void]
+        def located_from(dir, search:)
+          @started_in = dir
+          @search = search
+        end
+
+        # @return [Boolean] whether the lineup file exists
+        def config_found? = File.file?(config)
+
+        # @return [String] the lineup file's path, relative to where the command was typed
+        def relative_config = Pathname(config).relative_path_from(started_in).to_s
+
+        # @return [String] what to say when there is no lineup file
+        def config_missing_message
+          message = "Configuration file #{DEFAULT_CONFIG} was not found"
+          @search ? "#{message}, and the search #{MAX_SEARCH_DEPTH} directories down failed too" : message
+        end
+
+        # Where the cases are: a glob relative to root ending in the input file.
+        # Each match is a case; the expected answer is `expected.json` beside it.
+        #
+        # @param glob [String]
+        # @return [void]
+        def cases(glob)
+          @cases_glob = glob
+        end
+
+        # @param object [Actions::Adapter, nil] setting; reading when nil
+        # @return [Actions::Adapter, nil] nil until one is registered
+        def adapter(object = nil)
+          @adapter = object unless object.nil?
+          @adapter
+        end
+
+        # @param object [Scoring::Scorer, nil] setting; reading when nil
+        # @return [Scoring::Scorer, nil] nil until one is registered
+        def scorer(object = nil)
+          @scorer = object unless object.nil?
+          @scorer
+        end
+
+        # Runs once, after the env files are loaded and before any model is
+        # called. The place to reset a library that read ENV at require time.
+        #
+        # @param block [Proc] the hook
+        # @yield once, with no arguments
+        # @return [void]
+        def before_run(&block)
+          @hooks << block
+        end
+
+        # Adds a provider, ahead of the defaults, so a suite's own transport
+        # wins the ids it claims.
+        #
+        # @param provider [Providers::Base]
+        # @return [void]
+        def provider(provider)
+          @providers.register(provider)
+        end
+
+        # Adds a tracker class, so a run can choose it by name with `--tracker`
+        # or `tracker:`, or have its env key select it.
+        #
+        # @param klass [Class<Trackers::Base>]
+        # @return [void]
+        def tracker(klass)
+          @trackers.register(klass)
+        end
+
+        # @param name [String, nil] setting; reading when nil
+        # @return [String, nil] the Braintrust project, when .llm-evals.config.yml does not name one
+        def project(name = nil)
+          @project = name unless name.nil?
+          @project
+        end
+
+        # @return [Datasets::Discovery] the cases, as the glob finds them
+        def discovery = Datasets::Discovery.new(root: root, glob: cases_glob)
+
+        # @return [Config] the lineup file
+        def lineup = Config.new(config, providers:)
+
+        # @return [Environment]
+        def runtime = Environment.new(env_files: env_files, hooks: hooks, app_name: name)
+
+        # @return [Boolean] whether an adapter and a scorer are registered
+        def complete? = !adapter.nil? && !scorer.nil?
+
+        # Fails a command that needs the suite when no helper was loaded, with
+        # the fix in the message, instead of a NoMethodError on nil.
+        #
+        # @return [self]
+        # @raise [Error]
+        def require_suite!
+          resolve_from_config!
+          return self if complete?
+
+          raise Error,
+            "no suite is configured: register an adapter and a scorer in evals/eval_helper.rb, or set " \
+            "adapter: and scorer: (with require:) in #{config}; --helper PATH or LLM_EVALS_HELPER names another helper"
+        end
+
+        # Fills whatever the helper left unset from the config file's `require:`,
+        # `adapter:` and `scorer:` keys, so a suite can be config-only: its Ruby
+        # lives in its own gem and `evals/` holds the file and the datasets. The
+        # helper, when there is one, was loaded first and wins.
+        #
+        # @return [void]
+        # @raise [Error] naming the key and the value when a require fails or a constant is unknown
+        def resolve_from_config!
+          return if complete? || !File.exist?(config)
+
+          file = lineup
+          file.requires.each do |feature|
+            require feature
+          rescue LoadError => e
+            raise Error, "#{config}: require: #{feature} could not be loaded (#{e.message}); is its gem in the bundle?"
+          end
+          @adapter ||= instantiate("adapter", file.adapter_name)
+          @scorer  ||= instantiate("scorer", file.scorer_name) # rubocop:disable Naming/MemoizedInstanceVariableName
+        end
+
+        private
+
+        # @param key [String] the config key, for the error
+        # @param name [String, nil] the constant's name
+        # @return [Object, nil] a new instance of the named constant; nil when no name is set
+        # @raise [Error] when the constant is unknown or wants arguments
+        def instantiate(key, name)
+          return nil if name.nil? || name.empty?
+
+          Object.const_get(name).new
+        rescue NameError, ArgumentError => e
+          raise Error,
+            "#{config}: #{key}: #{name} could not be instantiated (#{e.class}: #{e.message}); " \
+            "name a constant the require: key loads, taking no arguments"
+        end
+      end
+    end
+  end
+end

data/lib/llm/evals/runtime/environment.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+require "dotenv"
+
+module LLM
+  module Evals
+    module Runtime
+      # One-time process setup: secrets from the suite's env files, a check that
+      # every key and gem the run needs is there, the suite's own hooks, and
+      # the providers' clients.
+      class Environment
+        attr_reader :env_files, :hooks, :app_name
+
+        # @!attribute [r] env_files
+        #   Absolute paths; those that exist load.
+        #   @return [Array<String>]
+
+        # @!attribute [r] hooks
+        #   The suite's `before_run` blocks.
+        #   @return [Array<Proc>]
+
+        # @!attribute [r] app_name
+        #   What a provider tells its API the caller is.
+        #   @return [String]
+
+        # @param env_files [Array<String>] absolute paths; those that exist load
+        # @param hooks [Array<Proc>] the suite's `before_run` blocks
+        # @param app_name [String] what a provider tells its API the caller is
+        def initialize(env_files: [], hooks: [], app_name: "llm-evals")
+          @env_files = env_files
+          @hooks = hooks
+          @app_name = app_name
+        end
+
+        # Loads the env files that exist, without overriding what is already
+        # set, so an exported variable beats the file.
+        #
+        # @return [void]
+        def load_env
+          files = env_files.select { File.exist?(it) }
+          Dotenv.load(*files) unless files.empty?
+        end
+
+        # Fails before anything is called when a key the run needs is neither
+        # exported nor in an env file.
+        #
+        # @param names [Array<String>] variable names; repeats are fine
+        # @return [void]
+        # @raise [Error] naming every missing variable
+        def require_env!(names)
+          missing = names.uniq.select { ENV[it].to_s.strip.empty? }
+          return if missing.empty?
+
+          found = env_files.select { File.exist?(it) }
+          where = if found.empty?
+                    env_files.empty? ? "no env files are configured" : "there is no #{env_files.join(" or ")}"
+                  else
+                    "checked the environment, then #{found.join(", ")}"
+                  end
+          raise Error,
+            "missing #{missing.join(", ")}: export them or add them to an env file (#{where}); " \
+            "an exported blank value wins over the file"
+        end
+
+        # Readies the process for a run: env files, checks, hooks, then provider clients, in that order.
+        #
+        # @param providers [Array<Providers::Base>] the providers the run's models use
+        # @param tracker [Class<Trackers::Base>, nil] the remote tracker chosen, whose gems and key are checked too
+        # @return [void]
+        # @raise [Error] when a needed variable or gem is missing
+        def setup!(providers: [], tracker: nil)
+          load_env
+          require_env!(providers.flat_map(&:env_keys))
+          tracker&.requirements!(ENV)
+          hooks.each(&:call) # after load_env: a hook usually re-reads ENV
+          providers.each { it.setup!(app_name: app_name) }
+        end
+
+        # Opens a URL in the desktop browser.
+        #
+        # @param url [String] the page to open
+        # @return [Boolean] whether the opener ran
+        def browse(url)
+          opener = RUBY_PLATFORM.include?("darwin") ? "open" : "xdg-open"
+          system(opener, url, out: File::NULL, err: File::NULL) || false
+        end
+      end
+    end
+  end
+end

data/lib/llm/evals/runtime/helper.rb

@@ -0,0 +1,153 @@
+# frozen_string_literal: true
+
+module LLM
+  module Evals
+    module Runtime
+      # Finds and loads the suite: `evals/eval_helper.rb`, the file that calls
+      # {LLM::Evals.configure}, or, for a config-only suite, the directory
+      # holding `evals/.llm-evals.config.yml`. The executable looks before
+      # dispatching, the way rspec finds spec_helper, so the suite's own
+      # directory needs no wrapper script.
+      module Helper
+        # Where the helper lives, relative to a directory searched by {.locate}.
+        FILE = File.join("evals", "eval_helper.rb")
+        # The variable that names a helper, when the flag does not.
+        ENV_VAR = "LLM_EVALS_HELPER"
+        # The flag that names a helper, read here because no command declares it.
+        FLAG = "--helper"
+        # Turns off the search below the working directory.
+        NO_SEARCH_FLAG = "--no-search"
+        # The flags naming a lineup file, which makes a search pointless.
+        CONFIG_FLAGS = %w[-C --config].freeze
+        # The directory a suite's files live in, under the project.
+        SUITE_DIR = "evals"
+
+        # What {.locate} found: a helper to load, a config file to read, or neither.
+        #
+        # @!attribute [r] helper
+        #   @return [String, nil] the path of the helper to load
+        # @!attribute [r] config
+        #   @return [String, nil] the path of the config file
+        # @!attribute [r] home
+        #   @return [String, nil] the directory to run in, when the search found the suite below this one
+        Found = Data.define(:helper, :config, :home) do
+          # Either may be left out, and {.locate} returns neither when it finds nothing.
+          #
+          # @param helper [String, nil] the path of the helper to load
+          # @param config [String, nil] the path of the config file
+          # @param home [String, nil] the directory to run in
+          def initialize(helper: nil, config: nil, home: nil) = super
+
+          # @return [Boolean] whether there is anything to load
+          def any? = !helper.nil? || !config.nil?
+
+          # @return [Boolean] whether there is nothing to load
+          def none? = !any?
+        end
+
+        module_function
+
+        # Pulls `--helper PATH` (or `--helper=PATH`) out of argv, so dry-cli
+        # never sees a flag no command declares.
+        #
+        # @param argv [Array<String>] the command line; left unchanged
+        # @return [Array(Array<String>, String, nil)] the remaining arguments and the path, if given
+        def extract(argv)
+          rest = []
+          path = nil
+          skip = false
+          argv.each_with_index do |arg, index|
+            next skip = false if skip
+
+            if arg == FLAG
+              path = argv[index + 1]
+              skip = true
+            elsif arg.start_with?("#{FLAG}=")
+              path = arg.delete_prefix("#{FLAG}=")
+            else
+              rest << arg
+            end
+          end
+          [rest, path]
+        end
+
+        # Pulls `--no-search` out of argv.
+        #
+        # @param argv [Array<String>] the command line; left unchanged
+        # @return [Array(Array<String>, Boolean)] the remaining arguments and whether a search may run
+        def extract_search(argv) = [argv - [NO_SEARCH_FLAG], !argv.include?(NO_SEARCH_FLAG)]
+
+        # @param argv [Array<String>] the command line
+        # @return [Boolean] whether it names a lineup file itself
+        def config_flag?(argv) = argv.any? { CONFIG_FLAGS.include?(it) || it.start_with?("--config=") }
+
+        # The directory a suite is run from: above `evals/`, or the config's own.
+        #
+        # @param config [String] the path of a config file
+        # @return [String]
+        def project_dir(config)
+          dir = File.dirname(config)
+          File.basename(dir) == SUITE_DIR ? File.dirname(dir) : dir
+        end
+
+        # The suite to load: the flag, then the variable, then, from the
+        # directory upward, the nearest `evals/eval_helper.rb` or, failing
+        # that, the nearest `evals/.llm-evals.config.yml` (also found from
+        # inside `evals/` itself), then, when allowed, a search below the directory.
+        #
+        # @param explicit [String, nil] what `--helper` was given
+        # @param from [String] the directory to search up from
+        # @param env [Hash] the environment
+        # @param search [Boolean] whether to search below the directory when the rest finds nothing
+        # @return [Found] absolute paths; empty when there is none
+        def locate(explicit = nil, from: Dir.pwd, env: ENV, search: false)
+          candidate = explicit || env[ENV_VAR]
+          return Found.new(helper: File.expand_path(candidate, from)) if candidate && !candidate.empty?
+
+          Pathname(from).expand_path.ascend do |dir|
+            helper = dir.join(FILE)
+            config = [dir.join("evals", Configuration::DEFAULT_CONFIG), dir.join(Configuration::DEFAULT_CONFIG)].find(&:file?)
+            return Found.new(helper: helper.to_s, config: config&.to_s) if helper.file?
+            return Found.new(config: config.to_s) if config
+          end
+          search ? discover(from) : Found.new
+        end
+
+        # What the search below a directory found, with the helper beside the file, if any.
+        #
+        # @param from [String] the directory to search below
+        # @return [Found] with `home` set; empty when there is none
+        def discover(from)
+          config = Configuration.search_config_file(from: File.expand_path(from))
+          return Found.new unless config
+
+          helper = File.join(File.dirname(config), File.basename(FILE))
+          Found.new(helper: File.file?(helper) ? helper : nil, config: config, home: project_dir(config))
+        end
+
+        # Loads the helper, once, or roots the configuration at a config-only
+        # suite's directory so {Configuration#resolve_from_config!} can read it.
+        # A helper is loaded first and wins; the config fills what it left unset.
+        #
+        # @param found [Found, String, nil] nothing is loaded for nil or an empty Found
+        # @param configuration [Configuration]
+        # @return [Boolean] whether anything was loaded
+        # @raise [Error] when the helper does not exist
+        def load!(found, configuration: LLM::Evals.configuration)
+          found = Found.new(helper: found) if found.is_a?(String)
+          return false if found.nil? || found.none?
+
+          if found.helper
+            raise Error, "no eval helper at #{found.helper}" unless File.file?(found.helper)
+
+            require File.expand_path(found.helper)
+          else
+            root = File.dirname(found.config)
+            configuration.suite(File.basename(project_dir(found.config)), root: root) unless configuration.complete?
+          end
+          true
+        end
+      end
+    end
+  end
+end