activeagent 1.9.6 → 1.9.7

data/lib/active_agent/providers/cheaper_inference/options.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require_relative "../open_ai/options"
+
+module ActiveAgent
+  module Providers
+    module CheaperInference
+      # Configuration options for the Cheaper Inference provider.
+      #
+      # Extends OpenAI::Options, overriding the base URL to point at Cheaper
+      # Inference's OpenAI-compatible gateway and resolving the API key from
+      # CHEAPER_INFERENCE_API_KEY. Cheaper Inference does not use organization
+      # or project identifiers.
+      #
+      # @example Basic configuration
+      #   options = Options.new(api_key: ENV["CHEAPER_INFERENCE_API_KEY"])
+      #
+      # @see https://cheaperinference.com/docs
+      # @see https://cheaperinference.com/signup Cheaper Inference API Keys
+      class Options < ActiveAgent::Providers::OpenAI::Options
+        # @!attribute base_url
+        #   @return [String] API endpoint (default: "https://api.cheaperinference.com/v1")
+        attribute :base_url, :string, as: "https://api.cheaperinference.com/v1"
+
+        private
+
+        def resolve_api_key(kwargs)
+          kwargs[:api_key] ||
+            kwargs[:access_token] ||
+            ENV["CHEAPER_INFERENCE_API_KEY"]
+        end
+
+        # Not used as part of Cheaper Inference
+        def resolve_organization_id(kwargs) = nil
+        def resolve_project_id(kwargs)      = nil
+      end
+    end
+  end
+end

data/lib/active_agent/providers/cheaper_inference_provider.rb

@@ -0,0 +1,72 @@
+require_relative "_base_provider"
+
+require_gem!(:openai, __FILE__)
+
+require_relative "open_ai_provider"
+require_relative "cheaper_inference/_types"
+
+module ActiveAgent
+  module Providers
+    # Provides access to Cheaper Inference's OpenAI-compatible LLM gateway.
+    #
+    # Extends the OpenAI provider to work with Cheaper Inference's
+    # OpenAI-compatible API, enabling access to models from many providers
+    # through a single interface. Model ids are bare names (e.g.
+    # +gpt-5.4-mini+).
+    #
+    # Cheaper Inference is a plain OpenAI-compatible gateway: requests,
+    # responses and transforms are identical to the OpenAI Chat API, so this
+    # provider reuses OpenAI::Chat::RequestType and OpenAI::Chat::Transforms
+    # directly. The only Cheaper Inference-specific configuration is the base
+    # URL and API key, which live in CheaperInference::Options.
+    #
+    # @example Configuration in active_agent.yml
+    #   cheaper_inference:
+    #     service: "CheaperInference"
+    #     api_key: <%= ENV["CHEAPER_INFERENCE_API_KEY"] %>
+    #     model: "gpt-5.4-mini"
+    #
+    # @see OpenAI::ChatProvider
+    # @see https://cheaperinference.com/docs
+    class CheaperInferenceProvider < OpenAI::ChatProvider
+      # @return [String]
+      def self.service_name
+        "CheaperInference"
+      end
+
+      # @return [Class]
+      def self.options_klass
+        CheaperInference::Options
+      end
+
+      # @return [ActiveModel::Type::Value]
+      def self.prompt_request_type
+        OpenAI::Chat::RequestType.new
+      end
+
+      # @return [ActiveModel::Type::Value]
+      def self.embed_request_type
+        OpenAI::Embedding::RequestType.new
+      end
+
+      protected
+
+      # Executes an embedding request via Cheaper Inference's OpenAI-compatible API.
+      #
+      # @param parameters [Hash]
+      # @return [Hash] symbolized API response
+      def api_embed_execute(parameters)
+        client.embeddings.create(**parameters).as_json.deep_symbolize_keys
+      end
+
+      # @see BaseProvider#api_response_normalize
+      # @param api_response [OpenAI::Models::ChatCompletion]
+      # @return [Hash] normalized response hash
+      def api_response_normalize(api_response)
+        return api_response unless api_response
+
+        OpenAI::Chat::Transforms.gem_to_hash(api_response)
+      end
+    end
+  end
+end

data/lib/active_agent/providers/ollama/_types.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
 require_relative "options"
+require_relative "catalog"
 require_relative "chat/_types"
 require_relative "embedding/_types"

data/lib/active_agent/providers/ollama/catalog.rb

@@ -0,0 +1,221 @@
+# frozen_string_literal: true
+
+require "json"
+require "net/http"
+require "openssl"
+require "uri"
+require_relative "cloud"
+
+module ActiveAgent
+  module Providers
+    module Ollama
+      # The models an Ollama host serves, from the host's own API: GET
+      # /api/tags lists them and POST /api/show describes one. Works against
+      # a local server and against Ollama Cloud alike. The host is the
+      # provider's base URL (".../v1"); the native API lives beside it.
+      #
+      #   catalog = ActiveAgent::Providers::Ollama::Catalog.new(host: "https://ollama.com", api_key: ENV["OLLAMA_API_KEY"])
+      #   catalog.models.map(&:name)         # => ["gpt-oss:120b", "kimi-k3", ...]
+      #   catalog.model_info("gpt-oss:120b") # => #<struct Model name="gpt-oss:120b", cloud=true, capabilities=["completion", "tools", "thinking"], ...>
+      #
+      # A local server lists the cloud models it relays (`gpt-oss:120b-cloud`)
+      # beside the ones it runs itself, and each Model says which it is. A
+      # server with no native API — an OpenAI-compatible proxy in front of
+      # one — is listed through GET /v1/models instead, which names models
+      # and nothing else.
+      #
+      # Prices are not here: the host's API does not publish them. The
+      # dashboard's ActionAgent::ModelPricing prices a cloud model by its
+      # canonical name.
+      class Catalog
+        class Error < StandardError; end
+        # The host could not be reached, or did not answer in time.
+        class ConnectionError < Error; end
+        # The host refused the API key (401 or 403).
+        class AuthenticationError < Error; end
+        # The host does not serve the model asked about.
+        class NotFoundError < Error; end
+
+        DEFAULT_TIMEOUT = 10
+
+        # One model, as the host lists (#models) or describes (#model_info)
+        # it. A listing carries details and size; a description adds the
+        # capabilities and the context length.
+        Model = Struct.new(:name, :cloud, :details, :capabilities, :context_length, :size, :modified_at, :raw,
+                           keyword_init: true) do
+          # The name the cloud's catalog and price list use (Cloud.canonical_model).
+          def canonical_name
+            Cloud.canonical_model(name)
+          end
+
+          # Whether Ollama Cloud serves the model: listed by the cloud, or
+          # relayed to it by a local server.
+          def cloud?
+            cloud == true
+          end
+
+          def family
+            details["family"]
+          end
+
+          def parameter_size
+            details["parameter_size"]
+          end
+
+          def quantization_level
+            details["quantization_level"]
+          end
+
+          def to_h
+            {
+              name: name, canonical_name: canonical_name, cloud: cloud?, details: details,
+              capabilities: capabilities, context_length: context_length, size: size, modified_at: modified_at
+            }
+          end
+        end
+
+        attr_reader :host, :timeout
+
+        # @param host [String] the server address, bare or with its /v1 path
+        # @param api_key [String, nil] sent as a Bearer token; the local
+        #   placeholder is not sent
+        # @param timeout [Numeric] seconds to connect and to read
+        def initialize(host:, api_key: nil, timeout: DEFAULT_TIMEOUT)
+          @host = Cloud.normalize_host(host) || Cloud::LOCAL_BASE_URL
+          key = api_key.to_s.strip
+          @api_key = key unless key.empty? || key == Cloud::PLACEHOLDER_API_KEY
+          @timeout = timeout
+        end
+
+        # @return [Boolean] whether the host is Ollama Cloud
+        def cloud?
+          Cloud.cloud_host?(host)
+        end
+
+        # The models the host serves.
+        #
+        # @return [Array<Model>]
+        # @raise [Error]
+        def models
+          response = request(Net::HTTP::Get, "/api/tags", allow_not_found: true)
+          return openai_models if response.code.to_i == 404
+
+          Array(parse(response, "/api/tags")["models"]).filter_map { |entry| from_listing(entry) }
+        end
+
+        # @return [Array<String>] the names of the models the host serves
+        def model_names
+          models.map(&:name)
+        end
+
+        # One model, described: its details, capabilities and context length.
+        #
+        # @param name [String]
+        # @return [Model]
+        # @raise [NotFoundError] when the host does not serve +name+
+        # @raise [Error]
+        def model_info(name)
+          response = request(Net::HTTP::Post, "/api/show", { model: name.to_s }, allow_not_found: true)
+          raise NotFoundError, "#{host} does not serve #{name}" if response.code.to_i == 404
+
+          from_show(name.to_s, parse(response, "/api/show"))
+        end
+
+        private
+
+        def openai_models
+          data = parse(request(Net::HTTP::Get, "/v1/models"), "/v1/models")
+          Array(data["data"]).filter_map do |entry|
+            id = entry.is_a?(Hash) ? entry["id"].to_s : ""
+            next if id.empty?
+
+            Model.new(name: id, cloud: cloud? || Cloud.cloud_model?(id), details: {}, capabilities: [], raw: entry)
+          end
+        end
+
+        def from_listing(entry)
+          return nil unless entry.is_a?(Hash)
+
+          name = (entry["name"] || entry["model"]).to_s
+          return nil if name.empty?
+
+          Model.new(
+            name: name,
+            cloud: cloud_model?(name, entry),
+            details: hash_at(entry, "details"),
+            capabilities: Array(entry["capabilities"]).map(&:to_s),
+            context_length: nil,
+            size: entry["size"],
+            modified_at: entry["modified_at"],
+            raw: entry
+          )
+        end
+
+        def from_show(name, data)
+          info = hash_at(data, "model_info")
+          architecture = info["general.architecture"]
+          context_length = info["#{architecture}.context_length"] ||
+            info.find { |key, _| key.to_s.end_with?(".context_length") }&.last
+
+          Model.new(
+            name: name,
+            cloud: cloud_model?(name, data),
+            details: hash_at(data, "details"),
+            capabilities: Array(data["capabilities"]).map(&:to_s),
+            context_length: context_length.is_a?(Numeric) ? context_length.to_i : nil,
+            size: nil,
+            modified_at: data["modified_at"],
+            raw: data
+          )
+        end
+
+        # Listed by the cloud itself, relayed to it by a local server (which
+        # names the remote host), or spelled with the cloud marker.
+        def cloud_model?(name, entry)
+          cloud? || !entry["remote_host"].to_s.empty? || Cloud.cloud_model?(name)
+        end
+
+        def hash_at(data, key)
+          value = data[key]
+          value.is_a?(Hash) ? value : {}
+        end
+
+        def request(method_class, path, body = nil, allow_not_found: false)
+          uri = URI.parse("#{Cloud.api_origin(host)}#{path}")
+          request = method_class.new(uri.request_uri, headers)
+          if body
+            request["Content-Type"] = "application/json"
+            request.body = JSON.generate(body)
+          end
+
+          response = Net::HTTP.start(
+            uri.host, uri.port,
+            use_ssl: uri.scheme == "https", open_timeout: timeout, read_timeout: timeout
+          ) { |http| http.request(request) }
+
+          status = response.code.to_i
+          raise AuthenticationError, "#{uri} refused the API key (#{status})" if [ 401, 403 ].include?(status)
+          return response if status == 200 || (status == 404 && allow_not_found)
+
+          raise Error, "#{uri} responded #{status}"
+        rescue Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Errno::ENETUNREACH, Errno::ECONNRESET, SocketError,
+               Net::OpenTimeout, Net::ReadTimeout, IOError, OpenSSL::SSL::SSLError => e
+          raise ConnectionError, "Could not reach #{host} (#{e.class.name.split('::').last}: #{e.message})"
+        end
+
+        def parse(response, path)
+          data = JSON.parse(response.body.to_s)
+          data.is_a?(Hash) ? data : {}
+        rescue JSON::ParserError
+          raise Error, "#{Cloud.api_origin(host)}#{path} did not return JSON"
+        end
+
+        def headers
+          headers = { "Accept" => "application/json" }
+          headers["Authorization"] = "Bearer #{@api_key}" if @api_key
+          headers
+        end
+      end
+    end
+  end
+end

data/lib/active_agent/providers/ollama/cloud.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+require "uri"
+
+module ActiveAgent
+  module Providers
+    module Ollama
+      # Ollama Cloud: the models Ollama serves from ollama.com, and the two
+      # routes a request takes to them.
+      #
+      # Directly, the host is https://ollama.com, every request carries an
+      # API key (https://ollama.com/settings/keys), and a model is named as
+      # the cloud lists it: `gpt-oss:120b`. Through a local server signed in
+      # to ollama.com (`ollama signin`), the host is the local one, no key is
+      # needed, and the same model carries the local server's cloud marker:
+      # `gpt-oss:120b-cloud`, or `glm-4.6:cloud`. These helpers tell the two
+      # routes apart and reduce either spelling to the name the cloud's
+      # catalog and price list use, so a trace or an evaluation prices the
+      # model the same way whichever route served it. The dashboard's
+      # ActionAgent::ModelPricing keeps a copy of the two naming rules, so a
+      # change here needs the same change there.
+      module Cloud
+        HOST = "https://ollama.com"
+        # The OpenAI-compatible endpoint the provider talks to.
+        BASE_URL = "#{HOST}/v1"
+        LOCAL_BASE_URL = "http://127.0.0.1:11434/v1"
+        # The port the ollama command assumes when a host names none.
+        DEFAULT_PORT = 11_434
+        # The key a local server is sent when none is configured; it ignores
+        # it, and the cloud refuses it.
+        PLACEHOLDER_API_KEY = "ollama"
+        # A local server's spellings of a cloud model.
+        MODEL_SUFFIX = "-cloud"
+        MODEL_TAG = ":cloud"
+
+        module_function
+
+        # Whether +url+ addresses Ollama Cloud.
+        #
+        # @param url [String, nil]
+        # @return [Boolean]
+        def cloud_host?(url)
+          host = URI.parse(url.to_s.strip).host.to_s.downcase
+          host == "ollama.com" || host.end_with?(".ollama.com")
+        rescue URI::InvalidURIError
+          false
+        end
+
+        # Whether +name+ carries a local server's cloud marker.
+        #
+        # @param name [String, nil]
+        # @return [Boolean]
+        def cloud_model?(name)
+          name = name.to_s.strip
+          name.end_with?(MODEL_SUFFIX) || name.end_with?(MODEL_TAG)
+        end
+
+        # The name the cloud lists a model under: "gpt-oss:120b-cloud" →
+        # "gpt-oss:120b", "glm-4.6:cloud" → "glm-4.6"; any other name as given.
+        #
+        # @param name [String, nil]
+        # @return [String]
+        def canonical_model(name)
+          name = name.to_s.strip
+          return name.delete_suffix(MODEL_TAG) if name.end_with?(MODEL_TAG)
+          return name.delete_suffix(MODEL_SUFFIX) if name.end_with?(MODEL_SUFFIX)
+
+          name
+        end
+
+        # A server address as the base URL of its OpenAI-compatible API, read
+        # the way the ollama command reads OLLAMA_HOST: a bare origin
+        # (http://localhost:11434, https://ollama.com) gets the /v1 path, a
+        # scheme-less "host[:port]" is http, an http address with no port is
+        # on 11434, trailing slashes go, and an explicit path is kept for a
+        # server behind a reverse proxy. nil for a blank value.
+        #
+        # @param value [String, nil]
+        # @return [String, nil]
+        def normalize_host(value)
+          host = value.to_s.strip.chomp("/")
+          return nil if host.empty?
+
+          unless host.match?(%r{\A[a-z][a-z0-9+.-]*://}i)
+            host = "#{cloud_host?("https://#{host}") ? 'https' : 'http'}://#{host}"
+          end
+          uri = URI.parse(host)
+          return host unless uri.is_a?(URI::HTTP)
+
+          uri.port = DEFAULT_PORT if uri.scheme == "http" && !host.match?(%r{\A[a-z][a-z0-9+.-]*://[^/]*:\d+}i)
+          uri.path = "/v1" if uri.path.empty? || uri.path == "/"
+          uri.to_s.chomp("/")
+        rescue URI::InvalidURIError
+          host
+        end
+
+        # Where the host's own API lives, beside its OpenAI-compatible one:
+        # "https://ollama.com/v1" → "https://ollama.com", and a reverse-proxied
+        # "https://ai.example.com/ollama/v1" → "https://ai.example.com/ollama".
+        #
+        # @param base_url [String]
+        # @return [String]
+        def api_origin(base_url)
+          base_url.to_s.strip.chomp("/").delete_suffix("/v1")
+        end
+      end
+    end
+  end
+end

data/lib/active_agent/providers/ollama/options.rb

@@ -1,16 +1,80 @@
 # frozen_string_literal: true
 
 require_relative "../open_ai/options"
+require_relative "../errors"
+require_relative "cloud"
 
 module ActiveAgent
   module Providers
     module Ollama
+      # Where the provider sends requests and how it signs them.
+      #
+      # The host is, in order: `host:` (also `base_url:` or `uri_base:`),
+      # `cloud: true` for Ollama Cloud, the OLLAMA_HOST variable the ollama
+      # command itself reads, and the local server at 127.0.0.1:11434. A bare
+      # origin is completed to its OpenAI-compatible /v1 endpoint
+      # ({Cloud.normalize_host}). The key is `api_key:` (also
+      # `access_token:`), else OLLAMA_API_KEY or OLLAMA_ACCESS_TOKEN, else a
+      # placeholder a local server ignores. Ollama Cloud refuses the
+      # placeholder, which {#require_cloud_api_key!} reports before a request
+      # is made.
+      #
+      # @example Ollama Cloud
+      #   generate_with :ollama, cloud: true, model: "gpt-oss:120b"   # key from OLLAMA_API_KEY
+      #
+      # @example A local server signed in to the cloud
+      #   generate_with :ollama, model: "gpt-oss:120b-cloud"          # no key needed
       class Options < ActiveAgent::Providers::OpenAI::Options
-        attribute :base_url, :string, fallback: "http://127.0.0.1:11434/v1"
-        attribute :api_key,  :string, fallback: "ollama"
+        attribute :base_url, :string, fallback: Cloud::LOCAL_BASE_URL
+        attribute :api_key,  :string, fallback: Cloud::PLACEHOLDER_API_KEY
+        # Ollama Cloud in one word: requests go to https://ollama.com/v1
+        # unless a host is given, and a key is expected.
+        attribute :cloud, :boolean
+
+        def initialize(kwargs = {})
+          kwargs = kwargs.deep_symbolize_keys if kwargs.respond_to?(:deep_symbolize_keys)
+
+          super(kwargs.except(:host, :uri_base).merge(base_url: resolve_base_url(kwargs)))
+        end
+
+        # @return [Boolean] whether requests go to Ollama Cloud
+        def cloud?
+          cloud == true || Cloud.cloud_host?(base_url)
+        end
+
+        # The OpenAI client's options: `cloud` is this provider's alone.
+        #
+        # @return [Hash]
+        def serialize
+          super.except(:cloud)
+        end
+
+        # Ollama Cloud signs every request with an API key. Raised ahead of
+        # the request the cloud would refuse with a 401, naming what to set.
+        #
+        # @return [void]
+        # @raise [Errors::AuthenticationFailed] on the cloud host without a key
+        def require_cloud_api_key!
+          return unless cloud?
+          return if api_key.present? && api_key != Cloud::PLACEHOLDER_API_KEY
+
+          raise Errors::AuthenticationFailed.new(
+            "Ollama Cloud (#{base_url}) needs an API key: set OLLAMA_API_KEY, or pass api_key: " \
+            "(keys are created at https://ollama.com/settings/keys)",
+            provider_tag: "Ollama"
+          )
+        end
 
         private
 
+        def resolve_base_url(kwargs)
+          explicit = kwargs[:base_url].presence || kwargs[:host].presence || kwargs[:uri_base].presence
+          return Cloud.normalize_host(explicit) if explicit
+          return Cloud::BASE_URL if ActiveModel::Type::Boolean.new.cast(kwargs[:cloud])
+
+          Cloud.normalize_host(ENV["OLLAMA_HOST"])
+        end
+
         def resolve_api_key(kwargs)
           kwargs[:api_key] ||
             kwargs[:access_token] ||

data/lib/active_agent/providers/ollama_provider.rb

@@ -7,12 +7,24 @@ require_relative "ollama/_types"
 
 module ActiveAgent
   module Providers
-    # Connects to local Ollama instances via OpenAI-compatible API.
+    # Connects to Ollama through its OpenAI-compatible API: a local server,
+    # a remote one, or Ollama Cloud (https://ollama.com).
     #
-    # Provides chat completion and embedding functionality through locally-hosted
-    # Ollama models. Handles connection errors specific to local deployments.
+    # Provides chat completion and embedding functionality, and lists and
+    # describes the models the host serves ({#models}, {#model_info}).
+    # Handles connection errors specific to local deployments, and asks for
+    # an API key before a request to the cloud that would be refused
+    # without one.
+    #
+    # @example A local server
+    #   generate_with :ollama, model: "qwen3:8b"
+    #
+    # @example Ollama Cloud, the key from OLLAMA_API_KEY
+    #   generate_with :ollama, cloud: true, model: "gpt-oss:120b"
     #
     # @see OpenAI::ChatProvider
+    # @see Ollama::Options
+    # @see Ollama::Cloud
     class OllamaProvider < OpenAI::ChatProvider
       # @return [String]
       def self.service_name
@@ -34,6 +46,46 @@ module ActiveAgent
         namespace::Embedding::RequestType.new
       end
 
+      # @return [Boolean] whether requests go to Ollama Cloud
+      def cloud?
+        options.cloud?
+      end
+
+      # The models the configured host serves, from the host's own API.
+      #
+      # @return [Ollama::Catalog]
+      def catalog
+        namespace::Catalog.new(host: options.base_url, api_key: options.api_key)
+      end
+
+      # Lists the models the configured host serves.
+      #
+      # @return [Array<Ollama::Catalog::Model>]
+      # @raise [Ollama::Catalog::Error] when the host cannot be read
+      def models
+        catalog.models
+      end
+
+      # Describes one model the configured host serves: its details,
+      # capabilities and context length.
+      #
+      # @param name [String]
+      # @return [Ollama::Catalog::Model]
+      # @raise [Ollama::Catalog::NotFoundError] when the host does not serve it
+      # @raise [Ollama::Catalog::Error] when the host cannot be read
+      def model_info(name)
+        catalog.model_info(name)
+      end
+
+      # The OpenAI client, once the cloud's key requirement is met.
+      #
+      # @see OpenAI::Base#client
+      # @raise [Errors::AuthenticationFailed] on the cloud host without a key
+      def client
+        options.require_cloud_api_key!
+        super
+      end
+
       protected
 
       # Executes chat completion request with Ollama-specific error handling.

data/lib/active_agent/railtie.rb

@@ -15,6 +15,10 @@ module ActiveAgent
     config.active_agent.telemetry = ActiveSupport::OrderedOptions.new
     config.eager_load_namespaces << ActiveAgent
 
+    rake_tasks do
+      load File.expand_path("../tasks/active_agent/evals.rake", __dir__)
+    end
+
     initializer "active_agent.deprecator", before: :load_environment_config do |app|
       app.deprecators[:active_agent] = ActiveAgent.deprecator
     end

data/lib/active_agent/version.rb

@@ -1,3 +1,3 @@
 module ActiveAgent
-  VERSION = "1.9.6"
+  VERSION = "1.9.7"
 end

data/lib/tasks/active_agent/evals.rake

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+# Evals are ordinary tests under test/evals that each call a model, so
+# `rails test` skips them (ActiveAgent::Evals::TestCase checks EVALS) and this
+# task runs them: it switches EVALS on and hands test/evals to the same runner
+# `rails test` uses, so seeds, parallel workers and reporters behave the same.
+#
+#   bin/rails test:evals
+#   bin/rails test:evals TEST=test/evals/support_agent_test.rb
+#   bin/rails test:evals TESTOPTS="--seed 1234 -v"
+#   EVALS=1 bin/rails test test/evals/support_agent_test.rb:12   # one eval, through rails test
+#   EVALS=1 bin/rails test                                       # the whole suite, evals included
+namespace :test do
+  desc "Run the agent evals in test/evals (switches EVALS on); TEST=path narrows, TESTOPTS passes options"
+  task :evals do
+    require "active_agent/evals/test_helper"
+    require "rails/test_unit/runner"
+
+    paths = Array(ENV["TEST"]).map(&:to_s).reject(&:empty?).presence || [ "test/evals" ]
+    # A path, a path:line or a glob, as `rails test` takes them.
+    missing = paths.reject { |path| File.exist?(path.sub(/:\d+\z/, "")) || Dir.glob(path).any? }
+    abort "No evals found: #{missing.join(", ")} does not exist. Evals are tests under test/evals." if missing.any?
+
+    ENV[ActiveAgent::Evals::TestHelper::ENV_KEY] = "1"
+    runner = Rails::TestUnit::Runner
+    runner.respond_to?(:run_from_rake) ? runner.run_from_rake("test", paths) : runner.rake_run(paths)
+  end
+end