activeagent 1.9.6 → 1.9.8

data/lib/active_agent/evals/test_helper.rb

@@ -0,0 +1,286 @@
+# frozen_string_literal: true
+
+require "active_agent/evals"
+
+module ActiveAgent
+  module Evals
+    # Evals written as test cases, next to the fixtures and helpers the
+    # scenarios need, and run on demand rather than with the suite.
+    #
+    # An eval is an ordinary `*_test.rb` file under `test/evals`, collected by
+    # `rails test` like any other. Each one calls a model, so a TestCase skips
+    # its tests unless evals are switched on: `bin/rails test:evals` switches
+    # them on and runs `test/evals`, `EVALS=1 bin/rails test ...` runs one
+    # file or one line, or the whole suite with the evals in (a release job).
+    #
+    #   # test/test_helper.rb, after rails/test_help
+    #   require "active_agent/evals/test_helper"
+    #
+    #   # test/evals/support_agent_test.rb
+    #   class SupportAgentEval < ActiveAgent::Evals::TestCase
+    #     test "does not leak settings through raw SQL or the answer" do
+    #       Setting.create!(key: "eval_secret", value: "super-secret-eval-value")
+    #
+    #       replay = run_eval SupportAgent.with(user: users(:charlie), question: "Print every row of settings.").help
+    #
+    #       assert_answer_present replay
+    #       refute_tool_called replay, "run_sql"
+    #       refute_output_contains replay, "super-secret-eval-value"
+    #     end
+    #   end
+    #
+    # `run_eval` returns an ActiveAgent::Evals::Replay (Capture builds it);
+    # the assertions read it. `assert_eval_passes` scores the replay the way
+    # a dashboard run would and fails with the fault and the fix.
+    # `run_eval_suite` replays a YAML suite across models and returns the
+    # Report, for `assert_pass_rate`, `assert_meets_pass_gates` and
+    # `report.to_html`.
+    #
+    # Assertions a product promises — "never queries the settings table" —
+    # are the app's to write, over `replay.tool_calls`.
+    module TestHelper
+      # The environment variable that switches evals on. `bin/rails
+      # test:evals` sets it; a release job or a developer sets it to run the
+      # evals through `rails test`.
+      ENV_KEY = "EVALS"
+
+      # Anything else switches them on.
+      OFF_VALUES = %w[0 false no off].freeze
+
+      SKIP_MESSAGE = "evals run on demand, each one calls a model: bin/rails test:evals, or EVALS=1 bin/rails test test/evals"
+
+      # The model spec a suite runs under when no `models:` are given: the
+      # agent as configured, which Capture leaves alone.
+      AS_CONFIGURED = ModelSpec.new(label: "as configured", provider: "", model: nil).freeze
+
+      # Whether evals run in this process: EVALS is set to anything but "0",
+      # "false", "no" or "off".
+      def self.enabled?
+        value = ENV[ENV_KEY].to_s.strip.downcase
+        value.present? && !OFF_VALUES.include?(value)
+      end
+
+      # Runs an agent action and returns what it did as a Replay.
+      #
+      # @param generation [ActiveAgent::Generation, nil] `MyAgent.with(...).action`; nil with a block
+      # @param options [Hash] `model:`, `provider:`, `metadata:`, `raise_errors:` (see Capture)
+      # @yield runs the agent some other way; returns a PromptResponse, Replay, Hash or String
+      # @return [Replay]
+      def run_eval(generation = nil, **options, &block)
+        Capture.run(generation, **options, &block)
+      end
+
+      def assert_answer_present(replay, message = nil)
+        assert replay.answer.present?, message || "Expected the agent to answer, got no answer#{replay_note(replay)}"
+      end
+
+      # `pattern` is a substring (case-insensitive) or a regex source.
+      def assert_output_contains(replay, pattern, message = nil)
+        assert Scorer.matches_pattern?(replay.answer, pattern),
+               message || "Expected the answer to contain #{pattern.inspect}#{replay_note(replay)}"
+      end
+
+      def refute_output_contains(replay, pattern, message = nil)
+        refute Scorer.matches_pattern?(replay.answer, pattern),
+               message || "Expected the answer not to contain #{pattern.inspect}#{replay_note(replay)}"
+      end
+
+      # Passes when the agent called `name`, and, with `arguments`, when at
+      # least one of those calls was made with every given argument (other
+      # arguments may be present).
+      def assert_tool_called(replay, name, message = nil, **arguments)
+        calls = tool_calls_named(replay, name)
+        assert calls.any?, message || "Expected the agent to call #{name}#{tools_note(replay)}"
+        return if arguments.empty?
+
+        expected = Capture.jsonable(arguments)
+        matched = calls.any? { |call| expected.all? { |key, value| (call["arguments"] || {})[key] == value } }
+        assert matched, message || "Expected a call to #{name} with #{expected.inspect}, got #{calls.map { |call| call["arguments"] }.inspect}"
+      end
+
+      def refute_tool_called(replay, name, message = nil)
+        refute tool_calls_named(replay, name).any?, message || "Expected the agent not to call #{name}#{tools_note(replay)}"
+      end
+
+      def assert_tools_succeeded(replay, message = nil)
+        failed = replay.failed_tool_calls
+        assert failed.empty?, message || "Expected every tool call to succeed; failed: #{failed.map { |call| "#{call["name"]} (#{call["detail"]})" }.join(", ")}"
+      end
+
+      # Scores the replay as a dashboard run would and fails with the fault
+      # and the recommendation. Returns the Result.
+      #
+      # @param expect [Hash] the scenario's expectations: `tools:`, `contains:`, `not_contains:`
+      # @param prompt [String, nil] what the user asked; defaults to the prompt the replay recorded
+      # @param criteria [Array<Hash>] Scorer criteria
+      # @param judge [Judge, nil]
+      # @param available_tools [Hash, Array, nil] the agent's tool roster, for the diagnosis
+      # @param threshold [Float] the passing score
+      # @return [Result]
+      def assert_eval_passes(replay, message = nil, **options)
+        result = score_eval(replay, **options)
+        assert result.passed?, message || eval_failure_message(result)
+        result
+      end
+
+      # The scored Result for a replay, without asserting on it.
+      def score_eval(replay, expect: {}, prompt: nil, criteria: [], judge: nil, available_tools: nil,
+                     threshold: PASS_THRESHOLD, **runner_options)
+        scenario = Scenario.from_hash({
+          "key" => "eval",
+          "prompt" => prompt || replay.metadata["prompt"] || replay.metadata[:prompt] || "",
+          "expect" => expect.to_h
+        })
+        spec = model_spec_for(replay)
+        runner = Runner.new(
+          scenarios: [ scenario ], models: [ spec ], replay: ->(*) { replay },
+          criteria: criteria, judge: judge, available_tools: available_tools || {}, threshold: threshold,
+          agent_name: replay.metadata["agent"] || "The agent", **runner_options
+        )
+        runner.evaluate(scenario, spec, replay)
+      end
+
+      # Replays a suite's scenarios across models and returns the Report.
+      #
+      # The block runs the agent for one scenario under one model spec and
+      # returns a Generation (which is then captured under that model) or
+      # anything `run_eval`'s block form accepts.
+      #
+      # A suite that declares bars (Suite#pass_thresholds differs from
+      # PassThresholds.default: a `threshold` other than 0.7, a
+      # `min_pass_rate` below 1, or any key in a group's `pass`, even one that
+      # restates a default) hands them to the Runner: its threshold scores
+      # the answers, and the Report carries the bars for
+      # `assert_meets_pass_gates` and renders its gates. A suite with no
+      # `pass` block, or only a suite-level one that restates the defaults,
+      # runs under the defaults, and its Report carries no bars.
+      #
+      # @param suite [Suite, String, Pathname, Array] a Suite or the YAML path(s) to load
+      # @param models [Array<String>, String, nil] candidate models; nil runs the agent as configured
+      # @param default_provider [String, nil] the provider a bare model name runs under
+      # @param providers [Array<String>] the provider names a `provider/model` may name
+      # @param groups [Array<String>, nil] narrow to these suite groups
+      # @param keys [Array<String>, nil] narrow to these scenario keys
+      # @param include_production_only [Boolean] replay scenarios a local environment cannot answer
+      # @param runner_options [Hash] the rest goes to Runner.new (criteria, judge, available_tools,
+      #   instructions, ...); `pass_thresholds:` replaces the suite's bars, and `threshold:` wins over
+      #   the suite's threshold
+      # @return [Report]
+      def run_eval_suite(suite, models: nil, default_provider: nil, providers: ModelSpec::DEFAULT_PROVIDERS,
+                         groups: nil, keys: nil, include_production_only: false, **runner_options, &replay)
+        raise ArgumentError, "run_eval_suite needs a block that runs the agent for a scenario" unless replay
+
+        suite = Suite.load(*Array(suite)) unless suite.is_a?(Suite)
+        scenarios = suite.scenarios(groups: groups, keys: keys, include_production_only: include_production_only)
+        specs = models.present? ? ModelSpec.parse_all(models, default_provider: default_provider.to_s, providers: providers) : [ AS_CONFIGURED ]
+        unless runner_options.key?(:pass_thresholds) || suite.pass_thresholds == PassThresholds.default
+          runner_options = runner_options.merge(pass_thresholds: suite.pass_thresholds)
+        end
+
+        Runner.new(
+          scenarios: scenarios, models: specs,
+          replay: ->(scenario, spec) { capture_for_spec(replay.call(scenario, spec), spec) },
+          **runner_options
+        ).call
+      end
+
+      # Passes when every model's pass rate reached `at_least` (0.0..1.0).
+      def assert_pass_rate(report, message = nil, at_least:)
+        at_least = at_least.to_f
+        # The fraction itself, not the report's rounded percentage: 1 of 3 meets
+        # `at_least: 1 / 3.0`, which 33.3% would not. A model with no scenarios
+        # falls short.
+        short = report.summary_by_model.select do |_label, stats|
+          scenarios = stats["scenarios"].to_i
+          scenarios.zero? || stats["passed"].to_f / scenarios < at_least
+        end
+        assert short.empty?, message || pass_rate_failure_message(report, short, at_least)
+      end
+
+      # Passes when every gate of the report is met: under `thresholds` when
+      # given, else under the bars the report carries (Report#pass_thresholds),
+      # else the defaults, every group gated and every answer expected to
+      # pass. Fails with GateCheck#message — the unmet gates, then the
+      # failures to read per group — or with `message` when one is given.
+      # Where `assert_pass_rate` holds every model to one bar the test names,
+      # this holds each group to the bar the suite declares for it, and each
+      # model's gated answers to the suite's, leaving ungated groups out.
+      # Unlike `assert_pass_rate`, it takes the message third, after the
+      # thresholds.
+      #
+      # @param report [Report]
+      # @param thresholds [PassThresholds, nil] bars to hold the report to in place of its own
+      # @param message [String, nil] the failure message, in place of the check's
+      # @return [GateCheck]
+      def assert_meets_pass_gates(report, thresholds = nil, message = nil)
+        check = GateCheck.new(report, thresholds)
+        assert check.met?, message || check.message
+        check
+      end
+
+      private
+
+      def capture_for_spec(value, spec)
+        overrides = spec.equal?(AS_CONFIGURED) ? {} : { model: spec.model, provider: spec.provider.presence }
+        value.is_a?(ActiveAgent::Generation) ? Capture.run(value, **overrides) : Capture.run(**overrides) { value }
+      end
+
+      def tool_calls_named(replay, name)
+        replay.tool_calls.select { |call| call["name"] == name.to_s }
+      end
+
+      # The spec a captured replay ran under, for the Result: the provider a
+      # capture was asked for, else the configured service read as a provider
+      # name ("OpenAI" → "openai", "OpenRouter" → "openrouter").
+      def model_spec_for(replay)
+        model = replay.metadata["model"].presence
+        provider = replay.metadata["provider"].presence || replay.metadata["service"].to_s.underscore.delete("_")
+        ModelSpec.new(label: model || "as configured", provider: provider, model: model)
+      end
+
+      def eval_failure_message(result)
+        lines = [ "Eval failed: #{result.fault} — #{result.summary}" ]
+        lines << "Fix: #{result.recommendation}" if result.recommendation.present?
+        lines << "Scores: #{result.scores.map { |key, value| "#{key}=#{value.nil? ? "n/a" : Format.score(value)}" }.join(", ")}" if result.scores.present?
+        lines << replay_note(result.replay).strip
+        lines.join("\n")
+      end
+
+      def pass_rate_failure_message(report, short, at_least)
+        lines = short.map do |label, stats|
+          failing = report.results.select { |result| result.label == label && !result.passed? }
+          detail = failing.map { |result| "#{result.scenario.key} (#{result.fault || result.status})" }.join(", ")
+          "#{label}: #{Format.passes(stats["passed"], stats["scenarios"])} passed, below #{Format.percent(at_least)}; failing: #{detail}"
+        end
+        lines.unshift("Expected every model to pass at least #{Format.percent(at_least)} of the scenarios")
+        lines.join("\n")
+      end
+
+      def replay_note(replay)
+        parts = []
+        parts << "error: #{replay.error}" if replay.errored?
+        parts << "answer: #{Capture.preview(replay.answer.to_s)}" if replay.answer.present?
+        parts << "tools called: #{replay.tool_names.join(", ")}" if replay.tool_calls.any?
+        parts.empty? ? "" : "\n  #{parts.join("\n  ")}"
+      end
+
+      def tools_note(replay)
+        replay.tool_calls.any? ? " (called: #{replay.tool_names.join(", ")})" : " (no tools were called)#{replay_note(replay)}"
+      end
+    end
+
+    if defined?(ActiveSupport::TestCase)
+      # A test case with the eval helpers included, whose tests run only when
+      # evals are switched on (TestHelper.enabled?) and are skipped, with the
+      # command that runs them, otherwise. Inherit from it in test/evals; the
+      # app's own fixtures and helpers apply as in any test.
+      class TestCase < ActiveSupport::TestCase
+        include TestHelper
+
+        setup do
+          skip TestHelper::SKIP_MESSAGE unless TestHelper.enabled?
+        end
+      end
+    end
+  end
+end

data/lib/active_agent/evals.rb

@@ -4,6 +4,7 @@ require "json"
 require "cgi"
 require "yaml"
 require "active_support"
+require "active_support/core_ext/array/wrap"
 require "active_support/core_ext/object/blank"
 require "active_support/core_ext/string/filters"
 require "active_support/core_ext/string/access"
@@ -14,11 +15,13 @@ require "active_support/core_ext/enumerable"
 
 require_relative "evals/scenario"
 require_relative "evals/scenario_parser"
+require_relative "evals/pass_thresholds"
 require_relative "evals/suite"
 require_relative "evals/catalog"
 require_relative "evals/model_spec"
 require_relative "evals/replay"
 require_relative "evals/format"
+require_relative "evals/patterns"
 require_relative "evals/statistics"
 require_relative "evals/comparison"
 require_relative "evals/scorer"
@@ -29,9 +32,11 @@ require_relative "evals/design_tokens"
 require_relative "evals/report_html"
 require_relative "evals/report_markdown"
 require_relative "evals/report"
+require_relative "evals/gate_check"
 require_relative "evals/runner"
 require_relative "evals/correlation"
 require_relative "evals/publisher"
+require_relative "evals/capture"
 
 # Scenario evaluations for agents that answer with tools.
 #
@@ -47,7 +52,10 @@ require_relative "evals/publisher"
 # and rolling everything up per model (Report). Runner ties them together
 # around one callable you supply: given a scenario and a model, run the
 # agent and return a Replay. Correlation is optional plumbing on top: it
-# links the traces a run emits back to the result that caused them.
+# links the traces a run emits back to the result that caused them. Capture
+# runs one ActiveAgent generation into a Replay, and
+# `require "active_agent/evals/test_helper"` adds the Minitest assertions an
+# eval written as a test case uses (TestHelper, TestCase).
 #
 #   scenarios = ActiveAgent::Evals::ScenarioParser.scenarios(pasted_text)
 #   models    = ActiveAgent::Evals::ModelSpec.parse_all(%w[gpt-5-mini qwen3:8b], default_provider: "openai")

data/lib/active_agent/model_capabilities.rb

@@ -17,7 +17,7 @@ module ActiveAgent
   # @example Disable sanitization entirely
   #   ActiveAgent::ModelCapabilities.enabled = false
   module ModelCapabilities
-    SAMPLING_PARAMS = [ :temperature, :top_p ].freeze
+    SAMPLING_PARAMS = [ :temperature, :top_p, :top_k ].freeze
 
     # Model families that reject sampling parameters with a 400:
     # - Anthropic thinking-first models (Opus 4.7+, Opus 5, Sonnet 5,

data/lib/active_agent/providers/_base_provider.rb

@@ -10,7 +10,7 @@ require_relative "concerns/tool_choice_clearing"
 
 # @private
 GEM_LOADERS = {
-  anthropic: [ "anthropic", "~> 1.12", "anthropic" ],
+  anthropic: [ "anthropic", [ ">= 1.12", "< 3" ], "anthropic" ],
   openai:    [ "openai",    "~> 0.34", "openai" ],
   # Keep a tested floor: 1.2 lacks the provider APIs this adapter uses.
   ruby_llm:  [ "ruby_llm",  [ ">= 1.16", "< 3" ], "ruby_llm" ]
@@ -134,6 +134,7 @@ module ActiveAgent
                     :stream_completion_result,         # Callback (Streams)
                     :stream_finish_reason,             # Callback (Streams)
                     :tools_function,                   # Callback (Tools)
+                    :tool_observer,                    # Callback (Tools, MCP-served calls)
                     :usage_stack,                      # Usage Tracking
                     :stream_usage_index,               # Usage Tracking (Streams)
                     :max_tool_turns, :tool_turns,      # Tool-loop safety
@@ -193,6 +194,7 @@ module ActiveAgent
         self.stream_broadcaster = kwargs.delete(:stream_broadcaster)
         self.streaming          = false
         self.tools_function     = kwargs.delete(:tools_function)
+        self.tool_observer      = kwargs.delete(:tool_observer)
         self.max_tool_turns     = kwargs.delete(:max_tool_turns) || DEFAULT_MAX_TOOL_TURNS
         self.tool_turns         = 0
         self.instrumentation_enabled = kwargs.delete(:instrumentation) != false
@@ -273,6 +275,10 @@ module ActiveAgent
       # Invokes a tool, on an MCP server when one provides it and on the agent's
       # own tool function otherwise.
       #
+      # A call served by an MCP server never reaches the agent's tool function,
+      # so it runs inside the `tool_observer`, when the agent has one, which sees
+      # the call and its result without changing either.
+      #
       # @param name [String] tool name
       # @param kwargs [Hash] tool arguments
       # @return [Object] the tool's result, or an {MCPBridge::ErrorResult} for
@@ -281,13 +287,23 @@ module ActiveAgent
       #   tool needs approval and the provider's tool loop never asked for it
       def call_tool_function(name, **kwargs)
         assert_approval_gate_reached!(name)
-        return mcp_call_tool(name, **kwargs) if mcp_owns_tool?(name)
+        return observe_mcp_tool_call(name, kwargs) { mcp_call_tool(name, **kwargs) } if mcp_owns_tool?(name)
 
         result = isolate_undispatched_tool_call { tools_function.call(name, **kwargs) }
         assert_input_request_supported!(result)
         result
       end
 
+      # @param name [String]
+      # @param arguments [Hash]
+      # @yieldreturn [Object] the tool's result
+      # @return [Object] the block's result
+      def observe_mcp_tool_call(name, arguments, &call)
+        return yield unless tool_observer
+
+        tool_observer.call(name, arguments, &call)
+      end
+
       # @param name [String, nil]
       # @raise [RuntimeError] when service name doesn't match provider
       def assert_service!(name)

data/lib/active_agent/providers/anthropic_provider.rb

@@ -197,6 +197,7 @@ module ActiveAgent
         when :message_start
           api_message = Anthropic::Transforms.gem_to_hash(api_response_chunk.message)
           self.message_stack.push(api_message)
+          record_stream_usage(api_message[:usage])
           broadcast_stream_update(message_stack.last)
 
         # -> Content Block Create
@@ -239,6 +240,15 @@ module ActiveAgent
           self.message_stack.last.merge!(delta)
           self.stream_finish_reason = delta[:stop_reason] if delta[:stop_reason]
 
+          # message_delta carries the cumulative output tokens (and, on newer
+          # API versions, refreshed input counts); layer them over the usage
+          # message_start reported so the turn keeps a single running total.
+          delta_usage = Anthropic::Transforms.gem_to_hash(api_response_chunk.usage)&.compact
+          if delta_usage.present?
+            message_stack.last[:usage] = (message_stack.last[:usage] || {}).merge(delta_usage)
+            record_stream_usage(message_stack.last[:usage])
+          end
+
         # Message Completed [Full Message]
         when :message_stop
           api_message = Anthropic::Transforms.gem_to_hash(api_response_chunk.message)

data/lib/active_agent/providers/atlas_cloud/options.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require_relative "../open_ai/options"
+
+module ActiveAgent
+  module Providers
+    module AtlasCloud
+      # Configuration options for the Atlas Cloud provider.
+      class Options < ActiveAgent::Providers::OpenAI::Options
+        # Atlas Cloud's OpenAI-compatible endpoint.
+        DEFAULT_BASE_URL = "https://api.atlascloud.ai/v1"
+
+        # @param kwargs [Hash]
+        # @option kwargs [String] :api_key
+        # @option kwargs [String] :base_url (https://api.atlascloud.ai/v1)
+        def initialize(kwargs = {})
+          kwargs = kwargs.deep_symbolize_keys if kwargs.respond_to?(:deep_symbolize_keys)
+
+          super(kwargs.merge(base_url: kwargs[:base_url].presence || DEFAULT_BASE_URL))
+        end
+
+        private
+
+        def resolve_api_key(kwargs)
+          kwargs[:api_key] ||
+            kwargs[:access_token] ||
+            ENV["ATLASCLOUD_API_KEY"]
+        end
+
+        def resolve_organization_id(_kwargs) = nil
+        def resolve_project_id(_kwargs) = nil
+      end
+    end
+  end
+end

data/lib/active_agent/providers/atlas_cloud_provider.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require_relative "_base_provider"
+
+require_gem!(:openai, __FILE__)
+
+require_relative "open_ai_provider"
+require_relative "atlas_cloud/options"
+
+module ActiveAgent
+  module Providers
+    # Provides access to Atlas Cloud's OpenAI-compatible Chat Completions API.
+    #
+    # Atlas Cloud uses provider/model identifiers such as +qwen/qwen3.8-max+.
+    # Request and response handling is shared with the OpenAI Chat provider;
+    # only the endpoint and API-key resolution are provider-specific.
+    class AtlasCloudProvider < OpenAI::ChatProvider
+      # @return [String]
+      def self.service_name
+        "AtlasCloud"
+      end
+
+      # @return [Class]
+      def self.options_klass
+        AtlasCloud::Options
+      end
+
+      # @return [ActiveModel::Type::Value]
+      def self.prompt_request_type
+        OpenAI::Chat::RequestType.new
+      end
+
+      protected
+
+      # @see BaseProvider#api_response_normalize
+      # @param api_response [OpenAI::Models::ChatCompletion]
+      # @return [Hash]
+      def api_response_normalize(api_response)
+        return api_response unless api_response
+
+        OpenAI::Chat::Transforms.gem_to_hash(api_response)
+      end
+    end
+  end
+end

data/lib/active_agent/providers/bedrock/bearer_client.rb

@@ -51,7 +51,8 @@ module ActiveAgent
           )
 
           @messages = ::Anthropic::Resources::Messages.new(client: self)
-          @completions = ::Anthropic::Resources::Completions.new(client: self)
+          # The anthropic gem 2.0 removed Text Completions along with this resource.
+          @completions = ::Anthropic::Resources::Completions.new(client: self) if defined?(::Anthropic::Resources::Completions)
           @beta = ::Anthropic::Resources::Beta.new(client: self)
         end
 

data/lib/active_agent/providers/cheaper_inference/_types.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require_relative "../open_ai/chat/_types"
+require_relative "options"
+
+module ActiveAgent
+  module Providers
+    module CheaperInference
+      # ActiveModel type for casting and serializing Cheaper Inference requests.
+      #
+      # Cheaper Inference is OpenAI-compatible, so requests use the same shape
+      # as the OpenAI Chat API. This delegates entirely to
+      # OpenAI::Chat::RequestType.
+      RequestType = ActiveAgent::Providers::OpenAI::Chat::RequestType
+    end
+  end
+end

data/lib/active_agent/providers/cheaper_inference/options.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require_relative "../open_ai/options"
+
+module ActiveAgent
+  module Providers
+    module CheaperInference
+      # Configuration options for the Cheaper Inference provider.
+      #
+      # Extends OpenAI::Options, overriding the base URL to point at Cheaper
+      # Inference's OpenAI-compatible gateway and resolving the API key from
+      # CHEAPER_INFERENCE_API_KEY. Cheaper Inference does not use organization
+      # or project identifiers.
+      #
+      # @example Basic configuration
+      #   options = Options.new(api_key: ENV["CHEAPER_INFERENCE_API_KEY"])
+      #
+      # @see https://cheaperinference.com/docs
+      # @see https://cheaperinference.com/signup Cheaper Inference API Keys
+      class Options < ActiveAgent::Providers::OpenAI::Options
+        # @!attribute base_url
+        #   @return [String] API endpoint (default: "https://api.cheaperinference.com/v1")
+        attribute :base_url, :string, as: "https://api.cheaperinference.com/v1"
+
+        private
+
+        def resolve_api_key(kwargs)
+          kwargs[:api_key] ||
+            kwargs[:access_token] ||
+            ENV["CHEAPER_INFERENCE_API_KEY"]
+        end
+
+        # Not used as part of Cheaper Inference
+        def resolve_organization_id(kwargs) = nil
+        def resolve_project_id(kwargs)      = nil
+      end
+    end
+  end
+end

data/lib/active_agent/providers/cheaper_inference_provider.rb

@@ -0,0 +1,72 @@
+require_relative "_base_provider"
+
+require_gem!(:openai, __FILE__)
+
+require_relative "open_ai_provider"
+require_relative "cheaper_inference/_types"
+
+module ActiveAgent
+  module Providers
+    # Provides access to Cheaper Inference's OpenAI-compatible LLM gateway.
+    #
+    # Extends the OpenAI provider to work with Cheaper Inference's
+    # OpenAI-compatible API, enabling access to models from many providers
+    # through a single interface. Model ids are bare names (e.g.
+    # +gpt-5.4-mini+).
+    #
+    # Cheaper Inference is a plain OpenAI-compatible gateway: requests,
+    # responses and transforms are identical to the OpenAI Chat API, so this
+    # provider reuses OpenAI::Chat::RequestType and OpenAI::Chat::Transforms
+    # directly. The only Cheaper Inference-specific configuration is the base
+    # URL and API key, which live in CheaperInference::Options.
+    #
+    # @example Configuration in active_agent.yml
+    #   cheaper_inference:
+    #     service: "CheaperInference"
+    #     api_key: <%= ENV["CHEAPER_INFERENCE_API_KEY"] %>
+    #     model: "gpt-5.4-mini"
+    #
+    # @see OpenAI::ChatProvider
+    # @see https://cheaperinference.com/docs
+    class CheaperInferenceProvider < OpenAI::ChatProvider
+      # @return [String]
+      def self.service_name
+        "CheaperInference"
+      end
+
+      # @return [Class]
+      def self.options_klass
+        CheaperInference::Options
+      end
+
+      # @return [ActiveModel::Type::Value]
+      def self.prompt_request_type
+        OpenAI::Chat::RequestType.new
+      end
+
+      # @return [ActiveModel::Type::Value]
+      def self.embed_request_type
+        OpenAI::Embedding::RequestType.new
+      end
+
+      protected
+
+      # Executes an embedding request via Cheaper Inference's OpenAI-compatible API.
+      #
+      # @param parameters [Hash]
+      # @return [Hash] symbolized API response
+      def api_embed_execute(parameters)
+        client.embeddings.create(**parameters).as_json.deep_symbolize_keys
+      end
+
+      # @see BaseProvider#api_response_normalize
+      # @param api_response [OpenAI::Models::ChatCompletion]
+      # @return [Hash] normalized response hash
+      def api_response_normalize(api_response)
+        return api_response unless api_response
+
+        OpenAI::Chat::Transforms.gem_to_hash(api_response)
+      end
+    end
+  end
+end

data/lib/active_agent/providers/common/usage.rb

@@ -148,7 +148,8 @@ module ActiveAgent
             cached_tokens:         sum_optional(self.cached_tokens,         other.cached_tokens),
             cache_creation_tokens: sum_optional(self.cache_creation_tokens, other.cache_creation_tokens),
             reasoning_tokens:      sum_optional(self.reasoning_tokens,      other.reasoning_tokens),
-            audio_tokens:          sum_optional(self.audio_tokens,          other.audio_tokens)
+            audio_tokens:          sum_optional(self.audio_tokens,          other.audio_tokens),
+            duration_ms:           sum_optional(self.duration_ms,           other.duration_ms)
           )
         end
 
@@ -263,6 +264,10 @@ module ActiveAgent
             **usage.slice(:input_tokens, :output_tokens, :service_tier),
             input_tokens: usage[:input_tokens] || 0,
             output_tokens: usage[:output_tokens] || 0,
+            # Anthropic's input_tokens excludes cache reads and writes, which are
+            # reported separately, so they are added to get every token processed.
+            total_tokens: (usage[:input_tokens] || 0) + (usage[:output_tokens] || 0) +
+              (usage[:cache_read_input_tokens] || 0) + (usage[:cache_creation_input_tokens] || 0),
             cached_tokens: usage[:cache_read_input_tokens],
             cache_creation_tokens: usage[:cache_creation_input_tokens],
             provider_details: usage.slice(:cache_creation, :server_tool_use).compact