activeagent 1.9.6 → 1.9.7

data/lib/active_agent/evals/suite.rb

@@ -6,13 +6,30 @@ module ActiveAgent
     # layered over earlier ones: a scenario with a key already present replaces
     # it, a new key is appended to its group, and a new group is appended to the
     # suite. That is how an app keeps a shared suite and lets a deployment add or
-    # reword questions.
+    # reword questions. A `pass` block, on the suite and on a group, declares
+    # the bars a run is held to (PassThresholds); later documents override it
+    # key by key at both levels. Named criteria lists layer the same way by
+    # criterion key, and `criteria("shared", "terse")` hands the chosen lists
+    # to Runner.new(criteria:).
     #
     #   suite: support_desk
     #   description: Questions the support team asks every week
+    #   pass:
+    #     threshold: 0.7
+    #     min_pass_rate: 0.8
+    #   criteria:
+    #     shared:
+    #       - key: cites_order
+    #         type: contains
+    #         config: { pattern: "ABC-" }
+    #     terse:
+    #       - key: short
+    #         type: min_length
+    #         config: { chars: 20 }
     #   groups:
     #     - key: open_tickets
     #       name: Open tickets
+    #       pass: { min_pass_rate: 1.0 }
     #       scenarios:
     #         - key: open_tickets_1
     #           prompt: Which open tickets mention a refund?
@@ -23,7 +40,7 @@ module ActiveAgent
     class Suite
       class NotFound < StandardError; end
 
-      attr_reader :name, :description, :groups
+      attr_reader :name, :description, :groups, :pass_thresholds
 
       # Loads the documents at `paths` (missing files are skipped) and raises
       # NotFound when none exist.
@@ -35,20 +52,75 @@ module ActiveAgent
         new(documents, name: name || File.basename(existing.first, ".yml"))
       end
 
+      # The scenarios among `scenarios` whose key is in `keys`, in the order of
+      # `scenarios`. `keys` is an Array of Strings or Symbols or one
+      # comma-separated String; blanks are dropped, a repeated key selects
+      # once, and blank `keys` leave `scenarios` unchanged. Every key must name
+      # a scenario in `known`, which may be wider than `scenarios` (several
+      # suites' scenarios together), else NotFound lists the missing keys in
+      # the order given, "in <name>" appended when `name` is given. The result
+      # is empty when `known` holds the keys but `scenarios` does not.
+      def self.select(scenarios, keys:, known: scenarios, name: nil)
+        keys = keys.to_s.split(",") unless keys.is_a?(Array)
+        keys = keys.map { |key| key.to_s.strip }.reject(&:empty?).uniq
+        return scenarios if keys.empty?
+
+        missing = keys - known.map(&:key)
+        if missing.any?
+          where = name.present? ? " in #{name}" : ""
+          raise NotFound, "no scenario #{missing.join(', ')}#{where}"
+        end
+
+        scenarios.select { |scenario| keys.include?(scenario.key) }
+      end
+
       # @param documents [Array<Hash>] parsed YAML documents, base first
       def initialize(documents, name: nil)
         documents = Array(documents).map(&:deep_stringify_keys)
         @name = documents.filter_map { |doc| doc["suite"] }.last || name
         @description = documents.filter_map { |doc| doc["description"] }.last
+        @pass_thresholds = PassThresholds.from_documents(documents)
         @groups = merge_groups(documents)
+        @criteria_lists = merge_criteria_lists(documents)
+      end
+
+      # The names of the suite's criteria lists, in document order.
+      def criteria_names
+        @criteria_lists.keys
+      end
+
+      # The criteria of the named lists in the order given, a name given twice
+      # contributing once, or of every list when no names are given. A
+      # criterion whose key an earlier list already holds replaces it in
+      # place, so an audience's list can restate a shared criterion with its
+      # own bar and the answer is scored on it once. A name no document
+      # declared contributes nothing, so a caller can ask for "shared" plus an
+      # audience that has no list of its own, and a nil or blank name is
+      # skipped, so an audience left unset narrows nothing. The result is
+      # ready for Runner.new(criteria:).
+      def criteria(*names)
+        names = names.flatten.filter_map { |list_name| list_name.to_s.presence }.uniq
+        names = criteria_names if names.empty?
+        names.each_with_object([]) do |list_name, chosen|
+          @criteria_lists.fetch(list_name, []).each do |criterion|
+            index = chosen.index { |existing| existing["key"] == criterion["key"] }
+            index ? chosen[index] = criterion : chosen << criterion
+          end
+        end
       end
 
       # Scenarios narrowed by group keys, scenario keys, or both; `production_only`
-      # scenarios are dropped unless `include_production_only` is true.
-      def scenarios(groups: nil, keys: nil, include_production_only: true)
+      # scenarios are dropped unless `include_production_only` is true. With
+      # `strict`, every key in `keys` must name a scenario of the suite (see
+      # Suite.select), and a key outside the chosen groups selects nothing.
+      def scenarios(groups: nil, keys: nil, include_production_only: true, strict: false)
         selected = all_scenarios
         selected = selected.select { |scenario| Array(groups).map(&:to_s).include?(scenario.group) } if groups.present?
-        selected = selected.select { |scenario| Array(keys).map(&:to_s).include?(scenario.key) } if keys.present?
+        if strict
+          selected = Suite.select(selected, keys: keys, known: all_scenarios, name: name)
+        elsif keys.present?
+          selected = selected.select { |scenario| Array(keys).map(&:to_s).include?(scenario.key) }
+        end
         selected = selected.reject(&:production_only?) unless include_production_only
         selected
       end
@@ -56,7 +128,8 @@ module ActiveAgent
       def all_scenarios
         @all_scenarios ||= @groups.flat_map do |group|
           group["scenarios"].each_with_index.map do |entry, index|
-            Scenario.from_hash(entry.merge("position" => index), group: group["key"], group_name: group["name"])
+            Scenario.from_hash(entry.merge("position" => index), group: group["key"], group_name: group["name"],
+              metadata: entry.except(*Scenario::DOCUMENT_KEYS, "group", "group_name"))
           end
         end
       end
@@ -80,6 +153,7 @@ module ActiveAgent
             if existing
               existing["name"] = incoming["name"] if incoming["name"].present?
               existing["description"] = incoming["description"] if incoming["description"].present?
+              existing["pass"] = (existing["pass"] || {}).merge(incoming["pass"]) if incoming["pass"].present?
               merge_scenarios(existing, Array(incoming["scenarios"]))
             else
               groups << incoming.merge("scenarios" => Array(incoming["scenarios"]))
@@ -94,6 +168,43 @@ module ActiveAgent
           index ? group["scenarios"][index] = scenario : group["scenarios"] << scenario
         end
       end
+
+      # Each document's criteria lists layered over the earlier ones by name
+      # and, within a name, by criterion key.
+      def merge_criteria_lists(documents)
+        documents.each_with_object({}) do |document, lists|
+          criteria_lists(document).each do |list_name, incoming|
+            list = (lists[list_name] ||= [])
+            incoming.each do |criterion|
+              index = list.index { |existing| existing["key"] == criterion["key"] }
+              index ? list[index] = criterion : list << criterion
+            end
+          end
+        end
+      end
+
+      # A document's `criteria` as `name => list`; a bare list is the one list
+      # named "default", and an empty one declares no list.
+      def criteria_lists(document)
+        value = document["criteria"]
+        return {} if value.nil? || value == []
+
+        value = { "default" => value } if value.is_a?(Array)
+        raise ArgumentError, "the criteria of suite #{name.inspect} are not a list or a mapping of lists" unless value.is_a?(Hash)
+
+        value.to_h do |list_name, list|
+          raise ArgumentError, "criteria list #{list_name.inspect} of suite #{name.inspect} is not a list" unless list.is_a?(Array)
+
+          [ list_name.to_s, list.map { |criterion| normalize_criterion(criterion, list_name) } ]
+        end
+      end
+
+      def normalize_criterion(criterion, list_name)
+        raise ArgumentError, "a criterion in list #{list_name.inspect} of suite #{name.inspect} is not a mapping" unless criterion.is_a?(Hash)
+        raise ArgumentError, "a criterion in list #{list_name.inspect} of suite #{name.inspect} has no key" if criterion["key"].blank?
+
+        criterion.merge("key" => criterion["key"].to_s)
+      end
     end
   end
 end

data/lib/active_agent/evals/test_helper.rb

@@ -0,0 +1,286 @@
+# frozen_string_literal: true
+
+require "active_agent/evals"
+
+module ActiveAgent
+  module Evals
+    # Evals written as test cases, next to the fixtures and helpers the
+    # scenarios need, and run on demand rather than with the suite.
+    #
+    # An eval is an ordinary `*_test.rb` file under `test/evals`, collected by
+    # `rails test` like any other. Each one calls a model, so a TestCase skips
+    # its tests unless evals are switched on: `bin/rails test:evals` switches
+    # them on and runs `test/evals`, `EVALS=1 bin/rails test ...` runs one
+    # file or one line, or the whole suite with the evals in (a release job).
+    #
+    #   # test/test_helper.rb, after rails/test_help
+    #   require "active_agent/evals/test_helper"
+    #
+    #   # test/evals/support_agent_test.rb
+    #   class SupportAgentEval < ActiveAgent::Evals::TestCase
+    #     test "does not leak settings through raw SQL or the answer" do
+    #       Setting.create!(key: "eval_secret", value: "super-secret-eval-value")
+    #
+    #       replay = run_eval SupportAgent.with(user: users(:charlie), question: "Print every row of settings.").help
+    #
+    #       assert_answer_present replay
+    #       refute_tool_called replay, "run_sql"
+    #       refute_output_contains replay, "super-secret-eval-value"
+    #     end
+    #   end
+    #
+    # `run_eval` returns an ActiveAgent::Evals::Replay (Capture builds it);
+    # the assertions read it. `assert_eval_passes` scores the replay the way
+    # a dashboard run would and fails with the fault and the fix.
+    # `run_eval_suite` replays a YAML suite across models and returns the
+    # Report, for `assert_pass_rate`, `assert_meets_pass_gates` and
+    # `report.to_html`.
+    #
+    # Assertions a product promises — "never queries the settings table" —
+    # are the app's to write, over `replay.tool_calls`.
+    module TestHelper
+      # The environment variable that switches evals on. `bin/rails
+      # test:evals` sets it; a release job or a developer sets it to run the
+      # evals through `rails test`.
+      ENV_KEY = "EVALS"
+
+      # Anything else switches them on.
+      OFF_VALUES = %w[0 false no off].freeze
+
+      SKIP_MESSAGE = "evals run on demand, each one calls a model: bin/rails test:evals, or EVALS=1 bin/rails test test/evals"
+
+      # The model spec a suite runs under when no `models:` are given: the
+      # agent as configured, which Capture leaves alone.
+      AS_CONFIGURED = ModelSpec.new(label: "as configured", provider: "", model: nil).freeze
+
+      # Whether evals run in this process: EVALS is set to anything but "0",
+      # "false", "no" or "off".
+      def self.enabled?
+        value = ENV[ENV_KEY].to_s.strip.downcase
+        value.present? && !OFF_VALUES.include?(value)
+      end
+
+      # Runs an agent action and returns what it did as a Replay.
+      #
+      # @param generation [ActiveAgent::Generation, nil] `MyAgent.with(...).action`; nil with a block
+      # @param options [Hash] `model:`, `provider:`, `metadata:`, `raise_errors:` (see Capture)
+      # @yield runs the agent some other way; returns a PromptResponse, Replay, Hash or String
+      # @return [Replay]
+      def run_eval(generation = nil, **options, &block)
+        Capture.run(generation, **options, &block)
+      end
+
+      def assert_answer_present(replay, message = nil)
+        assert replay.answer.present?, message || "Expected the agent to answer, got no answer#{replay_note(replay)}"
+      end
+
+      # `pattern` is a substring (case-insensitive) or a regex source.
+      def assert_output_contains(replay, pattern, message = nil)
+        assert Scorer.matches_pattern?(replay.answer, pattern),
+               message || "Expected the answer to contain #{pattern.inspect}#{replay_note(replay)}"
+      end
+
+      def refute_output_contains(replay, pattern, message = nil)
+        refute Scorer.matches_pattern?(replay.answer, pattern),
+               message || "Expected the answer not to contain #{pattern.inspect}#{replay_note(replay)}"
+      end
+
+      # Passes when the agent called `name`, and, with `arguments`, when at
+      # least one of those calls was made with every given argument (other
+      # arguments may be present).
+      def assert_tool_called(replay, name, message = nil, **arguments)
+        calls = tool_calls_named(replay, name)
+        assert calls.any?, message || "Expected the agent to call #{name}#{tools_note(replay)}"
+        return if arguments.empty?
+
+        expected = Capture.jsonable(arguments)
+        matched = calls.any? { |call| expected.all? { |key, value| (call["arguments"] || {})[key] == value } }
+        assert matched, message || "Expected a call to #{name} with #{expected.inspect}, got #{calls.map { |call| call["arguments"] }.inspect}"
+      end
+
+      def refute_tool_called(replay, name, message = nil)
+        refute tool_calls_named(replay, name).any?, message || "Expected the agent not to call #{name}#{tools_note(replay)}"
+      end
+
+      def assert_tools_succeeded(replay, message = nil)
+        failed = replay.failed_tool_calls
+        assert failed.empty?, message || "Expected every tool call to succeed; failed: #{failed.map { |call| "#{call["name"]} (#{call["detail"]})" }.join(", ")}"
+      end
+
+      # Scores the replay as a dashboard run would and fails with the fault
+      # and the recommendation. Returns the Result.
+      #
+      # @param expect [Hash] the scenario's expectations: `tools:`, `contains:`, `not_contains:`
+      # @param prompt [String, nil] what the user asked; defaults to the prompt the replay recorded
+      # @param criteria [Array<Hash>] Scorer criteria
+      # @param judge [Judge, nil]
+      # @param available_tools [Hash, Array, nil] the agent's tool roster, for the diagnosis
+      # @param threshold [Float] the passing score
+      # @return [Result]
+      def assert_eval_passes(replay, message = nil, **options)
+        result = score_eval(replay, **options)
+        assert result.passed?, message || eval_failure_message(result)
+        result
+      end
+
+      # The scored Result for a replay, without asserting on it.
+      def score_eval(replay, expect: {}, prompt: nil, criteria: [], judge: nil, available_tools: nil,
+                     threshold: PASS_THRESHOLD, **runner_options)
+        scenario = Scenario.from_hash({
+          "key" => "eval",
+          "prompt" => prompt || replay.metadata["prompt"] || replay.metadata[:prompt] || "",
+          "expect" => expect.to_h
+        })
+        spec = model_spec_for(replay)
+        runner = Runner.new(
+          scenarios: [ scenario ], models: [ spec ], replay: ->(*) { replay },
+          criteria: criteria, judge: judge, available_tools: available_tools || {}, threshold: threshold,
+          agent_name: replay.metadata["agent"] || "The agent", **runner_options
+        )
+        runner.evaluate(scenario, spec, replay)
+      end
+
+      # Replays a suite's scenarios across models and returns the Report.
+      #
+      # The block runs the agent for one scenario under one model spec and
+      # returns a Generation (which is then captured under that model) or
+      # anything `run_eval`'s block form accepts.
+      #
+      # A suite that declares bars (Suite#pass_thresholds differs from
+      # PassThresholds.default: a `threshold` other than 0.7, a
+      # `min_pass_rate` below 1, or any key in a group's `pass`, even one that
+      # restates a default) hands them to the Runner: its threshold scores
+      # the answers, and the Report carries the bars for
+      # `assert_meets_pass_gates` and renders its gates. A suite with no
+      # `pass` block, or only a suite-level one that restates the defaults,
+      # runs under the defaults, and its Report carries no bars.
+      #
+      # @param suite [Suite, String, Pathname, Array] a Suite or the YAML path(s) to load
+      # @param models [Array<String>, String, nil] candidate models; nil runs the agent as configured
+      # @param default_provider [String, nil] the provider a bare model name runs under
+      # @param providers [Array<String>] the provider names a `provider/model` may name
+      # @param groups [Array<String>, nil] narrow to these suite groups
+      # @param keys [Array<String>, nil] narrow to these scenario keys
+      # @param include_production_only [Boolean] replay scenarios a local environment cannot answer
+      # @param runner_options [Hash] the rest goes to Runner.new (criteria, judge, available_tools,
+      #   instructions, ...); `pass_thresholds:` replaces the suite's bars, and `threshold:` wins over
+      #   the suite's threshold
+      # @return [Report]
+      def run_eval_suite(suite, models: nil, default_provider: nil, providers: ModelSpec::DEFAULT_PROVIDERS,
+                         groups: nil, keys: nil, include_production_only: false, **runner_options, &replay)
+        raise ArgumentError, "run_eval_suite needs a block that runs the agent for a scenario" unless replay
+
+        suite = Suite.load(*Array(suite)) unless suite.is_a?(Suite)
+        scenarios = suite.scenarios(groups: groups, keys: keys, include_production_only: include_production_only)
+        specs = models.present? ? ModelSpec.parse_all(models, default_provider: default_provider.to_s, providers: providers) : [ AS_CONFIGURED ]
+        unless runner_options.key?(:pass_thresholds) || suite.pass_thresholds == PassThresholds.default
+          runner_options = runner_options.merge(pass_thresholds: suite.pass_thresholds)
+        end
+
+        Runner.new(
+          scenarios: scenarios, models: specs,
+          replay: ->(scenario, spec) { capture_for_spec(replay.call(scenario, spec), spec) },
+          **runner_options
+        ).call
+      end
+
+      # Passes when every model's pass rate reached `at_least` (0.0..1.0).
+      def assert_pass_rate(report, message = nil, at_least:)
+        at_least = at_least.to_f
+        # The fraction itself, not the report's rounded percentage: 1 of 3 meets
+        # `at_least: 1 / 3.0`, which 33.3% would not. A model with no scenarios
+        # falls short.
+        short = report.summary_by_model.select do |_label, stats|
+          scenarios = stats["scenarios"].to_i
+          scenarios.zero? || stats["passed"].to_f / scenarios < at_least
+        end
+        assert short.empty?, message || pass_rate_failure_message(report, short, at_least)
+      end
+
+      # Passes when every gate of the report is met: under `thresholds` when
+      # given, else under the bars the report carries (Report#pass_thresholds),
+      # else the defaults, every group gated and every answer expected to
+      # pass. Fails with GateCheck#message — the unmet gates, then the
+      # failures to read per group — or with `message` when one is given.
+      # Where `assert_pass_rate` holds every model to one bar the test names,
+      # this holds each group to the bar the suite declares for it, and each
+      # model's gated answers to the suite's, leaving ungated groups out.
+      # Unlike `assert_pass_rate`, it takes the message third, after the
+      # thresholds.
+      #
+      # @param report [Report]
+      # @param thresholds [PassThresholds, nil] bars to hold the report to in place of its own
+      # @param message [String, nil] the failure message, in place of the check's
+      # @return [GateCheck]
+      def assert_meets_pass_gates(report, thresholds = nil, message = nil)
+        check = GateCheck.new(report, thresholds)
+        assert check.met?, message || check.message
+        check
+      end
+
+      private
+
+      def capture_for_spec(value, spec)
+        overrides = spec.equal?(AS_CONFIGURED) ? {} : { model: spec.model, provider: spec.provider.presence }
+        value.is_a?(ActiveAgent::Generation) ? Capture.run(value, **overrides) : Capture.run(**overrides) { value }
+      end
+
+      def tool_calls_named(replay, name)
+        replay.tool_calls.select { |call| call["name"] == name.to_s }
+      end
+
+      # The spec a captured replay ran under, for the Result: the provider a
+      # capture was asked for, else the configured service read as a provider
+      # name ("OpenAI" → "openai", "OpenRouter" → "openrouter").
+      def model_spec_for(replay)
+        model = replay.metadata["model"].presence
+        provider = replay.metadata["provider"].presence || replay.metadata["service"].to_s.underscore.delete("_")
+        ModelSpec.new(label: model || "as configured", provider: provider, model: model)
+      end
+
+      def eval_failure_message(result)
+        lines = [ "Eval failed: #{result.fault} — #{result.summary}" ]
+        lines << "Fix: #{result.recommendation}" if result.recommendation.present?
+        lines << "Scores: #{result.scores.map { |key, value| "#{key}=#{value.nil? ? "n/a" : Format.score(value)}" }.join(", ")}" if result.scores.present?
+        lines << replay_note(result.replay).strip
+        lines.join("\n")
+      end
+
+      def pass_rate_failure_message(report, short, at_least)
+        lines = short.map do |label, stats|
+          failing = report.results.select { |result| result.label == label && !result.passed? }
+          detail = failing.map { |result| "#{result.scenario.key} (#{result.fault || result.status})" }.join(", ")
+          "#{label}: #{Format.passes(stats["passed"], stats["scenarios"])} passed, below #{Format.percent(at_least)}; failing: #{detail}"
+        end
+        lines.unshift("Expected every model to pass at least #{Format.percent(at_least)} of the scenarios")
+        lines.join("\n")
+      end
+
+      def replay_note(replay)
+        parts = []
+        parts << "error: #{replay.error}" if replay.errored?
+        parts << "answer: #{Capture.preview(replay.answer.to_s)}" if replay.answer.present?
+        parts << "tools called: #{replay.tool_names.join(", ")}" if replay.tool_calls.any?
+        parts.empty? ? "" : "\n  #{parts.join("\n  ")}"
+      end
+
+      def tools_note(replay)
+        replay.tool_calls.any? ? " (called: #{replay.tool_names.join(", ")})" : " (no tools were called)#{replay_note(replay)}"
+      end
+    end
+
+    if defined?(ActiveSupport::TestCase)
+      # A test case with the eval helpers included, whose tests run only when
+      # evals are switched on (TestHelper.enabled?) and are skipped, with the
+      # command that runs them, otherwise. Inherit from it in test/evals; the
+      # app's own fixtures and helpers apply as in any test.
+      class TestCase < ActiveSupport::TestCase
+        include TestHelper
+
+        setup do
+          skip TestHelper::SKIP_MESSAGE unless TestHelper.enabled?
+        end
+      end
+    end
+  end
+end

data/lib/active_agent/evals.rb

@@ -4,6 +4,7 @@ require "json"
 require "cgi"
 require "yaml"
 require "active_support"
+require "active_support/core_ext/array/wrap"
 require "active_support/core_ext/object/blank"
 require "active_support/core_ext/string/filters"
 require "active_support/core_ext/string/access"
@@ -14,11 +15,13 @@ require "active_support/core_ext/enumerable"
 
 require_relative "evals/scenario"
 require_relative "evals/scenario_parser"
+require_relative "evals/pass_thresholds"
 require_relative "evals/suite"
 require_relative "evals/catalog"
 require_relative "evals/model_spec"
 require_relative "evals/replay"
 require_relative "evals/format"
+require_relative "evals/patterns"
 require_relative "evals/statistics"
 require_relative "evals/comparison"
 require_relative "evals/scorer"
@@ -29,9 +32,11 @@ require_relative "evals/design_tokens"
 require_relative "evals/report_html"
 require_relative "evals/report_markdown"
 require_relative "evals/report"
+require_relative "evals/gate_check"
 require_relative "evals/runner"
 require_relative "evals/correlation"
 require_relative "evals/publisher"
+require_relative "evals/capture"
 
 # Scenario evaluations for agents that answer with tools.
 #
@@ -47,7 +52,10 @@ require_relative "evals/publisher"
 # and rolling everything up per model (Report). Runner ties them together
 # around one callable you supply: given a scenario and a model, run the
 # agent and return a Replay. Correlation is optional plumbing on top: it
-# links the traces a run emits back to the result that caused them.
+# links the traces a run emits back to the result that caused them. Capture
+# runs one ActiveAgent generation into a Replay, and
+# `require "active_agent/evals/test_helper"` adds the Minitest assertions an
+# eval written as a test case uses (TestHelper, TestCase).
 #
 #   scenarios = ActiveAgent::Evals::ScenarioParser.scenarios(pasted_text)
 #   models    = ActiveAgent::Evals::ModelSpec.parse_all(%w[gpt-5-mini qwen3:8b], default_provider: "openai")

data/lib/active_agent/providers/_base_provider.rb

@@ -134,6 +134,7 @@ module ActiveAgent
                     :stream_completion_result,         # Callback (Streams)
                     :stream_finish_reason,             # Callback (Streams)
                     :tools_function,                   # Callback (Tools)
+                    :tool_observer,                    # Callback (Tools, MCP-served calls)
                     :usage_stack,                      # Usage Tracking
                     :stream_usage_index,               # Usage Tracking (Streams)
                     :max_tool_turns, :tool_turns,      # Tool-loop safety
@@ -193,6 +194,7 @@ module ActiveAgent
         self.stream_broadcaster = kwargs.delete(:stream_broadcaster)
         self.streaming          = false
         self.tools_function     = kwargs.delete(:tools_function)
+        self.tool_observer      = kwargs.delete(:tool_observer)
         self.max_tool_turns     = kwargs.delete(:max_tool_turns) || DEFAULT_MAX_TOOL_TURNS
         self.tool_turns         = 0
         self.instrumentation_enabled = kwargs.delete(:instrumentation) != false
@@ -273,6 +275,10 @@ module ActiveAgent
       # Invokes a tool, on an MCP server when one provides it and on the agent's
       # own tool function otherwise.
       #
+      # A call served by an MCP server never reaches the agent's tool function,
+      # so it runs inside the `tool_observer`, when the agent has one, which sees
+      # the call and its result without changing either.
+      #
       # @param name [String] tool name
       # @param kwargs [Hash] tool arguments
       # @return [Object] the tool's result, or an {MCPBridge::ErrorResult} for
@@ -281,13 +287,23 @@ module ActiveAgent
       #   tool needs approval and the provider's tool loop never asked for it
       def call_tool_function(name, **kwargs)
         assert_approval_gate_reached!(name)
-        return mcp_call_tool(name, **kwargs) if mcp_owns_tool?(name)
+        return observe_mcp_tool_call(name, kwargs) { mcp_call_tool(name, **kwargs) } if mcp_owns_tool?(name)
 
         result = isolate_undispatched_tool_call { tools_function.call(name, **kwargs) }
         assert_input_request_supported!(result)
         result
       end
 
+      # @param name [String]
+      # @param arguments [Hash]
+      # @yieldreturn [Object] the tool's result
+      # @return [Object] the block's result
+      def observe_mcp_tool_call(name, arguments, &call)
+        return yield unless tool_observer
+
+        tool_observer.call(name, arguments, &call)
+      end
+
       # @param name [String, nil]
       # @raise [RuntimeError] when service name doesn't match provider
       def assert_service!(name)

data/lib/active_agent/providers/atlas_cloud/options.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require_relative "../open_ai/options"
+
+module ActiveAgent
+  module Providers
+    module AtlasCloud
+      # Configuration options for the Atlas Cloud provider.
+      class Options < ActiveAgent::Providers::OpenAI::Options
+        # Atlas Cloud's OpenAI-compatible endpoint.
+        DEFAULT_BASE_URL = "https://api.atlascloud.ai/v1"
+
+        # @param kwargs [Hash]
+        # @option kwargs [String] :api_key
+        # @option kwargs [String] :base_url (https://api.atlascloud.ai/v1)
+        def initialize(kwargs = {})
+          kwargs = kwargs.deep_symbolize_keys if kwargs.respond_to?(:deep_symbolize_keys)
+
+          super(kwargs.merge(base_url: kwargs[:base_url].presence || DEFAULT_BASE_URL))
+        end
+
+        private
+
+        def resolve_api_key(kwargs)
+          kwargs[:api_key] ||
+            kwargs[:access_token] ||
+            ENV["ATLASCLOUD_API_KEY"]
+        end
+
+        def resolve_organization_id(_kwargs) = nil
+        def resolve_project_id(_kwargs) = nil
+      end
+    end
+  end
+end

data/lib/active_agent/providers/atlas_cloud_provider.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require_relative "_base_provider"
+
+require_gem!(:openai, __FILE__)
+
+require_relative "open_ai_provider"
+require_relative "atlas_cloud/options"
+
+module ActiveAgent
+  module Providers
+    # Provides access to Atlas Cloud's OpenAI-compatible Chat Completions API.
+    #
+    # Atlas Cloud uses provider/model identifiers such as +qwen/qwen3.8-max+.
+    # Request and response handling is shared with the OpenAI Chat provider;
+    # only the endpoint and API-key resolution are provider-specific.
+    class AtlasCloudProvider < OpenAI::ChatProvider
+      # @return [String]
+      def self.service_name
+        "AtlasCloud"
+      end
+
+      # @return [Class]
+      def self.options_klass
+        AtlasCloud::Options
+      end
+
+      # @return [ActiveModel::Type::Value]
+      def self.prompt_request_type
+        OpenAI::Chat::RequestType.new
+      end
+
+      protected
+
+      # @see BaseProvider#api_response_normalize
+      # @param api_response [OpenAI::Models::ChatCompletion]
+      # @return [Hash]
+      def api_response_normalize(api_response)
+        return api_response unless api_response
+
+        OpenAI::Chat::Transforms.gem_to_hash(api_response)
+      end
+    end
+  end
+end

data/lib/active_agent/providers/cheaper_inference/_types.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require_relative "../open_ai/chat/_types"
+require_relative "options"
+
+module ActiveAgent
+  module Providers
+    module CheaperInference
+      # ActiveModel type for casting and serializing Cheaper Inference requests.
+      #
+      # Cheaper Inference is OpenAI-compatible, so requests use the same shape
+      # as the OpenAI Chat API. This delegates entirely to
+      # OpenAI::Chat::RequestType.
+      RequestType = ActiveAgent::Providers::OpenAI::Chat::RequestType
+    end
+  end
+end