activeagent 1.9.6 → 1.9.8

data/README.md

@@ -48,6 +48,9 @@ bundle add openai
 # Anthropic
 bundle add anthropic
 
+# Atlas Cloud (uses OpenAI-compatible API)
+bundle add openai
+
 # Ollama (uses OpenAI-compatible API)
 bundle add openai
 
@@ -60,6 +63,12 @@ bundle add openai
 # Opper (uses OpenAI-compatible API)
 bundle add openai
 
+# Cheaper Inference (uses OpenAI-compatible API)
+bundle add openai
+
+# Runcrate (uses OpenAI-compatible API)
+bundle add openai
+
 # DeepSeek (uses OpenAI-compatible API)
 bundle add openai
 

data/lib/active_agent/base.rb

@@ -336,6 +336,7 @@ module ActiveAgent
         exception_handler:,
         stream_broadcaster:,
         tools_function:,
+        tool_observer:,
         action_name: generation_action_name,
         input_request_resume: @_input_request_resume,
         announce_input_requests: !input_requests_silenced?

data/lib/active_agent/cli/eval_commands.rb

@@ -86,8 +86,8 @@ module ActiveAgent
           p.on("--keys LIST", "Replay only these scenario keys, comma-separated") { |v| arguments[:keys] = list_option(v) }
           p.on("--group GROUP", "Replay only this scenario group") { |v| arguments[:group] = v }
           p.on("--sandbox ID", "Replay against this ready checkout sandbox") { |v| arguments[:sandbox_id] = v }
-          p.on("--wait", "Wait for the run to finish") { wait = true }
-          p.on("--min-pass-rate RATE", "Fail (exit #{EXIT_GATE}) below this pass rate: 0.8 or 80%; implies --wait") do |v|
+          p.on("--wait", "Wait for the run to finish; fail (exit #{EXIT_GATE}) when it failed or missed a pass gate") { wait = true }
+          p.on("--min-pass-rate RATE", "Also fail below this whole-run pass rate: 0.8 or 80%; implies --wait") do |v|
             min_pass_rate = pass_rate_option(v)
             wait = true
           end
@@ -138,11 +138,12 @@ module ActiveAgent
 
       # Posts a saved report to the collector. FILE is a version-1 envelope
       # (what `evals report --format json` writes) or a bare Report#to_h
-      # hash, whose identities then come from the options.
+      # hash, whose identities then come from the options; the publisher
+      # reads the run_id from the report's metadata when neither names one.
       def evals_import(argv)
         identities = {}
         argv = parse(argv, "evals import FILE [--run-id ID --source NAME --agent NAME --suite NAME]") do |p|
-          p.on("--run-id ID", "The report's run_id (kept across retries)") { |v| identities["run_id"] = v }
+          p.on("--run-id ID", "The report's run_id, kept across retries; by default the one its metadata records") { |v| identities["run_id"] = v }
           p.on("--source NAME", "The application that ran it") { |v| identities["source"] = v }
           p.on("--agent NAME", "The agent's name") { |v| identities["agent_name"] = v }
           p.on("--suite NAME", "The suite's name") { |v| identities["suite"] = v }
@@ -156,7 +157,6 @@ module ActiveAgent
           raise Error.new("#{path} is a version #{envelope["version"].inspect} envelope; this CLI sends version 1", exit_code: EXIT_USAGE)
         end
         envelope = envelope.merge(identities)
-        envelope["run_id"] ||= envelope["report"].is_a?(Hash) ? envelope["report"].dig("metadata", "run_id") : nil
 
         receipt = client.publisher.call(report: envelope["report"], run_id: envelope["run_id"], source: envelope["source"],
                                         agent_name: envelope["agent_name"], suite: envelope["suite"])
@@ -199,18 +199,28 @@ module ActiveAgent
         end
       end
 
+      # A finished run fails the gate when it failed, when its whole pass
+      # rate is under --min-pass-rate, or when it was held to per-group pass
+      # gates (the run's "gates", from the suite's or catalog set's `pass`
+      # bars) and the dashboard says it did not meet every one. A run
+      # without gates is gated on its status and rate alone, as it was.
       def gate(payload, min_pass_rate)
         run = payload.fetch("run", {})
         rate = pass_rate(run)
         failed = run["status"] == "failed"
         below = min_pass_rate && (rate.nil? || rate < min_pass_rate)
-        verdict = failed || below ? "fail" : "pass"
+        gates = run_gates(run)
+        missed = !gates.nil? && run["gates_met"] != true
+        verdict = failed || below || missed ? "fail" : "pass"
         failures = payload.fetch("results", [])
+        summary = { "pass_rate" => rate, "min_pass_rate" => min_pass_rate }
+        summary["gates"] = gates if gates
 
-        emit(payload.merge("pass_rate" => rate, "min_pass_rate" => min_pass_rate, "verdict" => verdict)) do
+        emit(payload.merge(summary).merge("verdict" => verdict)) do
           @out.puts "Run #{run["id"]} #{run["status"]}: #{passed(run) || "no results"}" \
                     "#{rate ? format(" (%.1f%%)", rate * 100) : ""}#{min_pass_rate ? format(", threshold %.1f%%", min_pass_rate * 100) : ""}."
           @out.puts "Error: #{run["error_message"]}" if run["error_message"]
+          print_gates(gates, missed) if gates
           unless failures.empty?
             @out.puts "Failed results:"
             table(failures, {
@@ -227,6 +237,39 @@ module ActiveAgent
         verdict == "pass" ? EXIT_SUCCESS : EXIT_GATE
       end
 
+      # The run's per-group pass gates as the dashboard worked them out
+      # (EvaluationRun#gates), or nil for a run held to none.
+      def run_gates(run)
+        rows = run["gates"]
+        rows.select { |row| row.is_a?(Hash) } if rows.is_a?(Array)
+      end
+
+      def print_gates(gates, missed)
+        unmet = gates.reject { |gate| gate["met"] == true }
+        unless missed
+          @out.puts "Gates: every gate met (#{gates.size})."
+          return
+        end
+
+        @out.puts(unmet.empty? ? "Gates: no gate was reached." : "Gates not met:")
+        unmet.each { |gate| @out.puts "  #{gate_sentence(gate)}" }
+      end
+
+      # A gate the run missed, as ActiveAgent::Evals::Report::Gate#to_s
+      # reads it: "Orders · gpt-5-mini: 1/2 (50%) passed, below 100%".
+      def gate_sentence(gate)
+        passed = gate["passed"].to_i
+        total = gate["total"].to_i
+        passes = total.positive? ? "#{passed}/#{total} (#{whole_percent(passed.to_f / total)})" : "no results"
+        label = cell(gate["label"]) unless gate["label"].to_s.empty?
+        "#{cell(gate["name"] || gate["group"] || "All")}#{label ? " · #{label}" : ""}: #{passes} passed, below #{whole_percent(gate["min_pass_rate"])}"
+      end
+
+      def whole_percent(fraction)
+        value = Float(fraction, exception: false)
+        value ? "#{(value * 100).round}%" : "?"
+      end
+
       def pass_rate(run)
         evaluated = run["samples_evaluated"].to_i
         evaluated.positive? ? run["samples_passed"].to_f / evaluated : nil

data/lib/active_agent/concerns/tooling.rb

@@ -26,5 +26,16 @@ module ActiveAgent
         raise_without_input_request_secrets(error)
       end
     end
+
+    # A callback the provider runs the calls it serves from an MCP server
+    # around, which never reach {#tools_function}. It receives the tool name,
+    # its arguments and a block that makes the call, and returns the block's
+    # result. Nil unless the agent defines one, to audit its MCP calls say,
+    # or a host that records a generation's tool calls sets it on the
+    # instance. Evals::Capture keeps an agent's own observer and records the
+    # call that observer makes.
+    #
+    # @return [Proc, nil]
+    def tool_observer = nil
   end
 end

data/lib/active_agent/evals/capture.rb

@@ -0,0 +1,286 @@
+# frozen_string_literal: true
+
+module ActiveAgent
+  module Evals
+    # Runs one agent generation and records what the agent did, as a Replay:
+    # the answer, every tool it called (arguments, result or error, timing),
+    # how long the generation took and the tokens it used. It is the glue
+    # between `MyAgent.with(...).action` and the Runner's `replay` callable,
+    # and what TestHelper#run_eval returns.
+    #
+    #   replay = ActiveAgent::Evals::Capture.run(SupportAgent.with(question: "Where is order 4821?").help)
+    #   replay.answer       # => "Order 4821 shipped on Monday."
+    #   replay.tool_names   # => ["lookup_order"]
+    #   replay.tool_calls   # => [{ "name" => "lookup_order", "arguments" => { "number" => "4821" }, ... }]
+    #
+    # Tool calls are captured on the agent instance's `tools_function`, the
+    # proc every provider calls to run a tool, which is also where the
+    # telemetry tool spans hang. A tool served by a client-side MCP server
+    # (`mcp_strategy: :client`, or a provider with no MCP of its own) is
+    # dispatched before that proc, so the provider reports it through the
+    # agent's `tool_observer` instead, with a failure the server reported
+    # marked as one. An observer the agent defines itself still runs, around
+    # the call that is recorded. A tool a provider runs on its own side
+    # (Anthropic's or OpenAI's hosted MCP) never reaches this process and is
+    # not recorded.
+    #
+    # `model:` and `provider:` replay the same action under another model,
+    # the way a comparison run does. A model is an option on the agent
+    # instance. A provider is a subclass of the agent configured with
+    # `generate_with provider, model:`, named as its parent so its templates
+    # still resolve; the parent's `instructions:` option is carried, and the
+    # rest of its options follow `generate_with`'s own rule for a subclass
+    # (kept under the same service, dropped under another).
+    #
+    # A block runs the agent some other way; it may return a PromptResponse,
+    # a Replay, a Hash of Replay attributes, or the answer as a String. Tool
+    # calls are then whatever the block reports, since nothing wrapped the
+    # agent.
+    #
+    # An exception becomes an errored Replay, as it does under Runner, so a
+    # scenario that raised is scored and diagnosed like any other;
+    # `raise_errors: true` re-raises it instead, for the backtrace.
+    class Capture
+      # How much of a tool's result a tool call keeps as its "detail".
+      DETAIL_LIMIT = 500
+
+      # @param generation [ActiveAgent::Generation, nil] the action to run; nil with a block
+      # @param model [String, nil] the model to run under instead of the agent's own
+      # @param provider [Symbol, String, nil] the provider to run under instead of the agent's own
+      # @param metadata [Hash] carried onto the Replay's metadata
+      # @param raise_errors [Boolean] re-raise instead of returning an errored Replay
+      # @return [Replay]
+      def self.run(generation = nil, model: nil, provider: nil, metadata: {}, raise_errors: false, &block)
+        new(generation, model: model, provider: provider, metadata: metadata, raise_errors: raise_errors).call(&block)
+      end
+
+      def initialize(generation = nil, model: nil, provider: nil, metadata: {}, raise_errors: false)
+        @generation = generation
+        @model = model
+        @provider = provider
+        @metadata = (metadata || {}).to_h.transform_keys(&:to_s)
+        @raise_errors = raise_errors
+        @tool_calls = []
+        @agent = nil
+      end
+
+      def call(&block)
+        raise ArgumentError, "give Capture a generation to run, or a block that runs the agent" unless @generation || block
+
+        started = self.class.monotonic_ms
+        value =
+          begin
+            @generation ? run_generation : block.call
+          rescue StandardError => e
+            raise if @raise_errors
+
+            return Replay.failed(e, tool_calls: @tool_calls, duration_ms: self.class.elapsed_ms(started), metadata: base_metadata)
+          end
+
+        replay_for(value, duration_ms: self.class.elapsed_ms(started))
+      end
+
+      # The generation with `model:` and `provider:` applied: the one given
+      # when neither is set, otherwise a copy that instantiates the retargeted
+      # agent. Public so a host that runs generations itself can retarget the
+      # same way.
+      def self.retarget(generation, model: nil, provider: nil)
+        return generation unless model || provider
+
+        copy = generation.dup
+        # The memoized agent, if the generation was already touched, belongs
+        # to the original class and action.
+        copy.instance_variable_set(:@agent, nil)
+        copy.agent_class = provider ? subclass_under(generation.agent_class, provider, model) : generation.agent_class
+        copy
+      end
+
+      # A subclass of `agent_class` that generates with `provider` (and
+      # `model`, when given), named as its parent so `agent_name`, the view
+      # prefixes and the telemetry span names stay the parent's.
+      def self.subclass_under(agent_class, provider, model)
+        parent_name = agent_class.name
+        parent_options = agent_class.prompt_options || {}
+
+        Class.new(agent_class).tap do |klass|
+          klass.define_singleton_method(:name) { parent_name }
+          klass.generate_with(provider, **{ model: model }.compact)
+          # generate_with never inherits a parent's instructions. A replay is
+          # the same agent under another model, so they come along.
+          klass.prompt_options[:instructions] = parent_options[:instructions] if parent_options.key?(:instructions)
+        end
+      end
+
+      # Runs one tool call and appends its record to +calls+: the name, the
+      # arguments, a preview of the result and the time it took. A call that
+      # raises is recorded as an error and re-raised. So is one an MCP server
+      # reported as failed: it comes back as an MCPBridge::ErrorResult rather
+      # than an exception, so the model can recover, and is returned to the
+      # provider unchanged.
+      #
+      # @api private
+      # @return [Object] what the call returned
+      def self.record_call(calls, name, arguments)
+        call = { "name" => name.to_s }
+        call["arguments"] = jsonable(arguments) unless arguments.nil?
+        started = monotonic_ms
+
+        begin
+          result = yield
+          if mcp_error?(result)
+            call["error"] = true
+            call["detail"] = preview(result.message)
+          else
+            call["detail"] = preview(result)
+          end
+          result
+        rescue StandardError => e
+          call["error"] = true
+          call["detail"] = "#{e.class}: #{e.message}"
+          raise
+        ensure
+          call["duration_ms"] = elapsed_ms(started)
+          calls << call
+        end
+      end
+
+      def self.mcp_error?(result)
+        defined?(ActiveAgent::Providers::MCPBridge::ErrorResult) && result.is_a?(ActiveAgent::Providers::MCPBridge::ErrorResult)
+      end
+      private_class_method :mcp_error?
+
+      def self.monotonic_ms
+        Process.clock_gettime(Process::CLOCK_MONOTONIC, :millisecond)
+      end
+
+      def self.elapsed_ms(started)
+        monotonic_ms - started
+      end
+
+      # Tool arguments and results as JSON would carry them (string keys,
+      # plain values), so a recorded call compares with what a test writes.
+      def self.jsonable(value)
+        JSON.parse(JSON.generate(value))
+      rescue StandardError
+        value.to_s
+      end
+
+      def self.preview(value)
+        text = value.is_a?(String) ? value : (JSON.generate(value) rescue value.to_s)
+        text.length > DETAIL_LIMIT ? "#{text[0, DETAIL_LIMIT]}…" : text
+      end
+
+      private
+
+      def run_generation
+        generation = self.class.retarget(@generation, model: @model, provider: @provider)
+        # Generation#agent is private: nothing outside should touch the agent
+        # before prompt_later. A capture runs it now, and needs the instance
+        # to hook its tool dispatch, so it reaches in on purpose.
+        @agent = generation.send(:agent)
+        @agent.prompt_options[:model] = @model if @model && !@provider
+        record_tool_calls(@agent)
+        generation.generate_now
+      end
+
+      # Wraps the agent's tools_function so each dispatch lands in
+      # @tool_calls. The singleton method takes precedence over the telemetry
+      # module prepended on the class, and `base` is that module's wrapped
+      # proc, so tool spans still fire.
+      #
+      # A tool served by a client-side MCP server never reaches tools_function:
+      # the provider runs that call inside the agent's `tool_observer`, which
+      # records it the same way. The two paths are disjoint, so a call is
+      # recorded once, and the observer sees the call without making it again.
+      #
+      # An observer the agent already has is kept, as tools_function's `base`
+      # is: it runs around the call this records, so it sees the same name,
+      # arguments, result and exception, and a call it does not make is not
+      # recorded as made.
+      def record_tool_calls(agent)
+        calls = @tool_calls
+        base = agent.tools_function
+        observer = agent.tool_observer
+        capture = self.class
+
+        agent.define_singleton_method(:tools_function) do
+          proc do |name, *args, **kwargs|
+            arguments = kwargs.presence || (args.length == 1 ? args.first : args.presence)
+            capture.record_call(calls, name, arguments) { base.call(name, *args, **kwargs) }
+          end
+        end
+
+        agent.define_singleton_method(:tool_observer) do
+          record = proc { |name, arguments, &call| capture.record_call(calls, name, arguments.presence, &call) }
+          next record unless observer
+
+          proc { |name, arguments, &call| observer.call(name, arguments) { record.call(name, arguments, &call) } }
+        end
+      end
+
+      def replay_for(value, duration_ms:)
+        case value
+        when Replay
+          attributes = value.to_h.transform_keys(&:to_sym).merge(
+            tool_calls: value.tool_calls.presence || @tool_calls,
+            duration_ms: value.duration_ms || duration_ms,
+            metadata: base_metadata.merge(value.metadata)
+          )
+          Replay.new(**attributes)
+        when Hash
+          replay_for(Replay.new(**value.to_h.transform_keys(&:to_sym)), duration_ms: duration_ms)
+        when String, nil
+          Replay.new(answer: value, tool_calls: @tool_calls, duration_ms: duration_ms, metadata: base_metadata)
+        else
+          unless value.respond_to?(:messages)
+            raise ArgumentError, "expected the agent run to return a PromptResponse, a Replay, a Hash or a String, got #{value.class}"
+          end
+
+          replay_from_response(value, duration_ms: duration_ms)
+        end
+      end
+
+      def replay_from_response(response, duration_ms:)
+        usage = response.respond_to?(:usage) ? response.usage : nil
+        messages = Array(response.messages)
+
+        Replay.new(
+          answer: final_answer(messages),
+          tool_calls: @tool_calls,
+          duration_ms: duration_ms,
+          input_tokens: usage&.input_tokens,
+          output_tokens: usage&.output_tokens,
+          metadata: base_metadata.merge({ "prompt" => user_prompt(messages) }.compact)
+        )
+      end
+
+      # The agent's last assistant turn. A tool loop leaves tool calls and
+      # results before it; the final text comes last.
+      def final_answer(messages)
+        message = messages.reverse.find { |candidate| candidate.respond_to?(:role) && candidate.role.to_s == "assistant" }
+        return nil unless message
+
+        message.respond_to?(:text) ? message.text : message.content.to_s
+      end
+
+      # What the user asked, for the scenario a test scores the replay
+      # against: the first user turn, when it is plain text.
+      def user_prompt(messages)
+        message = messages.find { |candidate| candidate.respond_to?(:role) && candidate.role.to_s == "user" }
+        content = message&.content
+        content if content.is_a?(String) && content.present?
+      end
+
+      def base_metadata
+        options = @agent&.prompt_options || {}
+        {
+          "agent" => @agent&.class&.name,
+          "action" => @generation&.action_name&.to_s,
+          "provider" => @provider&.to_s,
+          "service" => options[:service]&.to_s,
+          "model" => (@model || options[:model])&.to_s
+        }.compact.merge(@metadata)
+      end
+    end
+  end
+end