activeagent 1.9.6 → 1.9.7

data/lib/active_agent/evals/capture.rb

@@ -0,0 +1,275 @@
+# frozen_string_literal: true
+
+module ActiveAgent
+  module Evals
+    # Runs one agent generation and records what the agent did, as a Replay:
+    # the answer, every tool it called (arguments, result or error, timing),
+    # how long the generation took and the tokens it used. It is the glue
+    # between `MyAgent.with(...).action` and the Runner's `replay` callable,
+    # and what TestHelper#run_eval returns.
+    #
+    #   replay = ActiveAgent::Evals::Capture.run(SupportAgent.with(question: "Where is order 4821?").help)
+    #   replay.answer       # => "Order 4821 shipped on Monday."
+    #   replay.tool_names   # => ["lookup_order"]
+    #   replay.tool_calls   # => [{ "name" => "lookup_order", "arguments" => { "number" => "4821" }, ... }]
+    #
+    # Tool calls are captured on the agent instance's `tools_function`, the
+    # proc every provider calls to run a tool, which is also where the
+    # telemetry tool spans hang. A tool served by a client-side MCP server
+    # (`mcp_strategy: :client`, or a provider with no MCP of its own) is
+    # dispatched before that proc, so the provider reports it through the
+    # agent's `tool_observer` instead, with a failure the server reported
+    # marked as one. A tool a provider runs on its own side (Anthropic's or
+    # OpenAI's hosted MCP) never reaches this process and is not recorded.
+    #
+    # `model:` and `provider:` replay the same action under another model,
+    # the way a comparison run does. A model is an option on the agent
+    # instance. A provider is a subclass of the agent configured with
+    # `generate_with provider, model:`, named as its parent so its templates
+    # still resolve; the parent's `instructions:` option is carried, and the
+    # rest of its options follow `generate_with`'s own rule for a subclass
+    # (kept under the same service, dropped under another).
+    #
+    # A block runs the agent some other way; it may return a PromptResponse,
+    # a Replay, a Hash of Replay attributes, or the answer as a String. Tool
+    # calls are then whatever the block reports, since nothing wrapped the
+    # agent.
+    #
+    # An exception becomes an errored Replay, as it does under Runner, so a
+    # scenario that raised is scored and diagnosed like any other;
+    # `raise_errors: true` re-raises it instead, for the backtrace.
+    class Capture
+      # How much of a tool's result a tool call keeps as its "detail".
+      DETAIL_LIMIT = 500
+
+      # @param generation [ActiveAgent::Generation, nil] the action to run; nil with a block
+      # @param model [String, nil] the model to run under instead of the agent's own
+      # @param provider [Symbol, String, nil] the provider to run under instead of the agent's own
+      # @param metadata [Hash] carried onto the Replay's metadata
+      # @param raise_errors [Boolean] re-raise instead of returning an errored Replay
+      # @return [Replay]
+      def self.run(generation = nil, model: nil, provider: nil, metadata: {}, raise_errors: false, &block)
+        new(generation, model: model, provider: provider, metadata: metadata, raise_errors: raise_errors).call(&block)
+      end
+
+      def initialize(generation = nil, model: nil, provider: nil, metadata: {}, raise_errors: false)
+        @generation = generation
+        @model = model
+        @provider = provider
+        @metadata = (metadata || {}).to_h.transform_keys(&:to_s)
+        @raise_errors = raise_errors
+        @tool_calls = []
+        @agent = nil
+      end
+
+      def call(&block)
+        raise ArgumentError, "give Capture a generation to run, or a block that runs the agent" unless @generation || block
+
+        started = self.class.monotonic_ms
+        value =
+          begin
+            @generation ? run_generation : block.call
+          rescue StandardError => e
+            raise if @raise_errors
+
+            return Replay.failed(e, tool_calls: @tool_calls, duration_ms: self.class.elapsed_ms(started), metadata: base_metadata)
+          end
+
+        replay_for(value, duration_ms: self.class.elapsed_ms(started))
+      end
+
+      # The generation with `model:` and `provider:` applied: the one given
+      # when neither is set, otherwise a copy that instantiates the retargeted
+      # agent. Public so a host that runs generations itself can retarget the
+      # same way.
+      def self.retarget(generation, model: nil, provider: nil)
+        return generation unless model || provider
+
+        copy = generation.dup
+        # The memoized agent, if the generation was already touched, belongs
+        # to the original class and action.
+        copy.instance_variable_set(:@agent, nil)
+        copy.agent_class = provider ? subclass_under(generation.agent_class, provider, model) : generation.agent_class
+        copy
+      end
+
+      # A subclass of `agent_class` that generates with `provider` (and
+      # `model`, when given), named as its parent so `agent_name`, the view
+      # prefixes and the telemetry span names stay the parent's.
+      def self.subclass_under(agent_class, provider, model)
+        parent_name = agent_class.name
+        parent_options = agent_class.prompt_options || {}
+
+        Class.new(agent_class).tap do |klass|
+          klass.define_singleton_method(:name) { parent_name }
+          klass.generate_with(provider, **{ model: model }.compact)
+          # generate_with never inherits a parent's instructions. A replay is
+          # the same agent under another model, so they come along.
+          klass.prompt_options[:instructions] = parent_options[:instructions] if parent_options.key?(:instructions)
+        end
+      end
+
+      # Runs one tool call and appends its record to +calls+: the name, the
+      # arguments, a preview of the result and the time it took. A call that
+      # raises is recorded as an error and re-raised. So is one an MCP server
+      # reported as failed: it comes back as an MCPBridge::ErrorResult rather
+      # than an exception, so the model can recover, and is returned to the
+      # provider unchanged.
+      #
+      # @api private
+      # @return [Object] what the call returned
+      def self.record_call(calls, name, arguments)
+        call = { "name" => name.to_s }
+        call["arguments"] = jsonable(arguments) unless arguments.nil?
+        started = monotonic_ms
+
+        begin
+          result = yield
+          if mcp_error?(result)
+            call["error"] = true
+            call["detail"] = preview(result.message)
+          else
+            call["detail"] = preview(result)
+          end
+          result
+        rescue StandardError => e
+          call["error"] = true
+          call["detail"] = "#{e.class}: #{e.message}"
+          raise
+        ensure
+          call["duration_ms"] = elapsed_ms(started)
+          calls << call
+        end
+      end
+
+      def self.mcp_error?(result)
+        defined?(ActiveAgent::Providers::MCPBridge::ErrorResult) && result.is_a?(ActiveAgent::Providers::MCPBridge::ErrorResult)
+      end
+      private_class_method :mcp_error?
+
+      def self.monotonic_ms
+        Process.clock_gettime(Process::CLOCK_MONOTONIC, :millisecond)
+      end
+
+      def self.elapsed_ms(started)
+        monotonic_ms - started
+      end
+
+      # Tool arguments and results as JSON would carry them (string keys,
+      # plain values), so a recorded call compares with what a test writes.
+      def self.jsonable(value)
+        JSON.parse(JSON.generate(value))
+      rescue StandardError
+        value.to_s
+      end
+
+      def self.preview(value)
+        text = value.is_a?(String) ? value : (JSON.generate(value) rescue value.to_s)
+        text.length > DETAIL_LIMIT ? "#{text[0, DETAIL_LIMIT]}…" : text
+      end
+
+      private
+
+      def run_generation
+        generation = self.class.retarget(@generation, model: @model, provider: @provider)
+        # Generation#agent is private: nothing outside should touch the agent
+        # before prompt_later. A capture runs it now, and needs the instance
+        # to hook its tool dispatch, so it reaches in on purpose.
+        @agent = generation.send(:agent)
+        @agent.prompt_options[:model] = @model if @model && !@provider
+        record_tool_calls(@agent)
+        generation.generate_now
+      end
+
+      # Wraps the agent's tools_function so each dispatch lands in
+      # @tool_calls. The singleton method takes precedence over the telemetry
+      # module prepended on the class, and `base` is that module's wrapped
+      # proc, so tool spans still fire.
+      #
+      # A tool served by a client-side MCP server never reaches tools_function:
+      # the provider runs that call inside the agent's `tool_observer`, which
+      # records it the same way. The two paths are disjoint, so a call is
+      # recorded once, and the observer sees the call without making it again.
+      def record_tool_calls(agent)
+        calls = @tool_calls
+        base = agent.tools_function
+        capture = self.class
+
+        agent.define_singleton_method(:tools_function) do
+          proc do |name, *args, **kwargs|
+            arguments = kwargs.presence || (args.length == 1 ? args.first : args.presence)
+            capture.record_call(calls, name, arguments) { base.call(name, *args, **kwargs) }
+          end
+        end
+
+        agent.define_singleton_method(:tool_observer) do
+          proc { |name, arguments, &call| capture.record_call(calls, name, arguments.presence, &call) }
+        end
+      end
+
+      def replay_for(value, duration_ms:)
+        case value
+        when Replay
+          attributes = value.to_h.transform_keys(&:to_sym).merge(
+            tool_calls: value.tool_calls.presence || @tool_calls,
+            duration_ms: value.duration_ms || duration_ms,
+            metadata: base_metadata.merge(value.metadata)
+          )
+          Replay.new(**attributes)
+        when Hash
+          replay_for(Replay.new(**value.to_h.transform_keys(&:to_sym)), duration_ms: duration_ms)
+        when String, nil
+          Replay.new(answer: value, tool_calls: @tool_calls, duration_ms: duration_ms, metadata: base_metadata)
+        else
+          unless value.respond_to?(:messages)
+            raise ArgumentError, "expected the agent run to return a PromptResponse, a Replay, a Hash or a String, got #{value.class}"
+          end
+
+          replay_from_response(value, duration_ms: duration_ms)
+        end
+      end
+
+      def replay_from_response(response, duration_ms:)
+        usage = response.respond_to?(:usage) ? response.usage : nil
+        messages = Array(response.messages)
+
+        Replay.new(
+          answer: final_answer(messages),
+          tool_calls: @tool_calls,
+          duration_ms: duration_ms,
+          input_tokens: usage&.input_tokens,
+          output_tokens: usage&.output_tokens,
+          metadata: base_metadata.merge({ "prompt" => user_prompt(messages) }.compact)
+        )
+      end
+
+      # The agent's last assistant turn. A tool loop leaves tool calls and
+      # results before it; the final text comes last.
+      def final_answer(messages)
+        message = messages.reverse.find { |candidate| candidate.respond_to?(:role) && candidate.role.to_s == "assistant" }
+        return nil unless message
+
+        message.respond_to?(:text) ? message.text : message.content.to_s
+      end
+
+      # What the user asked, for the scenario a test scores the replay
+      # against: the first user turn, when it is plain text.
+      def user_prompt(messages)
+        message = messages.find { |candidate| candidate.respond_to?(:role) && candidate.role.to_s == "user" }
+        content = message&.content
+        content if content.is_a?(String) && content.present?
+      end
+
+      def base_metadata
+        options = @agent&.prompt_options || {}
+        {
+          "agent" => @agent&.class&.name,
+          "action" => @generation&.action_name&.to_s,
+          "provider" => @provider&.to_s,
+          "service" => options[:service]&.to_s,
+          "model" => (@model || options[:model])&.to_s
+        }.compact.merge(@metadata)
+      end
+    end
+  end
+end

data/lib/active_agent/evals/catalog.rb

@@ -6,8 +6,9 @@ module ActiveAgent
   module Evals
     # A catalog of evaluation scenarios: the products an app evaluates, each
     # with the named sets of scenarios it is tested with. One YAML document,
-    # or several layered like Suite's: a later document's product, set or
-    # scenario with a key already present replaces it, a new key is appended.
+    # or several layered like Suite's: a later document's product, set,
+    # scenario or criterion with a key already present replaces it, a new key
+    # is appended.
     #
     #   catalog: support_desk
     #   name: Support Desk
@@ -16,11 +17,14 @@ module ActiveAgent
     #     - key: triage
     #       name: Triage agent
     #       agent: TriageAgent            # the agent class, or a dashboard agent's name
+    #       criteria: { shared: [] }      # optional; named criteria lists, as a Suite's
+    #       pass: { min_pass_rate: 0.8 }  # optional; the product's bars (PassThresholds)
     #       sets:
     #         - key: smoke
     #           name: Smoke
     #           judge: { kind: rules }    # optional; an evaluation's judge_kind and judge_model
     #           criteria: []              # optional; an evaluation's criteria
+    #           pass: { gated: false }    # optional; the set's own bars
     #           scenarios:
     #             - key: refund_request
     #               prompt: A customer asks for a refund on order 1042.
@@ -47,12 +51,12 @@ module ActiveAgent
 
       # The keys a product, set or scenario may carry. Anything else is kept
       # under `metadata` so a document round-trips.
-      PRODUCT_KEYS = %w[key name description agent sets].freeze
-      SET_KEYS = %w[key name description judge criteria scenarios].freeze
-      SCENARIO_KEYS = %w[key prompt expect expectations tools contains not_contains notes tags params production_only position].freeze
+      PRODUCT_KEYS = %w[key name description agent criteria pass sets].freeze
+      SET_KEYS = %w[key name description judge criteria pass scenarios].freeze
+      SCENARIO_KEYS = Scenario::DOCUMENT_KEYS
       CATALOG_KEYS = %w[catalog name description products suite groups].freeze
 
-      Product = Struct.new(:key, :name, :description, :agent, :metadata, :sets, :position, keyword_init: true) do
+      Product = Struct.new(:key, :name, :description, :agent, :criteria, :pass, :metadata, :sets, :position, keyword_init: true) do
         def set(key)
           sets.find { |set| set.key == key.to_s }
         end
@@ -62,7 +66,7 @@ module ActiveAgent
         end
       end
 
-      Set = Struct.new(:key, :name, :description, :judge, :criteria, :metadata, :scenarios, :position, keyword_init: true)
+      Set = Struct.new(:key, :name, :description, :judge, :criteria, :pass, :metadata, :scenarios, :position, keyword_init: true)
 
       attr_reader :key, :name, :description, :metadata, :products
 
@@ -114,14 +118,21 @@ module ActiveAgent
       # Scenarios across the catalog, narrowed by product key, set key,
       # scenario keys, or any of them; `production_only` scenarios are
       # dropped unless `include_production_only` is true. Each scenario's
-      # group is its set's key and its group_name the set's name.
-      def scenarios(product: nil, set: nil, keys: nil, include_production_only: true)
+      # group is its set's key and its group_name the set's name. With
+      # `strict`, every key in `keys` must name a scenario of the catalog (see
+      # Suite.select), and a key outside the chosen product or set selects
+      # nothing.
+      def scenarios(product: nil, set: nil, keys: nil, include_production_only: true, strict: false)
         selected = products
         selected = selected.select { |entry| entry.key == product.to_s } if product.present?
         selected = selected.flat_map(&:sets)
         selected = selected.select { |entry| entry.key == set.to_s } if set.present?
         selected = selected.flat_map(&:scenarios)
-        selected = selected.select { |scenario| Array(keys).map(&:to_s).include?(scenario.key) } if keys.present?
+        if strict
+          selected = Suite.select(selected, keys: keys, known: all_scenarios, name: key)
+        elsif keys.present?
+          selected = selected.select { |scenario| Array(keys).map(&:to_s).include?(scenario.key) }
+        end
         selected = selected.reject(&:production_only?) unless include_production_only
         selected
       end
@@ -136,16 +147,24 @@ module ActiveAgent
 
       # One product's sets as a Suite (every set a group), or one set alone,
       # for code that takes a suite: `Runner.new(scenarios: catalog.suite_for(:triage).scenarios, ...)`.
+      # The product's `pass` is the suite's and each set's its group's.
       def suite_for(product_key, set_key = nil)
         found = product(product_key)
         sets = set_key ? [ set(product_key, set_key) ] : found.sets
         Suite.new([ {
           "suite" => [ key, found.key, set_key ].compact.join("/"),
           "description" => found.description || description,
+          "criteria" => found.criteria,
+          "pass" => found.pass,
           "groups" => sets.map { |entry| set_document(entry) }
         } ])
       end
 
+      # The bars a product, or one of its sets, declares (PassThresholds).
+      def pass_thresholds(product_key, set_key = nil)
+        suite_for(product_key, set_key).pass_thresholds
+      end
+
       # The canonical document: keys in a fixed order, nothing empty, so the
       # same content always reads the same.
       def to_h
@@ -182,8 +201,10 @@ module ActiveAgent
             "key" => product_key,
             "name" => document["name"].presence || product_key,
             "description" => document["description"],
+            "criteria" => document["criteria"],
+            "pass" => document["pass"],
             "sets" => Array(document["groups"])
-          } ]
+          }.compact ]
         }.compact
       end
 
@@ -196,16 +217,70 @@ module ActiveAgent
             existing = products.find { |product| product["key"] == incoming["key"].to_s }
             if existing
               %w[name description agent].each { |field| existing[field] = incoming[field] if incoming[field].present? }
+              merge_pass(existing, incoming["pass"], "product #{existing['key'].inspect}")
               existing.merge!(incoming.except(*PRODUCT_KEYS))
+              merge_criteria_lists(existing, incoming)
               merge_sets(existing, Array(incoming["sets"]))
             else
-              products << incoming.merge("key" => incoming["key"].to_s, "sets" => [])
+              products << incoming.merge("key" => incoming["key"].to_s, "criteria" => {}, "sets" => [])
+                .tap { |product| merge_criteria_lists(product, incoming) }
                 .tap { |product| merge_sets(product, Array(incoming["sets"])) }
             end
           end
         end
       end
 
+      # Layers a product document's named criteria lists over the product's:
+      # by name, and within a name by criterion key.
+      def merge_criteria_lists(product, incoming)
+        criteria_lists(incoming).each do |list_name, list|
+          merge_criteria(product["criteria"][list_name] ||= [], list, "list #{list_name.inspect} of product #{product['key'].inspect}")
+        end
+      end
+
+      # A product document's `criteria` as `name => list`; a bare list is the
+      # one list named "default", and an empty one declares no list.
+      def criteria_lists(product)
+        value = product["criteria"]
+        return {} if value.nil? || value == []
+
+        value = { "default" => value } if value.is_a?(Array)
+        unless value.is_a?(Hash)
+          raise InvalidDocument, "the criteria of product #{product['key'].inspect} are not a list or a mapping of lists"
+        end
+
+        value.to_h do |list_name, list|
+          raise InvalidDocument, "criteria list #{list_name.inspect} of product #{product['key'].inspect} is not a list" unless list.is_a?(Array)
+
+          [ list_name.to_s, list ]
+        end
+      end
+
+      # Layers `incoming` criteria over `existing` by key: a criterion with a
+      # key already present replaces it in place and a new key is appended;
+      # an empty or missing `incoming` declares none. A criterion without a
+      # key is appended once, when `allow_keyless` is true, and refused
+      # otherwise.
+      def merge_criteria(existing, incoming, owner, allow_keyless: false)
+        return if incoming.blank?
+        raise InvalidDocument, "the criteria of #{owner} are not a list" unless incoming.is_a?(Array)
+
+        incoming.each do |criterion|
+          raise InvalidDocument, "a criterion in #{owner} is not a mapping" unless criterion.is_a?(Hash)
+
+          criterion = criterion.deep_stringify_keys
+          if criterion["key"].present?
+            criterion["key"] = criterion["key"].to_s
+            index = existing.index { |entry| entry["key"] == criterion["key"] }
+            index ? existing[index] = criterion : existing << criterion
+          elsif allow_keyless
+            existing << criterion unless existing.include?(criterion)
+          else
+            raise InvalidDocument, "a criterion in #{owner} has no key"
+          end
+        end
+      end
+
       def merge_sets(product, incoming_sets)
         incoming_sets.each do |incoming|
           incoming = incoming.to_h.deep_stringify_keys
@@ -213,16 +288,31 @@ module ActiveAgent
 
           existing = product["sets"].find { |set| set["key"] == incoming["key"].to_s }
           if existing
-            %w[name description judge criteria].each { |field| existing[field] = incoming[field] if incoming[field].present? }
+            %w[name description judge].each { |field| existing[field] = incoming[field] if incoming[field].present? }
+            merge_pass(existing, incoming["pass"], "set #{existing['key'].inspect} of product #{product['key'].inspect}")
             existing.merge!(incoming.except(*SET_KEYS))
+            merge_set_criteria(existing, incoming, product)
             merge_scenarios(existing, Array(incoming["scenarios"]))
           else
-            product["sets"] << incoming.merge("key" => incoming["key"].to_s, "scenarios" => [])
+            product["sets"] << incoming.merge("key" => incoming["key"].to_s, "criteria" => [], "scenarios" => [])
+              .tap { |set| merge_set_criteria(set, incoming, product) }
               .tap { |set| merge_scenarios(set, Array(incoming["scenarios"])) }
           end
         end
       end
 
+      # A later document's `pass` layers over the existing one key by key.
+      def merge_pass(existing, incoming, place)
+        incoming = pass_block(incoming, place)
+        existing["pass"] = pass_block(existing["pass"], place).merge(incoming) if incoming.any?
+      end
+
+      # Layers a set document's `criteria` over the set's by key; a set keeps
+      # a criterion without a key.
+      def merge_set_criteria(set, incoming, product)
+        merge_criteria(set["criteria"], incoming["criteria"], "set #{set['key'].inspect} of product #{product['key'].inspect}", allow_keyless: true)
+      end
+
       def merge_scenarios(set, incoming_scenarios)
         incoming_scenarios.each do |scenario|
           scenario = scenario.to_h.deep_stringify_keys
@@ -237,39 +327,67 @@ module ActiveAgent
 
       def build_products(documents)
         documents.each_with_index.map do |product, position|
+          sets = product["sets"].each_with_index.map { |set, set_position| build_set(set, set_position, product["key"]) }
           Product.new(
             key: product["key"],
             name: product["name"].presence || product["key"],
             description: product["description"].presence,
             agent: product["agent"].presence&.to_s,
+            criteria: product["criteria"],
+            pass: validated_pass(product["pass"], PassThresholds::SUITE_KEYS, "product #{product['key'].inspect}"),
             metadata: product.except(*PRODUCT_KEYS),
             position: position,
-            sets: product["sets"].each_with_index.map { |set, set_position| build_set(set, set_position) }
+            sets: sets
           )
         end
       end
 
-      def build_set(set, position)
+      def build_set(set, position, product_key)
         Set.new(
           key: set["key"],
           name: set["name"].presence || set["key"],
           description: set["description"].presence,
           judge: (set["judge"] || {}).to_h.stringify_keys,
-          criteria: Array(set["criteria"]).map { |criterion| criterion.to_h.deep_stringify_keys },
+          criteria: set["criteria"],
+          pass: validated_pass(set["pass"], PassThresholds::GROUP_KEYS, "set #{set['key'].inspect} of product #{product_key.inspect}"),
           metadata: set.except(*SET_KEYS),
           position: position,
           scenarios: set["scenarios"].each_with_index.map do |entry, index|
-            Scenario.from_hash(entry.merge("position" => index), group: set["key"], group_name: set["name"].presence || set["key"])
+            Scenario.from_hash(entry.merge("position" => index), group: set["key"], group_name: set["name"].presence || set["key"],
+              metadata: entry.except(*SCENARIO_KEYS))
           end
         )
       end
 
+      # A `pass` block as a string-keyed Hash, {} when absent, checked the
+      # way PassThresholds reads it (`allowed` is SUITE_KEYS for a product,
+      # GROUP_KEYS for a set) so a block the suite would refuse fails the
+      # catalog as it loads rather than the first run that asks for it; a
+      # refusal names the product or set the block sits on.
+      def validated_pass(value, allowed, place)
+        pass = pass_block(value, place)
+        PassThresholds.validate(pass, allowed, place)
+        pass
+      rescue ArgumentError => e
+        raise InvalidDocument, e.message
+      end
+
+      # A `pass` block as a string-keyed Hash, {} when absent; anything but
+      # a mapping is refused naming the product or set it sits on.
+      def pass_block(value, place)
+        PassThresholds.pass_block(value, place)
+      rescue ArgumentError => e
+        raise InvalidDocument, e.message
+      end
+
       def product_document(product)
         compact({
           "key" => product.key,
           "name" => product.name,
           "description" => product.description,
           "agent" => product.agent,
+          "criteria" => product.criteria,
+          "pass" => product.pass,
           "sets" => product.sets.map { |set| set_document(set) }
         }.merge(product.metadata))
       end
@@ -281,6 +399,7 @@ module ActiveAgent
           "description" => set.description,
           "judge" => set.judge,
           "criteria" => set.criteria,
+          "pass" => set.pass,
           "scenarios" => set.scenarios.map { |scenario| scenario_document(scenario) }
         }.merge(set.metadata))
       end
@@ -294,7 +413,19 @@ module ActiveAgent
           "tags" => scenario.tags,
           "params" => scenario.params,
           "production_only" => (true if scenario.production_only?)
-        })
+        }).merge(canonical_metadata(scenario.metadata.to_h.deep_stringify_keys.except(*SCENARIO_KEYS)))
+      end
+
+      # JSON object storage may reorder keys; array order remains meaningful.
+      def canonical_metadata(value)
+        case value
+        when Hash
+          value.keys.sort.to_h { |key| [ key, canonical_metadata(value[key]) ] }
+        when Array
+          value.map { |entry| canonical_metadata(entry) }
+        else
+          value
+        end
       end
 
       def compact(hash)

data/lib/active_agent/evals/gate_check.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+module ActiveAgent
+  module Evals
+    # A report's pass gates as one check with the message a failing
+    # assertion prints, so TestHelper#assert_meets_pass_gates and the RSpec
+    # matcher (`active_agent/evals/rspec`) fail with the same text: how many
+    # gates are not met, each of them as a sentence, then the failures to
+    # read under a heading per group.
+    #
+    #   check = ActiveAgent::Evals::GateCheck.new(report)
+    #   check.met?   # => false
+    #   puts check.message
+    #   # 2 of 5 pass gates not met
+    #   # - Order support · gpt-5-mini: 1/2 (50%) passed, below 100%
+    #   # - All · gpt-5-mini: 3/4 (75%) passed, below 80%
+    #   #
+    #   # Order support failures:
+    #   # - `order_status` on `gpt-5-mini`: low quality — Scored 50% against a pass threshold of 70%, weakest on response_present (100%).
+    class GateCheck
+      # @param report [Report] the report whose gates are checked
+      # @param thresholds [PassThresholds, nil] the bars to hold the report
+      #   to; without them Report#gates decides, taking the report's own,
+      #   else the defaults. Only their `min_pass_rate` and `gated` apply:
+      #   whether an answer passed was settled at the threshold the Runner
+      #   scored it at, so a different `threshold` here changes nothing.
+      def initialize(report, thresholds = nil)
+        @report = report
+        @bars = [ thresholds ].compact
+      end
+
+      # Whether every gate is met.
+      def met?
+        @report.passed?(*@bars)
+      end
+
+      # Every gate, in Report#gates order.
+      def gates
+        @report.gates(*@bars)
+      end
+
+      # The gates that are not met, in gates order.
+      def unmet
+        gates.reject(&:met?)
+      end
+
+      # The text a failing assertion prints. The first line counts the gates
+      # not met, or is the one line "every pass gate met" when none is; then
+      # each unmet gate as Gate#to_s; then, after a blank line, the failures
+      # to read: for each group with an unmet gate, in gates order, "<group>
+      # failures:" over Report#failures_markdown for that group. An unmet All
+      # row adds the failures it counts, those in gated groups or in no
+      # group: under "Failures:" when no group row is unmet, else the ones
+      # outside the groups already listed under "Other failures:", when there
+      # are any. An ungated group's failures count toward no gate and are
+      # never listed.
+      def message
+        all = gates
+        return "no result reached a pass gate; the run evaluated nothing" if all.empty?
+
+        unmet = all.reject(&:met?)
+        return "every pass gate met" if unmet.empty?
+
+        lines = [ "#{unmet.size} of #{all.size} pass gates not met", *unmet.map { |gate| "- #{gate}" } ]
+        grouped = unmet.reject { |gate| gate.group.nil? }.uniq(&:group)
+        grouped.each { |gate| lines << "" << "#{gate.name} failures:" << @report.failures_markdown(group: gate.group) }
+        if unmet.any? { |gate| gate.group.nil? }
+          ungated = all.reject(&:gated).map(&:group)
+          listed = grouped.map(&:group)
+          others = @report.failures.reject { |result| (ungated + listed).include?(result.scenario.group) }
+          heading = listed.empty? ? "Failures:" : "Other failures:"
+          lines << "" << heading << others.map { |result| @report.failure_markdown(result) }.join("\n") if others.any?
+        end
+        lines.join("\n")
+      end
+      alias to_s message
+    end
+  end
+end