activeagent 1.9.6 → 1.9.7

data/lib/active_agent/evals/rspec.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+# Loaded by an explicit `require "active_agent/evals/rspec"` — never by
+# `require "active_agent/evals"` — so a host on another test framework never
+# sees it. It needs nothing from rspec: the matcher speaks RSpec's matcher
+# protocol on its own and loads with rspec absent.
+require_relative "../evals"
+
+module ActiveAgent
+  module Evals
+    # `expect(report).to meet_its_pass_gates`: an RSpec matcher over
+    # GateCheck, failing with its message, as TestHelper's
+    # `assert_meets_pass_gates` does. Include it beside TestHelper where the
+    # evals are set up, tagged and run only when EVALS is on:
+    #
+    #   # spec/rails_helper.rb
+    #   require "active_agent/evals/test_helper"
+    #   require "active_agent/evals/rspec"
+    #
+    #   RSpec.configure do |config|
+    #     config.expect_with :rspec, :minitest
+    #     config.define_derived_metadata(file_path: %r{/spec/evals/}) { |metadata| metadata[:eval] = true }
+    #     config.include ActiveAgent::Evals::TestHelper, eval: true
+    #     config.include ActiveAgent::Evals::RSpec, eval: true
+    #     config.filter_run_excluding eval: true unless ActiveAgent::Evals::TestHelper.enabled?
+    #   end
+    #
+    #   # spec/evals/support_desk_spec.rb
+    #   RSpec.describe "the support desk suite" do
+    #     it "clears its pass gates" do
+    #       report = run_eval_suite("config/evals/support_desk.yml") { |scenario, _spec| SupportAgent.with(question: scenario.prompt).help }
+    #
+    #       expect(report).to meet_its_pass_gates
+    #     end
+    #   end
+    module RSpec
+      # The matcher: every gate of the report met under `thresholds`, or
+      # without them under the report's own bars, else the defaults.
+      def meet_its_pass_gates(thresholds = nil)
+        MeetItsPassGates.new(thresholds)
+      end
+
+      # The matcher object, speaking RSpec's matcher protocol (`matches?`,
+      # the failure messages and `description`) without rspec loaded.
+      class MeetItsPassGates
+        def initialize(thresholds = nil)
+          @thresholds = thresholds
+        end
+
+        # Checks `report` and keeps the check for the failure messages.
+        def matches?(report)
+          @check = GateCheck.new(report, @thresholds)
+          @check.met?
+        end
+
+        # GateCheck#message: the unmet gates and the failures to read.
+        def failure_message
+          @check.message
+        end
+
+        # For `expect(report).not_to meet_its_pass_gates`: every gate, each
+        # met, as Gate#to_s.
+        def failure_message_when_negated
+          [ "expected the report not to meet its pass gates, but every gate was met:", *@check.gates.map(&:to_s) ].join("\n")
+        end
+
+        # The generated example description: "meet its pass gates", or
+        # "meet the given pass gates" when thresholds were passed.
+        def description
+          @thresholds ? "meet the given pass gates" : "meet its pass gates"
+        end
+
+        # RSpec shows no diff: the failure message already lists the unmet
+        # gates.
+        def diffable?
+          false
+        end
+
+        # The matcher takes a report, not a block, so RSpec refuses
+        # `expect { ... }.to meet_its_pass_gates`.
+        def supports_block_expectations?
+          false
+        end
+      end
+    end
+  end
+end

data/lib/active_agent/evals/ruby_llm.rb

@@ -55,7 +55,13 @@ module ActiveAgent
         # `chat_options` go to `chat` itself, so they must be keywords
         # `RubyLLM::Chat.new` takes (`protocol:` on RubyLLM 2.x); anything else
         # raises ArgumentError here rather than failing every judge call
-        # later. Configure the chat through `temperature:` or `configure:`:
+        # later. With `verify: true`, a missing key fails here too: the judge's
+        # label, model and provider go through `verify!` with `context` and
+        # `providers`, so a judge whose provider has no key, or is not one of
+        # the providers a host has keys for, refuses when it is built rather
+        # than at its first grade, after the agent has already been replayed;
+        # the default stays lazy. Configure the chat through `temperature:` or
+        # `configure:`:
         #
         #   ActiveAgent::Evals::RubyLLM.judge(label: "judge", model: "gpt-5-mini", provider: :openai,
         #                                     temperature: 0,
@@ -74,12 +80,23 @@ module ActiveAgent
         #   `{ "kind", "model", "input_tokens", "output_tokens", "cost" }`, for a
         #   host that meters its judge's spend; an error it raises is logged,
         #   not propagated, so bookkeeping never costs a grade
+        # @param verify [Boolean] run `verify!` on the judge's own model with
+        #   `context` when the judge is built, so a provider with no key
+        #   refuses here rather than at the first grade
+        # @param providers [Array<String, Symbol>, nil] with `verify: true`, the
+        #   providers a host has keys for, handed to `verify!` as its
+        #   `providers:`; given without `verify: true` it raises ArgumentError
         # @param chat_options [Hash] other `chat` keywords, such as `protocol:`
         # @return [Judge]
-        # @raise [ArgumentError] when `chat_options` names a keyword `RubyLLM::Chat.new` does not take
+        # @raise [ArgumentError] when `chat_options` names a keyword `RubyLLM::Chat.new` does not take,
+        #   or `providers:` is given without `verify: true`
+        # @raise [::RubyLLM::ConfigurationError] with `verify: true`, when the judge's provider is not
+        #   among `providers` or `context` cannot build the judge's chat
         def judge(label:, model:, provider:, context: ::RubyLLM, correlation: nil, assume_model_exists: true,
-                  temperature: nil, configure: nil, on_usage: nil, **chat_options)
+                  temperature: nil, configure: nil, on_usage: nil, verify: false, providers: nil, **chat_options)
           check_chat_options!(chat_options)
+          raise ArgumentError, "providers: is checked only with verify: true" if providers && !verify
+          verify!(ModelSpec.new(label: label, provider: provider, model: model), context: context, providers: providers) if verify
 
           Judge.new(label: label) do |instructions:, prompt:, kind:|
             complete = lambda do
@@ -96,6 +113,65 @@ module ActiveAgent
           end
         end
 
+        # Checks, before a run spends anything, that each model can be built
+        # on `context`: its provider is one RubyLLM registered, is one of
+        # `providers` when those are given, and is configured, which
+        # `context.chat(model:, provider:, assume_model_exists: true)` settles
+        # while it instantiates the provider. Nothing is sent over the
+        # network: RubyLLM resolves the model locally. The specs are checked
+        # in order, and the first that cannot run raises, naming what to
+        # change: the provider's configuration requirements that `context`
+        # leaves unset, as environment variable names, or RubyLLM's own
+        # message when those are all set and it refused for another reason,
+        # such as a credential it checks beyond them or a deployment it
+        # cannot resolve:
+        #
+        #   context = RubyLLM.context { |config| config.openai_api_key = ENV["OPENAI_API_KEY"] }
+        #   models = ActiveAgent::Evals::ModelSpec.parse_all(%w[gpt-5-mini claude-sonnet-5], default_provider: "openai")
+        #
+        #   ActiveAgent::Evals::RubyLLM.verify!(models, context: context)
+        #   # raises RubyLLM::ConfigurationError: claude-sonnet-5 runs on anthropic, which has no API key. Set ANTHROPIC_API_KEY.
+        #
+        # @param models [Array<ModelSpec>, ModelSpec] the run's models, or anything
+        #   answering `label`, `provider` and `model`; a single spec is wrapped
+        # @param context [#chat] where `chat` is called; defaults to `::RubyLLM`.
+        #   When it also answers `config`, as `::RubyLLM` and a `RubyLLM.context`
+        #   do, a refusal names only the requirements that configuration leaves
+        #   unset
+        # @param providers [Array<String, Symbol>, String, Symbol, nil] the
+        #   providers the host has credentials for; a model on any other provider
+        #   is refused before its chat is built
+        # @return [Array<ModelSpec>] the given models
+        # @raise [::RubyLLM::ConfigurationError] for the first model whose provider
+        #   RubyLLM does not know, is outside `providers`, or cannot be built on
+        #   `context`; in the last case RubyLLM's own error is the cause, and the
+        #   message names what to set or carries RubyLLM's own
+        def verify!(models, context: ::RubyLLM, providers: nil)
+          specs = models.is_a?(Array) ? models : [ models ]
+          configured = Array(providers).map(&:to_s) if providers
+
+          specs.each do |spec|
+            provider = spec.provider.to_s
+            unless ::RubyLLM::Provider.providers.key?(provider.to_sym)
+              raise ::RubyLLM::ConfigurationError,
+                    "#{spec.label} runs on #{provider}, which is not a RubyLLM provider; " \
+                    "RubyLLM knows #{::RubyLLM::Provider.providers.keys.map(&:to_s).sort.join(', ')}"
+            end
+            if configured && !configured.include?(provider)
+              configured_for = configured.empty? ? "no provider has credentials configured" : "credentials are configured for #{configured.join(', ')}"
+              raise ::RubyLLM::ConfigurationError, "#{spec.label} runs on #{provider}; #{configured_for}"
+            end
+
+            begin
+              context.chat(model: spec.model, provider: provider.to_sym, assume_model_exists: true)
+            rescue ::RubyLLM::ConfigurationError => e
+              raise ::RubyLLM::ConfigurationError, refusal(spec, provider, context, e)
+            end
+          end
+
+          specs
+        end
+
         # Builds a Replay from the ordered messages of a RubyLLM conversation:
         # `acts_as_chat` message records of either RubyLLM generation (read
         # through their own `to_llm`), `RubyLLM::Message` values, or anything
@@ -167,6 +243,32 @@ module ActiveAgent
                 "Pass temperature: to judge, or configure the chat with configure: ->(chat) { chat.with_... }."
         end
 
+        # The message for a model whose provider refused to be built: the
+        # requirement names `context` leaves unset, as environment variable
+        # names, called an API key only when each of them is one; RubyLLM's
+        # own message when nothing is left to set.
+        def refusal(spec, provider, context, error)
+          names = unset_requirements(provider, context).map { |name| name.to_s.upcase }
+          return "#{spec.label} runs on #{provider}: #{error.message.strip}" if names.empty?
+
+          state = names.all? { |name| name.end_with?("_API_KEY") } ? "has no API key" : "is not configured"
+          "#{spec.label} runs on #{provider}, which #{state}. Set #{names.join(' and ')}."
+        end
+
+        # The provider's configuration requirements that `context` leaves
+        # unset: all of them when the context exposes no configuration, and
+        # none on a release whose providers do not list them.
+        def unset_requirements(provider, context)
+          provider_class = ::RubyLLM::Provider.providers[provider.to_sym]
+          return [] unless provider_class.respond_to?(:configuration_requirements)
+
+          requirements = provider_class.configuration_requirements
+          return requirements unless context.respond_to?(:config)
+
+          config = context.config
+          requirements.reject { |name| config.respond_to?(name) && config.public_send(name) }
+        end
+
         def report_usage(callback, kind, model, response)
           return unless callback
 

data/lib/active_agent/evals/runner.rb

@@ -30,8 +30,9 @@ module ActiveAgent
       # tool to add, or how to change the instructions.
       DEFAULT_REFINE_FAULTS = %w[missing_capability expected_tool_not_called ungrounded_answer low_quality missing_content].freeze
       DEFAULT_JUDGE_LIMIT = 25
+      DEFAULT_TRIALS = 1
 
-      attr_reader :scenarios, :models, :criteria, :judge, :threshold
+      attr_reader :scenarios, :models, :criteria, :judge, :threshold, :pass_thresholds, :judge_failed_expectations, :trials
 
       # @param scenarios [Array<Scenario>]
       # @param models [Array<ModelSpec>]
@@ -43,6 +44,12 @@ module ActiveAgent
       # @param available_tools [Hash{String=>String}, Array<String>] the agent's tool roster
       # @param instructions [String, nil] the agent's instructions, for the judge
       # @param agent_name [String] how recommendations refer to the agent
+      # @param threshold [Float, nil] the score an answer must reach to pass;
+      #   an explicit value wins over the `pass_thresholds` bar, and without
+      #   either it is PASS_THRESHOLD
+      # @param pass_thresholds [PassThresholds, nil] the bars the suite
+      #   declared (Suite#pass_thresholds): its threshold applies when none
+      #   is given here, and the Report gates each group by it (Report#gates)
       # @param on_result [#call, nil] called with each Result as it lands
       # @param around_evaluation [#call, nil] called with (scenario, spec) and a
       #   block that returns the Result. Establishes context for replay, scoring
@@ -58,20 +65,38 @@ module ActiveAgent
       #   after `judge_limit`; false skips it. A caller that scores each
       #   scenario in its own Runner (a background job per scenario) shares one
       #   run-wide limit this way, since `judge_limit` counts per instance.
+      # @param judge_failed_expectations [Boolean] whether the judge grades a
+      #   replay that missed one of its scenario's expectations (an expected
+      #   tool not called, forbidden or missing content, a failed tool call).
+      #   false skips its task grade, its llm_judge criteria and its judged
+      #   recommendation, since it fails whatever the judge says, and the
+      #   calls saved do not count against `judge_limit`. The default grades
+      #   every answered replay.
+      # @param trials [Integer] how many times each scenario is replayed under
+      #   each model, 1 or more. Every replay is a Result of its own under the
+      #   scenario's key, told apart by Result#trial, so a flaky scenario
+      #   reads as a pass share rather than one pass or fail.
       def initialize(scenarios:, models:, replay:, criteria: [], judge: nil, judge_task: true, available_tools: {},
-                     instructions: nil, agent_name: "The agent", threshold: PASS_THRESHOLD,
+                     instructions: nil, agent_name: "The agent", threshold: nil, pass_thresholds: nil,
                      refine_faults: DEFAULT_REFINE_FAULTS, judge_limit: DEFAULT_JUDGE_LIMIT, on_result: nil,
-                     around_evaluation: nil, require_judge_scores: false, metadata: {}, release: nil, refine_gate: nil)
+                     around_evaluation: nil, require_judge_scores: false, metadata: {}, release: nil, refine_gate: nil,
+                     judge_failed_expectations: true, trials: DEFAULT_TRIALS)
+        unless trials.is_a?(Integer) && trials >= 1
+          raise ArgumentError, "trials must be an Integer of 1 or more, got #{trials.inspect}"
+        end
+
         @scenarios = scenarios
         @models = models
         @replay = replay
+        @trials = trials
         @criteria = criteria
         @judge = judge
         @judge_task = judge_task
         @available_tools = normalize_tools(available_tools)
         @instructions = instructions
         @agent_name = agent_name
-        @threshold = threshold
+        @threshold = threshold || pass_thresholds&.threshold || PASS_THRESHOLD
+        @pass_thresholds = pass_thresholds
         @refine_faults = refine_faults
         @judge_limit = judge_limit
         @on_result = on_result
@@ -80,27 +105,33 @@ module ActiveAgent
         @metadata = metadata
         @release = release
         @refine_gate = refine_gate
+        @judge_failed_expectations = judge_failed_expectations
         @judge_calls = 0
-        @scorer = Scorer.new(criteria: criteria, judge: judge)
+        @scorer = Scorer.new(criteria: criteria, judge: judge, judge_failed_expectations: judge_failed_expectations)
       end
 
+      # Replays every scenario under every model `trials` times: scenario by
+      # scenario, model by model, trial 1 to `trials`.
       def call
         results = @scenarios.flat_map do |scenario|
-          @models.map do |spec|
-            evaluate_with_context(scenario, spec).tap { |result| @on_result&.call(result) }
+          @models.flat_map do |spec|
+            (1..@trials).map do |trial|
+              evaluate_with_context(scenario, spec, trial: trial).tap { |result| @on_result&.call(result) }
+            end
           end
         end
 
         Report.new(results: results, models: @models, judge: @judge, instructions: @instructions,
-                   threshold: @threshold, metadata: @metadata, release: @release)
+                   threshold: @threshold, metadata: @metadata, release: @release, pass_thresholds: @pass_thresholds)
       end
 
       # Scores and diagnoses one replay. Public so a caller that has already run
       # the agent (a background job per scenario, say) can score without the loop.
-      def evaluate(scenario, spec, replay = nil)
+      # `trial` numbers the Result when the caller replays a scenario more than once.
+      def evaluate(scenario, spec, replay = nil, trial: 1)
         replay ||= run_replay(scenario, spec)
         scores = @scorer.score(scenario, replay)
-        if judge_task? && replay.answer.present?
+        if judge_task? && replay.answer.present? && judge_answer?(scores)
           scores["task_completion"] = @judge.score_task(scenario: scenario, answer: replay.answer)
         end
         score = Scorer.mean(scores)
@@ -110,7 +141,7 @@ module ActiveAgent
                                    judge_keys: llm_judge_keys)
         diagnosis ||= unavailable_judge_diagnosis(scores)
         diagnosis_hash = diagnosis&.to_h
-        refine!(diagnosis_hash, scenario, replay, diagnosis) if diagnosis_hash
+        refine!(diagnosis_hash, scenario, replay, diagnosis, scores) if diagnosis_hash
 
         Result.new(
           scenario: scenario,
@@ -119,16 +150,17 @@ module ActiveAgent
           scores: scores,
           score: score,
           status: replay.errored? ? "errored" : (diagnosis ? "failed" : "passed"),
-          diagnosis: diagnosis_hash
+          diagnosis: diagnosis_hash,
+          trial: trial
         )
       end
 
       private
 
-      def evaluate_with_context(scenario, spec)
-        return evaluate(scenario, spec) unless @around_evaluation
+      def evaluate_with_context(scenario, spec, trial:)
+        return evaluate(scenario, spec, trial: trial) unless @around_evaluation
 
-        result = @around_evaluation.call(scenario, spec) { evaluate(scenario, spec) }
+        result = @around_evaluation.call(scenario, spec) { evaluate(scenario, spec, trial: trial) }
         # A wrapper written the natural way — do something, yield, do something
         # after — returns that last value rather than the Result. Left alone it
         # reaches on_result and the Report, and fails somewhere far from the
@@ -144,6 +176,16 @@ module ActiveAgent
         @judge && @judge_task && @criteria.none? { |criterion| criterion.to_h.stringify_keys["type"] == "llm_judge" }
       end
 
+      # Whether the judge is asked about a replay with these scores: always by
+      # default, or only when it met every expectation of its scenario. A
+      # replay that did not already carries one of the faults Diagnosis
+      # assigns ahead of low_quality (tool_error, missing_capability,
+      # expected_tool_not_called, ungrounded_answer, forbidden_content or
+      # missing_content), so its status is failed whatever the judge would say.
+      def judge_answer?(scores)
+        @judge_failed_expectations || Scorer.expectations_met?(scores)
+      end
+
       # The keys in `scores` a judge graded, so a low grade is not averaged
       # away against rule checks.
       def llm_judge_keys
@@ -189,8 +231,9 @@ module ActiveAgent
         end
       end
 
-      def refine!(diagnosis, scenario, replay, result)
+      def refine!(diagnosis, scenario, replay, result, scores)
         return unless @judge && @refine_faults.include?(result.fault)
+        return unless judge_answer?(scores)
         return if @judge_calls >= @judge_limit
         return if @refine_gate && !@refine_gate.call
 

data/lib/active_agent/evals/scenario.rb

@@ -24,16 +24,21 @@ module ActiveAgent
     #   @return [Array<String>] labels a catalog files the scenario under ("billing")
     # @!attribute params
     #   @return [Hash] free-form parameters a replay may read; kept as written
+    # @!attribute metadata
+    #   @return [Hash, nil] opaque extension fields from a catalog document
     Scenario = Struct.new(:key, :group, :group_name, :prompt, :expected_tools, :expected_patterns,
-                          :forbidden_patterns, :notes, :production_only, :position, :tags, :params,
+                          :forbidden_patterns, :notes, :production_only, :position, :tags, :params, :metadata,
                           keyword_init: true) do
       # Builds a scenario from a hash — ScenarioParser output, a suite entry, or
       # a persisted record's attributes. Expectations are read from an
       # `expectations`/`expect` sub-hash (`tools`, `contains`, `not_contains`)
       # or from those keys at the top level.
-      def self.from_hash(attributes, group: nil, group_name: nil)
+      # Pass +metadata+ when extension fields have been separated from a
+      # document; its literal `metadata` key may itself be an extension.
+      def self.from_hash(attributes, group: nil, group_name: nil, metadata: nil)
         attrs = attributes.to_h.deep_stringify_keys
         expect = (attrs["expectations"] || attrs["expect"] || {}).to_h.stringify_keys
+        metadata = attrs["metadata"] if metadata.nil? && attrs["metadata"].is_a?(Hash)
 
         new(
           key: attrs["key"].to_s,
@@ -47,7 +52,8 @@ module ActiveAgent
           production_only: attrs["production_only"] == true,
           position: attrs["position"],
           tags: list(attrs["tags"]),
-          params: (attrs["params"] || {}).to_h.deep_stringify_keys
+          params: (attrs["params"] || {}).to_h.deep_stringify_keys,
+          metadata: (metadata || {}).to_h.deep_stringify_keys
         )
       end
 
@@ -68,8 +74,10 @@ module ActiveAgent
       end
 
       def to_h
-        super.compact.reject { |field, value| %i[tags params].include?(field) && value.blank? }
+        super.compact.reject { |field, value| %i[tags params metadata].include?(field) && value.blank? }
       end
     end
+
+    Scenario::DOCUMENT_KEYS = %w[key prompt expect expectations tools contains not_contains notes tags params production_only position].freeze
   end
 end

data/lib/active_agent/evals/scenario_parser.rb

@@ -138,12 +138,20 @@ module ActiveAgent
           end
         end
 
-        Suite.new([ document ]).all_scenarios.map do |item|
+        suite_from(document).all_scenarios.map do |item|
           scenario(prompt: item.prompt, group: item.group, group_name: item.group_name, key: item.key,
                    notes: item.notes, expectations: item.expectations, production_only: item.production_only?)
         end
       end
 
+      # The document as a Suite; a `criteria` block the Suite refuses is a
+      # ParseError, as every other fault in a pasted document is.
+      def suite_from(document)
+        Suite.new([ document ])
+      rescue ArgumentError => e
+        raise ParseError, e.message
+      end
+
       def scenario_from_hash(entry)
         entry = entry.stringify_keys
         prompt = entry["prompt"] || entry["message"] || entry["input"] || entry["question"]

data/lib/active_agent/evals/scorer.rb

@@ -4,7 +4,8 @@ module ActiveAgent
   module Evals
     # Scores one Replay against an evaluation's criteria and the scenario's
     # expectations. Returns `criterion key => 0.0..1.0`, with nil for a
-    # criterion that could not be scored (an llm_judge criterion with no judge).
+    # criterion that could not be scored (an llm_judge criterion with no judge,
+    # or one the Scorer did not ask because the expectations failed).
     #
     # Criteria are `{ "key", "type", "config" }` hashes:
     #
@@ -22,41 +23,37 @@ module ActiveAgent
     class Scorer
       RULE_CRITERION_TYPES = %w[response_present min_length max_latency_ms token_budget contains not_contains].freeze
       CRITERION_TYPES = (RULE_CRITERION_TYPES + %w[llm_judge]).freeze
+      # The keys `score` adds from the scenario's own expectations.
+      EXPECTATION_KEYS = %w[expected_tools expected_content forbidden_content tools_succeeded].freeze
 
-      attr_reader :criteria, :judge
+      attr_reader :criteria, :judge, :judge_failed_expectations
 
-      def initialize(criteria: [], judge: nil)
+      # @param criteria [Array<Hash>] the criteria every replay is scored on
+      # @param judge [Judge, nil] scores the llm_judge criteria
+      # @param judge_failed_expectations [Boolean] whether an llm_judge criterion
+      #   is asked of a replay that missed one of its scenario's expectations;
+      #   false leaves it nil, since the replay fails on the expectation anyway
+      def initialize(criteria: [], judge: nil, judge_failed_expectations: true)
         @criteria = Array(criteria).map { |criterion| criterion.to_h.deep_stringify_keys }
         @judge = judge
+        @judge_failed_expectations = judge_failed_expectations
       end
 
       def score(scenario, replay)
         scores = {}
         answer = replay.answer.to_s
+        expectations = expectation_scores(scenario, replay)
+        ask_judge = answer.present? && (@judge_failed_expectations || self.class.expectations_met?(expectations))
 
         @criteria.each do |criterion|
-          scores[criterion["key"]] = answer.present? ? score_criterion(criterion, scenario, replay) : 0.0
+          scores[criterion["key"]] =
+            if answer.blank? then 0.0
+            elsif criterion["type"] == "llm_judge" && !ask_judge then nil
+            else score_criterion(criterion, scenario, replay)
+            end
         end
 
-        if scenario.expected_tools.any?
-          scores["expected_tools"] = (scenario.expected_tools & replay.tool_names).any? ? 1.0 : 0.0
-        end
-        if scenario.expected_patterns.any?
-          hits = scenario.expected_patterns.count { |pattern| self.class.matches_pattern?(answer, pattern) }
-          scores["expected_content"] = (hits.to_f / scenario.expected_patterns.size).round(3)
-        end
-        if scenario.forbidden_patterns.any?
-          hit = scenario.forbidden_patterns.any? { |pattern| self.class.matches_pattern?(answer, pattern) }
-          scores["forbidden_content"] = hit ? 0.0 : 1.0
-        end
-        # A tool that ran without erroring is evidence only when it is one the
-        # scenario expected: a wrong tool that succeeded used to outscore
-        # calling nothing at all.
-        if replay.tool_calls.any? && (scenario.expected_tools.empty? || (scenario.expected_tools & replay.tool_names).any?)
-          scores["tools_succeeded"] = replay.failed_tool_calls.any? ? 0.0 : 1.0
-        end
-
-        scores
+        scores.merge(expectations)
       end
 
       # The mean of the scored criteria, or nil when nothing could be scored.
@@ -67,6 +64,12 @@ module ActiveAgent
         (scored.sum / scored.size).round(3)
       end
 
+      # Whether every expectation score in `scores` is a full 1.0. A scores
+      # hash with no expectation keys is met.
+      def self.expectations_met?(scores)
+        scores.slice(*EXPECTATION_KEYS).values.all? { |value| value.to_f >= 1.0 }
+      end
+
       # How long one pattern may take to match one answer. Patterns are
       # whatever the scenario's author typed, and a pathological one must not
       # stall the evaluation.
@@ -90,6 +93,33 @@ module ActiveAgent
 
       private
 
+      # The scores the scenario's own expectations add, in the order they appear
+      # after the criteria.
+      def expectation_scores(scenario, replay)
+        scores = {}
+        answer = replay.answer.to_s
+
+        if scenario.expected_tools.any?
+          scores["expected_tools"] = (scenario.expected_tools & replay.tool_names).any? ? 1.0 : 0.0
+        end
+        if scenario.expected_patterns.any?
+          hits = scenario.expected_patterns.count { |pattern| self.class.matches_pattern?(answer, pattern) }
+          scores["expected_content"] = (hits.to_f / scenario.expected_patterns.size).round(3)
+        end
+        if scenario.forbidden_patterns.any?
+          hit = scenario.forbidden_patterns.any? { |pattern| self.class.matches_pattern?(answer, pattern) }
+          scores["forbidden_content"] = hit ? 0.0 : 1.0
+        end
+        # A tool that ran without erroring is evidence only when it is one the
+        # scenario expected: a wrong tool that succeeded used to outscore
+        # calling nothing at all.
+        if replay.tool_calls.any? && (scenario.expected_tools.empty? || (scenario.expected_tools & replay.tool_names).any?)
+          scores["tools_succeeded"] = replay.failed_tool_calls.any? ? 0.0 : 1.0
+        end
+
+        scores
+      end
+
       def score_criterion(criterion, scenario, replay)
         config = criterion["config"] || {}
         answer = replay.answer.to_s