activeagent 1.9.6 → 1.9.8

data/lib/active_agent/evals/report_html.rb

@@ -215,7 +215,13 @@ module ActiveAgent
       # --- page ------------------------------------------------------------
 
       def html_title
-        "Evaluation — #{plural(scenario_cohorts.size, 'scenario')} × #{plural(@models.size, 'model')}"
+        with_trials("Evaluation — #{plural(scenario_cohorts.size, 'scenario')} × #{plural(@models.size, 'model')}")
+      end
+
+      # +text+ with "3 trials per scenario" beside it when each scenario was
+      # replayed more than once; +text+ alone otherwise.
+      def with_trials(text)
+        trials > 1 ? "#{text} · #{trials} trials per scenario" : text
       end
 
       def html_styles
@@ -282,7 +288,7 @@ module ActiveAgent
         passed = @results.count(&:passed?)
         ratio = total.positive? ? passed.to_f / total : 0.0
         tiles = [
-          html_tile("Scenario runs", total, "#{plural(scenario_cohorts.size, 'scenario')} × #{plural(@models.size, 'model')}"),
+          html_tile("Scenario runs", total, with_trials("#{plural(scenario_cohorts.size, 'scenario')} × #{plural(@models.size, 'model')}")),
           html_tile("Pass rate", Format.percent(total.positive? ? ratio : nil), "#{passed}/#{total} passed", tone: tone_for(ratio)),
           html_tile("Open faults", total - passed, plural(fix_items.size, "fix item")),
           html_cost_tile,
@@ -496,29 +502,49 @@ module ActiveAgent
 
         <<~MATRIX
           <section class="section">
-          <div class="section-head"><span class="micro">Scenarios</span><span class="meta">#{h(meta)}</span></div>
+          <div class="section-head"><span class="micro">Scenarios</span><span class="meta">#{h(with_trials(meta))}</span></div>
           <div class="matrix"><div class="inner">#{rows.join}</div></div>
           </section>
         MATRIX
       end
 
+      # A group's row: each model's passes over every replay of the row's
+      # scenarios, toned against the group's bar when the suite declared one —
+      # the row's own pass rate at or above it green, below it red, an ungated
+      # group plain, with the bar as the tooltip — and otherwise green when
+      # every replay passed and red when none did. A row with a group name but
+      # no group key has no bar to read.
       def html_group_row(cohorts)
-        name = group_name(cohorts.first.first.scenario)
+        scenario = cohorts.first.first.scenario
+        name = group_name(scenario)
+        gated = pass_thresholds && scenario.group.present?
+        gates_by_label = gated ? gates.select { |gate| gate.group == scenario.group }.index_by(&:label) : {}
         passes = @models.map do |spec|
-          results = cohorts.filter_map { |cohort| cohort.find { |result| result.label == spec.label } }
+          results = cohorts.flat_map { |cohort| cohort.select { |result| result.label == spec.label } }
+          next %(<span class="group-pass">—</span>) if results.empty?
+
           passed = results.count(&:passed?)
-          tone =
-            if results.empty? then ""
-            elsif passed == results.size then " text-success"
-            elsif passed.zero? then " text-error"
-            else ""
-            end
-          %(<span class="group-pass#{tone}">#{results.empty? ? '—' : "#{h(fmt_passes(passed, results.size))} passed"}</span>)
+          tone, title = html_group_tone(passed, results.size, gates_by_label[spec.label])
+          %(<span class="group-pass#{tone}"#{title}>#{h(fmt_passes(passed, results.size))} passed</span>)
         end
         %(<div class="mx group"><span class="group-name">#{h(name)}</span>) +
           %(<span class="count">#{h(plural(cohorts.size, 'scenario'))}</span>#{passes.join}#{html_group_cost(cohorts)}</div>)
       end
 
+      # [tone class, title attribute] for a group-pass span: the row's own
+      # counts against its gate's bar when there is one, so a group split
+      # across the run tones each row by what that row shows; else from the
+      # counts alone, with no title.
+      def html_group_tone(passed, total, gate)
+        if gate
+          tone = gate.gated ? (passed.to_f / total >= gate.min_pass_rate ? " text-success" : " text-error") : ""
+          [ tone, %( title="#{h(gate.gated ? "pass rate ≥ #{Format.percent(gate.min_pass_rate)}" : 'not gated')}") ]
+        elsif passed == total then [ " text-success", "" ]
+        elsif passed.zero? then [ " text-error", "" ]
+        else [ "", "" ]
+        end
+      end
+
       # The group's subtotal — its scenarios' costs summed, judge apart — in
       # the trailing column, so the grid stays aligned under a group row.
       def html_group_cost(cohorts)
@@ -542,8 +568,11 @@ module ActiveAgent
         scenario = cohort.first.scenario
         expects = scenario.expected_tools.map { |tool| %(<span class="expect">#{h(tool)}</span>) }.join
         cells = @models.map do |spec|
-          result = cohort.find { |candidate| candidate.label == spec.label }
-          result ? html_result_cell(result) : %(<div class="cell"><div class="top"><span class="muted">—</span></div></div>)
+          mine = cohort.select { |candidate| candidate.label == spec.label }.sort_by(&:trial)
+          if mine.empty? then %(<div class="cell"><div class="top"><span class="muted">—</span></div></div>)
+          elsif mine.one? then html_result_cell(mine.first)
+          else html_trials_cell(mine)
+          end
         end
         costs = scenario_costs[scenario.key] || {}
         total_cell = if costs["cost"].nil? && costs["judge_cost"].nil?
@@ -564,6 +593,26 @@ module ActiveAgent
           %(<div class="calls">#{html_calls(result, empty: 'no tools called')}</div>#{html_cell_cost(result)}</div>)
       end
 
+      # The cell of a scenario replayed more than once under one model: how
+      # many replays passed, one line per trial (its number, the glyph, the
+      # score and the fault), and what the replays cost together.
+      def html_trials_cell(results)
+        passed = results.count(&:passed?)
+        tone =
+          if passed == results.size then " tone-success"
+          elsif passed.zero? then " tone-error"
+          else ""
+          end
+        lines = results.map do |result|
+          glyph_tone = result.passed? ? "success" : "error"
+          fault = result.fault ? %(<span class="f">#{h(fault_name(result.fault))}</span>) : ""
+          %(<div class="top trial"><span class="muted">#{result.trial}</span><span class="g tone-#{glyph_tone}">#{result.passed? ? '[+]' : '[!]'}</span>) +
+            %(<span class="s tone-#{glyph_tone}">#{h(fmt_score(result.score))}</span>#{fault}</div>)
+        end
+        %(<div class="cell"><div class="top"><span class="s#{tone}">#{h(fmt_passes(passed, results.size))} passed</span></div>) +
+          %(#{lines.join}#{html_trials_cost(results)}</div>)
+      end
+
       # "~$0.0243 · judge ~$0.0015" under a cell: what this answer cost and
       # what judging it cost. A result with no cost at all shows no line.
       def html_cell_cost(result)
@@ -573,6 +622,25 @@ module ActiveAgent
         %(<div class="cost-line"#{cost_title_attr([ result ], estimated: result.estimated_cost?)}>#{h(text)}</div>)
       end
 
+      # The cost line under a trials cell: the replays' costs summed, the
+      # judge's apart. The sum reads as an estimate when any replay's cost
+      # is one, or when a replay went unpriced (a lower bound), and shows its
+      # working in a tooltip only when a replay was priced from tokens.
+      # Nothing when no replay carried a cost.
+      def html_trials_cost(results)
+        costs = results.filter_map { |result| result.replay.cost }
+        judge_costs = results.filter_map { |result| result.judge_usage&.dig("cost") }
+        return "" if costs.empty? && judge_costs.empty?
+
+        priced_from_tokens = results.any?(&:estimated_cost?)
+        estimated = priced_from_tokens || (costs.any? && costs.size < results.size)
+        parts = [ fmt_cost(costs.any? ? costs.sum : nil, estimated: estimated) ]
+        if judge_costs.any?
+          parts << "judge #{fmt_cost(judge_costs.sum, estimated: results.any? { |result| estimated_usage?(result.judge_usage) })}"
+        end
+        %(<div class="cost-line"#{cost_title_attr(results, estimated: priced_from_tokens)}>#{h(parts.join(' · '))}</div>)
+      end
+
       def result_cost_text(result)
         judge_cost = result.judge_usage&.dig("cost")
         return nil if result.replay.cost.nil? && judge_cost.nil?
@@ -619,7 +687,10 @@ module ActiveAgent
         scenario = cohort.first.scenario
         expects = scenario.expected_tools.any? ? %(<span class="exp">expects <b>#{h(scenario.expected_tools.join(' or '))}</b></span>) : ""
         labels = @models.map(&:label)
-        cards = cohort.sort_by { |result| labels.index(result.label) || labels.size }.map { |result| html_result_card(result) }
+        # A model that replayed the scenario more than once numbers its cards.
+        repeated = cohort.map(&:label).tally.select { |_label, count| count > 1 }
+        cards = cohort.sort_by { |result| [ labels.index(result.label) || labels.size, result.trial ] }
+                      .map { |result| html_result_card(result, numbered: repeated.key?(result.label)) }
         md_id = "md-#{anchor(scenario)}"
 
         <<~BLOCK
@@ -632,8 +703,11 @@ module ActiveAgent
         BLOCK
       end
 
-      def html_result_card(result)
+      # +numbered+ names the card's trial, for a model with several cards on
+      # one scenario.
+      def html_result_card(result, numbered: false)
         short, = split_label(result.spec)
+        trial = numbered ? %(<span class="micro sm">trial #{result.trial}</span>) : ""
         status =
           if result.passed? then %(<span class="badge success">passed · #{h(fmt_score(result.score))}</span>)
           elsif result.errored? then %(<span class="badge error">errored</span>)
@@ -650,7 +724,7 @@ module ActiveAgent
 
         <<~CARD
           <div class="result">
-          <div class="head"><span class="name">#{h(short)}</span>#{status}<span class="meta">#{h(result_meta(result))}</span></div>
+          <div class="head"><span class="name">#{h(short)}</span>#{trial}#{status}<span class="meta">#{h(result_meta(result))}</span></div>
           <div class="body"><div class="tools-line"><span class="micro sm">Tools</span>#{html_calls(result, empty: 'none called')}</div>#{fault}#{error}#{answer}</div>
           </div>
         CARD

data/lib/active_agent/evals/report_markdown.rb

@@ -11,8 +11,9 @@ module ActiveAgent
     #
     # Each piece is public on its own, for the pieces people paste one at a
     # time: #fix_item_markdown, #fix_items_markdown (optionally for one
-    # model), #verdict_markdown and #scenario_markdown. The whole report is
-    # built from them, so a piece copied alone reads as it does in the report.
+    # model), #verdict_markdown, #scenario_markdown, #gates_markdown,
+    # #failures_markdown and #failure_markdown. The whole report is built
+    # from them, so a piece copied alone reads as it does in the report.
     #
     # Table cells escape `|` and fold line breaks; text set inside the HTML
     # of a `<details>` block escapes `<`, `>` and `&`.
@@ -25,9 +26,11 @@ module ActiveAgent
         lines = [ "# #{markdown_title}", "" ]
         lines << judged_by_line
         lines << ""
+        lines << trials_line << "" if trials > 1
         lines.concat(summary_table)
         lines << ""
         lines.concat(total_lines)
+        lines.concat(gates_markdown).push("") if pass_thresholds && gates.any?
         lines.concat(verdict_lines)
         lines.concat(comparison_lines)
         lines.concat(matrix_table)
@@ -84,6 +87,47 @@ module ActiveAgent
         lines.join("\n")
       end
 
+      # Pass rate by group: one row per Report#gates row, in that order —
+      # the group, the model, the passes, the pass rate with its mark (✅
+      # met, ❌ not met, — not gated), the bar and the group's mean score
+      # for that model (the whole report's for an "All" row).
+      def gates_markdown(thresholds = pass_thresholds || PassThresholds.default)
+        lines = [ "## Pass rate by group", "", "| Group | Model | Passed | Pass rate | Bar | Mean score |", "|---|---|---|---|---|---|" ]
+        gates(thresholds).each do |gate|
+          stats = gate.group ? summary_by_group.dig(gate.group, "models", gate.label) : summary_by_model[gate.label]
+          mark = gate.gated ? (gate.met? ? "✅" : "❌") : "—"
+          cells = [ cell(gate.name), cell("`#{gate.label}`"), "#{gate.passed} of #{gate.total}", "#{Format.percent(gate.pass_rate)} #{mark}",
+                    gate.gated ? Format.percent(gate.min_pass_rate) : "not gated", Format.score(stats&.dig("avg_score")) ]
+          lines << "| #{cells.join(' | ')} |"
+        end
+        lines
+      end
+
+      # Every result that did not pass (Report#failures), one
+      # #failure_markdown line each, narrowed to one group when given.
+      def failures_markdown(group: nil)
+        failed = failures(group: group)
+        return "_Nothing failed._" if failed.empty?
+
+        failed.map { |result| failure_markdown(result) }.join("\n")
+      end
+
+      # One result that did not pass, as a line naming its scenario and
+      # model with its fault and summary — without a diagnosis, the replay's
+      # error for an errored result and the status alone otherwise. When the
+      # report holds more than one replay of that scenario × model, the line
+      # names the trial as #scenario_markdown does ("`gpt-5-mini` · trial 2"),
+      # so each line points at one answer.
+      def failure_markdown(result)
+        detail =
+          if result.fault then [ result.fault.tr("_", " "), result.summary.presence ].compact.join(" — ")
+          elsif result.errored? then [ "errored", result.replay.error.presence ].compact.join(" — ")
+          else result.status
+          end
+        trial = " · trial #{result.trial}" if replay_counts[[ result.scenario.key, result.label ]].to_i > 1
+        "- `#{result.scenario.key}` on `#{result.label}`#{trial}: #{detail}"
+      end
+
       # The verdict — the best model, why, and who picked it — then the
       # per-model table and, comparing models, the comparison with its
       # intervals.
@@ -108,7 +152,10 @@ module ActiveAgent
 
         scenario = cohort.first.scenario
         labels = @models.map(&:label)
-        cohort = cohort.sort_by { |result| labels.index(result.label) || labels.size }
+        cohort = cohort.sort_by { |result| [ labels.index(result.label) || labels.size, result.trial ] }
+        # A model that replayed the scenario more than once names each trial.
+        repeated = cohort.map(&:label).tally.select { |_label, count| count > 1 }
+        name = ->(result) { repeated.key?(result.label) ? "`#{result.label}` · trial #{result.trial}" : "`#{result.label}`" }
         body = []
         body << "**Prompt**" << "" << quoted(scenario.prompt).join("\n") << ""
         expectations = expectation_lines(scenario)
@@ -116,12 +163,12 @@ module ActiveAgent
         body << "| Model | Status | Score | Tools | Fault | Cost |" << "|---|---|---|---|---|---|"
         cohort.each do |result|
           tools = result.replay.tool_calls.map { |call| "#{call['name']}#{' ✗' if call['error']}" }.join(", ")
-          body << "| #{[ "`#{result.label}`", status_text(result), (result.score ? Format.score(result.score) : '—'), tools.presence || '—',
+          body << "| #{[ name.call(result), status_text(result), (result.score ? Format.score(result.score) : '—'), tools.presence || '—',
                          result.fault&.tr('_', ' ') || '—', (result.replay.cost ? answer_cost(result) : '—') ].map { |text| cell(text) }.join(' | ')} |"
         end
         body << ""
         cohort.each do |result|
-          body << "**`#{result.label}`** — #{status_text(result)}"
+          body << "**#{name.call(result)}** — #{status_text(result)}"
           body << ""
           body << "Fault: #{result.fault.tr('_', ' ')} — #{[ result.summary, result.recommendation ].compact.join(' ')}" << "" if result.fault
           body << "Error: #{result.replay.error}" << "" if result.replay.error
@@ -132,7 +179,8 @@ module ActiveAgent
         text = body.join("\n").rstrip
         return "### Scenario `#{scenario.key}`\n\n#{text}" unless details
 
-        marks = cohort.map { |result| "#{status_mark(result)} #{result.label}" }.join(" · ")
+        # A model that replayed the scenario shows one mark per trial, run together.
+        marks = cohort.group_by(&:label).map { |label, replays| "#{replays.map { |result| status_mark(result) }.join} #{label}" }.join(" · ")
         summary = "<code>#{html_text(scenario.key)}</code> #{html_text(scenario.prompt.truncate(80))} — #{html_text(marks)}"
         "<details>\n<summary>#{summary}</summary>\n\n#{text}\n\n</details>"
       end
@@ -148,11 +196,20 @@ module ActiveAgent
         @results.map { |result| result.scenario.key }.uniq
       end
 
+      # How many results the report holds per scenario key × model label.
+      def replay_counts
+        @replay_counts ||= @results.map { |result| [ result.scenario.key, result.label ] }.tally
+      end
+
       def judged_by_line
         label = @judge_label || @judge&.label
         label ? "Judged by `#{label}`." : "No judge; scored on rules and expectations alone."
       end
 
+      def trials_line
+        "Each scenario was replayed #{trials} times; a model's pass rate counts every replay."
+      end
+
       # "**Best model: gpt-5-mini**" with the verdict's rationale under it
       # and, when a judge (not the pass-rate ranking) wrote it, who did.
       def verdict_lines

data/lib/active_agent/evals/result.rb

@@ -7,12 +7,19 @@ module ActiveAgent
     #
     # `status` is "errored" when the run raised, "failed" when a fault was
     # assigned, and "passed" otherwise. `diagnosis` is the Diagnosis::Result
-    # hash, with a `"judge"` sub-hash when a Judge refined it.
-    Result = Struct.new(:scenario, :spec, :replay, :scores, :score, :status, :diagnosis, keyword_init: true) do
+    # hash, with a `"judge"` sub-hash when a Judge refined it. `trial` numbers
+    # the replays of one scenario under one model from 1 when the Runner
+    # replays each more than once (Runner.new trials:).
+    Result = Struct.new(:scenario, :spec, :replay, :scores, :score, :status, :diagnosis, :trial, keyword_init: true) do
       def passed?
         status == "passed"
       end
 
+      # A Result built without a trial is the first.
+      def trial
+        self[:trial] || 1
+      end
+
       def failed?
         status == "failed"
       end
@@ -88,6 +95,7 @@ module ActiveAgent
           "label" => label,
           "provider" => provider,
           "model" => model,
+          "trial" => trial,
           "status" => status,
           "score" => score,
           "scores" => scores,

data/lib/active_agent/evals/rspec.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+# Loaded by an explicit `require "active_agent/evals/rspec"` — never by
+# `require "active_agent/evals"` — so a host on another test framework never
+# sees it. It needs nothing from rspec: the matcher speaks RSpec's matcher
+# protocol on its own and loads with rspec absent.
+require_relative "../evals"
+
+module ActiveAgent
+  module Evals
+    # `expect(report).to meet_its_pass_gates`: an RSpec matcher over
+    # GateCheck, failing with its message, as TestHelper's
+    # `assert_meets_pass_gates` does. Include it beside TestHelper where the
+    # evals are set up, tagged and run only when EVALS is on:
+    #
+    #   # spec/rails_helper.rb
+    #   require "active_agent/evals/test_helper"
+    #   require "active_agent/evals/rspec"
+    #
+    #   RSpec.configure do |config|
+    #     config.expect_with :rspec, :minitest
+    #     config.define_derived_metadata(file_path: %r{/spec/evals/}) { |metadata| metadata[:eval] = true }
+    #     config.include ActiveAgent::Evals::TestHelper, eval: true
+    #     config.include ActiveAgent::Evals::RSpec, eval: true
+    #     config.filter_run_excluding eval: true unless ActiveAgent::Evals::TestHelper.enabled?
+    #   end
+    #
+    #   # spec/evals/support_desk_spec.rb
+    #   RSpec.describe "the support desk suite" do
+    #     it "clears its pass gates" do
+    #       report = run_eval_suite("config/evals/support_desk.yml") { |scenario, _spec| SupportAgent.with(question: scenario.prompt).help }
+    #
+    #       expect(report).to meet_its_pass_gates
+    #     end
+    #   end
+    module RSpec
+      # The matcher: every gate of the report met under `thresholds`, or
+      # without them under the report's own bars, else the defaults.
+      def meet_its_pass_gates(thresholds = nil)
+        MeetItsPassGates.new(thresholds)
+      end
+
+      # The matcher object, speaking RSpec's matcher protocol (`matches?`,
+      # the failure messages and `description`) without rspec loaded.
+      class MeetItsPassGates
+        def initialize(thresholds = nil)
+          @thresholds = thresholds
+        end
+
+        # Checks `report` and keeps the check for the failure messages.
+        def matches?(report)
+          @check = GateCheck.new(report, @thresholds)
+          @check.met?
+        end
+
+        # GateCheck#message: the unmet gates and the failures to read.
+        def failure_message
+          @check.message
+        end
+
+        # For `expect(report).not_to meet_its_pass_gates`: every gate, each
+        # met, as Gate#to_s.
+        def failure_message_when_negated
+          [ "expected the report not to meet its pass gates, but every gate was met:", *@check.gates.map(&:to_s) ].join("\n")
+        end
+
+        # The generated example description: "meet its pass gates", or
+        # "meet the given pass gates" when thresholds were passed.
+        def description
+          @thresholds ? "meet the given pass gates" : "meet its pass gates"
+        end
+
+        # RSpec shows no diff: the failure message already lists the unmet
+        # gates.
+        def diffable?
+          false
+        end
+
+        # The matcher takes a report, not a block, so RSpec refuses
+        # `expect { ... }.to meet_its_pass_gates`.
+        def supports_block_expectations?
+          false
+        end
+      end
+    end
+  end
+end

data/lib/active_agent/evals/ruby_llm.rb

@@ -55,7 +55,13 @@ module ActiveAgent
         # `chat_options` go to `chat` itself, so they must be keywords
         # `RubyLLM::Chat.new` takes (`protocol:` on RubyLLM 2.x); anything else
         # raises ArgumentError here rather than failing every judge call
-        # later. Configure the chat through `temperature:` or `configure:`:
+        # later. With `verify: true`, a missing key fails here too: the judge's
+        # label, model and provider go through `verify!` with `context` and
+        # `providers`, so a judge whose provider has no key, or is not one of
+        # the providers a host has keys for, refuses when it is built rather
+        # than at its first grade, after the agent has already been replayed;
+        # the default stays lazy. Configure the chat through `temperature:` or
+        # `configure:`:
         #
         #   ActiveAgent::Evals::RubyLLM.judge(label: "judge", model: "gpt-5-mini", provider: :openai,
         #                                     temperature: 0,
@@ -74,12 +80,23 @@ module ActiveAgent
         #   `{ "kind", "model", "input_tokens", "output_tokens", "cost" }`, for a
         #   host that meters its judge's spend; an error it raises is logged,
         #   not propagated, so bookkeeping never costs a grade
+        # @param verify [Boolean] run `verify!` on the judge's own model with
+        #   `context` when the judge is built, so a provider with no key
+        #   refuses here rather than at the first grade
+        # @param providers [Array<String, Symbol>, nil] with `verify: true`, the
+        #   providers a host has keys for, handed to `verify!` as its
+        #   `providers:`; given without `verify: true` it raises ArgumentError
         # @param chat_options [Hash] other `chat` keywords, such as `protocol:`
         # @return [Judge]
-        # @raise [ArgumentError] when `chat_options` names a keyword `RubyLLM::Chat.new` does not take
+        # @raise [ArgumentError] when `chat_options` names a keyword `RubyLLM::Chat.new` does not take,
+        #   or `providers:` is given without `verify: true`
+        # @raise [::RubyLLM::ConfigurationError] with `verify: true`, when the judge's provider is not
+        #   among `providers` or `context` cannot build the judge's chat
         def judge(label:, model:, provider:, context: ::RubyLLM, correlation: nil, assume_model_exists: true,
-                  temperature: nil, configure: nil, on_usage: nil, **chat_options)
+                  temperature: nil, configure: nil, on_usage: nil, verify: false, providers: nil, **chat_options)
           check_chat_options!(chat_options)
+          raise ArgumentError, "providers: is checked only with verify: true" if providers && !verify
+          verify!(ModelSpec.new(label: label, provider: provider, model: model), context: context, providers: providers) if verify
 
           Judge.new(label: label) do |instructions:, prompt:, kind:|
             complete = lambda do
@@ -96,6 +113,65 @@ module ActiveAgent
           end
         end
 
+        # Checks, before a run spends anything, that each model can be built
+        # on `context`: its provider is one RubyLLM registered, is one of
+        # `providers` when those are given, and is configured, which
+        # `context.chat(model:, provider:, assume_model_exists: true)` settles
+        # while it instantiates the provider. Nothing is sent over the
+        # network: RubyLLM resolves the model locally. The specs are checked
+        # in order, and the first that cannot run raises, naming what to
+        # change: the provider's configuration requirements that `context`
+        # leaves unset, as environment variable names, or RubyLLM's own
+        # message when those are all set and it refused for another reason,
+        # such as a credential it checks beyond them or a deployment it
+        # cannot resolve:
+        #
+        #   context = RubyLLM.context { |config| config.openai_api_key = ENV["OPENAI_API_KEY"] }
+        #   models = ActiveAgent::Evals::ModelSpec.parse_all(%w[gpt-5-mini claude-sonnet-5], default_provider: "openai")
+        #
+        #   ActiveAgent::Evals::RubyLLM.verify!(models, context: context)
+        #   # raises RubyLLM::ConfigurationError: claude-sonnet-5 runs on anthropic, which has no API key. Set ANTHROPIC_API_KEY.
+        #
+        # @param models [Array<ModelSpec>, ModelSpec] the run's models, or anything
+        #   answering `label`, `provider` and `model`; a single spec is wrapped
+        # @param context [#chat] where `chat` is called; defaults to `::RubyLLM`.
+        #   When it also answers `config`, as `::RubyLLM` and a `RubyLLM.context`
+        #   do, a refusal names only the requirements that configuration leaves
+        #   unset
+        # @param providers [Array<String, Symbol>, String, Symbol, nil] the
+        #   providers the host has credentials for; a model on any other provider
+        #   is refused before its chat is built
+        # @return [Array<ModelSpec>] the given models
+        # @raise [::RubyLLM::ConfigurationError] for the first model whose provider
+        #   RubyLLM does not know, is outside `providers`, or cannot be built on
+        #   `context`; in the last case RubyLLM's own error is the cause, and the
+        #   message names what to set or carries RubyLLM's own
+        def verify!(models, context: ::RubyLLM, providers: nil)
+          specs = models.is_a?(Array) ? models : [ models ]
+          configured = Array(providers).map(&:to_s) if providers
+
+          specs.each do |spec|
+            provider = spec.provider.to_s
+            unless ::RubyLLM::Provider.providers.key?(provider.to_sym)
+              raise ::RubyLLM::ConfigurationError,
+                    "#{spec.label} runs on #{provider}, which is not a RubyLLM provider; " \
+                    "RubyLLM knows #{::RubyLLM::Provider.providers.keys.map(&:to_s).sort.join(', ')}"
+            end
+            if configured && !configured.include?(provider)
+              configured_for = configured.empty? ? "no provider has credentials configured" : "credentials are configured for #{configured.join(', ')}"
+              raise ::RubyLLM::ConfigurationError, "#{spec.label} runs on #{provider}; #{configured_for}"
+            end
+
+            begin
+              context.chat(model: spec.model, provider: provider.to_sym, assume_model_exists: true)
+            rescue ::RubyLLM::ConfigurationError => e
+              raise ::RubyLLM::ConfigurationError, refusal(spec, provider, context, e)
+            end
+          end
+
+          specs
+        end
+
         # Builds a Replay from the ordered messages of a RubyLLM conversation:
         # `acts_as_chat` message records of either RubyLLM generation (read
         # through their own `to_llm`), `RubyLLM::Message` values, or anything
@@ -167,6 +243,32 @@ module ActiveAgent
                 "Pass temperature: to judge, or configure the chat with configure: ->(chat) { chat.with_... }."
         end
 
+        # The message for a model whose provider refused to be built: the
+        # requirement names `context` leaves unset, as environment variable
+        # names, called an API key only when each of them is one; RubyLLM's
+        # own message when nothing is left to set.
+        def refusal(spec, provider, context, error)
+          names = unset_requirements(provider, context).map { |name| name.to_s.upcase }
+          return "#{spec.label} runs on #{provider}: #{error.message.strip}" if names.empty?
+
+          state = names.all? { |name| name.end_with?("_API_KEY") } ? "has no API key" : "is not configured"
+          "#{spec.label} runs on #{provider}, which #{state}. Set #{names.join(' and ')}."
+        end
+
+        # The provider's configuration requirements that `context` leaves
+        # unset: all of them when the context exposes no configuration, and
+        # none on a release whose providers do not list them.
+        def unset_requirements(provider, context)
+          provider_class = ::RubyLLM::Provider.providers[provider.to_sym]
+          return [] unless provider_class.respond_to?(:configuration_requirements)
+
+          requirements = provider_class.configuration_requirements
+          return requirements unless context.respond_to?(:config)
+
+          config = context.config
+          requirements.reject { |name| config.respond_to?(name) && config.public_send(name) }
+        end
+
         def report_usage(callback, kind, model, response)
           return unless callback