activeagent 1.9.6 → 1.9.7

data/lib/active_agent/evals/report.rb

@@ -14,7 +14,33 @@ module ActiveAgent
       # pass rate itself, no judge having been available to rule on them.
       PASS_RATE_JUDGE = "pass rate"
 
-      attr_reader :results, :models, :judge, :judge_label, :instructions, :metadata, :agent_name, :links, :tool_resolver
+      attr_reader :results, :models, :judge, :judge_label, :instructions, :metadata, :agent_name, :links, :tool_resolver,
+                  :pass_thresholds
+
+      # One group's (or the whole report's, with group nil and name "All")
+      # pass rate under one model against the bar the suite set for it.
+      # `pass_rate` and `min_pass_rate` are 0..1 fractions, so they compare
+      # directly and render with Format.percent; an ungated row is always met.
+      Gate = Struct.new(:group, :name, :label, :passed, :total, :pass_rate, :min_pass_rate, :gated, keyword_init: true) do
+        def met?
+          !gated || pass_rate >= min_pass_rate
+        end
+
+        def to_h
+          super.transform_keys(&:to_s).merge("met" => met?)
+        end
+
+        # "Order support · gpt-5-mini: 1/2 (50%) passed, below 100%" — one
+        # sentence a failing assertion can print.
+        def to_s
+          bar =
+            if !gated then "not gated"
+            elsif met? then "at or above #{Format.percent(min_pass_rate)}"
+            else "below #{Format.percent(min_pass_rate)}"
+            end
+          "#{name} · #{label}: #{Format.passes(passed, total, style: :markdown)} passed, #{bar}"
+        end
+      end
 
       # @param tool_resolver [#call, nil] maps a tool name to the MCP server
       #   that provides it — `{ "key", "name", "status" }` with status
@@ -41,15 +67,22 @@ module ActiveAgent
       # @param comparison [Hash, nil] a comparison already recorded for these
       #   results (Comparison#to_h). A report rebuilt from a persisted run
       #   renders the intervals that run recorded instead of resampling them.
-      def initialize(results:, models:, judge: nil, instructions: nil, threshold: PASS_THRESHOLD, metadata: {},
+      # @param threshold [Float, nil] the score an answer must reach to pass
+      #   a criterion; an explicit value wins over the `pass_thresholds` bar,
+      #   and without either it is PASS_THRESHOLD
+      # @param pass_thresholds [PassThresholds, nil] the bars the suite
+      #   declared: its threshold applies when none is given here, #gates
+      #   and #passed? read them, and `to_h` carries them with the per-group
+      #   summary and the gates.
+      def initialize(results:, models:, judge: nil, instructions: nil, threshold: nil, metadata: {},
                      tool_resolver: nil, agent_name: nil, links: {}, verdict: nil, judge_label: nil,
-                     judge_usage: nil, release: nil, comparison: nil)
+                     judge_usage: nil, release: nil, comparison: nil, pass_thresholds: nil)
         @results = results
         @models = models
         @judge = judge
         @judge_label = judge_label.presence
         @instructions = instructions
-        @threshold = threshold
+        @threshold = threshold || pass_thresholds&.threshold || PASS_THRESHOLD
         @metadata = metadata
         @tool_resolver = tool_resolver
         @agent_name = agent_name.presence || "the agent"
@@ -58,6 +91,7 @@ module ActiveAgent
         @run_judge_usage = judge_usage.is_a?(Hash) ? judge_usage.to_h.stringify_keys.presence : nil
         @release = release.is_a?(Hash) ? release.to_h.stringify_keys.compact.presence : nil
         @recorded_comparison = comparison.is_a?(Hash) ? comparison.to_h.deep_stringify_keys.presence : nil
+        @pass_thresholds = pass_thresholds
       end
 
       # The release of the agent this report scored, `{ "digest",
@@ -68,12 +102,20 @@ module ActiveAgent
         @models.size > 1
       end
 
-      # Per model, keyed by label: scenario count, passes, errors, pass rate,
-      # mean score, latency (mean, and the P50 and P95 by nearest rank,
-      # Statistics.percentile), tokens, cost and fault counts. "cost" sums
-      # the replays that carried a cost — "priced" of the "scenarios", of
-      # which "reported" came from the caller and "estimated" from tokens ×
-      # a model rate (Result#cost_source) — and is nil when none did.
+      # How many times each scenario was replayed under each model: the
+      # highest trial number any result carries, 1 for a run with none.
+      def trials
+        @results.map(&:trial).max || 1
+      end
+
+      # Per model, keyed by label: replay count ("scenarios", every trial of
+      # every scenario), distinct scenarios ("unique_scenarios"), passes,
+      # errors, pass rate over every replay, mean score, latency (mean, and
+      # the P50 and P95 by nearest rank, Statistics.percentile), tokens,
+      # cost and fault counts. "cost" sums the replays that carried a cost
+      # — "priced" of the "scenarios", of which "reported" came from the
+      # caller and "estimated" from tokens × a model rate
+      # (Result#cost_source) — and is nil when none did.
       # "judge_cost" and "judge_calls" sum what the judge spent on the
       # cohort's results (Result#judge_usage); the judge's cost never joins
       # the agent's.
@@ -90,6 +132,7 @@ module ActiveAgent
             "provider" => spec.provider,
             "model" => spec.model,
             "scenarios" => cohort.size,
+            "unique_scenarios" => cohort.map { |result| result.scenario.key }.uniq.size,
             "passed" => cohort.count(&:passed?),
             "errored" => cohort.count(&:errored?),
             "pass_rate" => cohort.any? ? (cohort.count(&:passed?) * 100.0 / cohort.size).round(1) : 0.0,
@@ -111,6 +154,58 @@ module ActiveAgent
         end
       end
 
+      # Per group key, in run order, then per model under "models" (keyed by
+      # label, only the models with a result in the group): scenario count,
+      # passes, errors, pass rate as a percent, mean score and fault counts,
+      # counted as summary_by_model counts them. "name" is the group's
+      # display name, or its key. Scenarios with no group are left out.
+      def summary_by_group
+        @summary_by_group ||= begin
+          grouped = @results.select { |result| result.scenario.group.present? }.group_by { |result| result.scenario.group }
+          grouped.to_h do |group, cohort|
+            models = @models.filter_map do |spec|
+              rows = cohort.select { |result| result.label == spec.label }
+              [ spec.label, cohort_summary(rows) ] if rows.any?
+            end
+            [ group, { "name" => group_name(cohort.first.scenario), "models" => models.to_h } ]
+          end
+        end
+      end
+
+      # One Gate per group and model with results there, in summary_by_group
+      # order, then one per model for the whole report (group nil, name
+      # "All") over every gated result of that model, ungrouped results
+      # included; a model with no gated result has no "All" row. Each row
+      # carries the bar `thresholds` sets for its group.
+      def gates(thresholds = pass_thresholds || PassThresholds.default)
+        rows = summary_by_group.flat_map do |group, summary|
+          summary["models"].map do |label, stats|
+            gate(group, summary["name"], label, stats["passed"], stats["scenarios"],
+                 thresholds.min_pass_rate(group), thresholds.gated?(group))
+          end
+        end
+        gated = @results.select { |result| thresholds.gated?(result.scenario.group) }
+        rows + @models.filter_map do |spec|
+          cohort = gated.select { |result| result.label == spec.label }
+          gate(nil, "All", spec.label, cohort.count(&:passed?), cohort.size, thresholds.min_pass_rate(nil), true) if cohort.any?
+        end
+      end
+
+      # Whether every gate is met. A report with no gate, because no result
+      # reached it (a selection that matched nothing, every scenario left
+      # out as production_only), has shown nothing and does not pass.
+      def passed?(thresholds = pass_thresholds || PassThresholds.default)
+        rows = gates(thresholds)
+        rows.any? && rows.all?(&:met?)
+      end
+
+      # The results that did not pass — failed and errored — in run order,
+      # narrowed to one group when given.
+      def failures(group: nil)
+        failed = @results.reject(&:passed?)
+        group.nil? ? failed : failed.select { |result| result.scenario.group.to_s == group.to_s }
+      end
+
       # Per scenario key, in run order: what every model's answer cost
       # together — the agent's cost summed over the models, the judge's, and
       # the two as "total" — plus the same per model under "models", keyed
@@ -258,7 +353,7 @@ module ActiveAgent
             cost = " at #{Format.money(stats['cost'], estimated: estimated_cost?(stats))}"
             cost += " (estimated)" if estimated_cost?(stats)
           end
-          rationale = "Passed #{stats['passed']} of #{stats['scenarios']} scenarios" \
+          rationale = "Passed #{stats['passed']} of #{counted_scenarios(stats)}" \
             " (#{Format.percent(stats['scenarios'].to_i.positive? ? stats['passed'].to_f / stats['scenarios'] : nil)})" \
             "#{" with a mean score of #{Format.score(stats['avg_score'])}" if stats['avg_score']}#{cost}."
           judged = @judge&.verdict(summary_by_model, instructions: @instructions)
@@ -286,21 +381,28 @@ module ActiveAgent
         @comparison ||= @recorded_comparison || Comparison.new(results: @results, models: @models).to_h
       end
 
-      # Adds "judge_usage" and "release" only when there is one to add, and
-      # "comparison" only when comparing models, so a single-model report
-      # built the way it always was serializes exactly as before.
+      # Adds "judge_usage" and "release" only when there is one to add,
+      # "comparison" only when comparing models, "trials" only when a scenario
+      # was replayed more than once, and "pass_thresholds" with the per-group
+      # summary ("groups") and the "gates" only when the suite declared its
+      # bars, so a single-model, single-trial report built the way it always
+      # was serializes exactly as before.
       def to_h
         {
           "models" => summary_by_model,
+          "trials" => (trials if trials > 1),
           "criteria" => criterion_scores,
           "recommendations" => recommendations,
           "verdict" => verdict,
           "comparison" => comparison,
+          "pass_thresholds" => pass_thresholds&.to_h&.merge("threshold" => @threshold),
+          "groups" => (summary_by_group if pass_thresholds),
+          "gates" => (gates.map(&:to_h) if pass_thresholds),
           "judge" => @judge_label || @judge&.label,
           "judge_usage" => judge_usage,
           "release" => release,
           "metadata" => @metadata.presence,
-          "results" => @results.map(&:to_h)
+          "results" => serialized_results
         }.compact
       end
 
@@ -310,6 +412,14 @@ module ActiveAgent
 
       private
 
+      # Each result's hash; a single-trial report leaves out every "trial",
+      # so it is the JSON an earlier gem wrote and a re-delivery of it under
+      # the same run_id matches the stored digest.
+      def serialized_results
+        hashes = @results.map(&:to_h)
+        trials > 1 ? hashes : hashes.map { |hash| hash.except("trial") }
+      end
+
       # Whether a model summary's cost is an estimate — any replay priced
       # from tokens, or some replay unpriced, which leaves the sum a lower
       # bound — and so reads with a "~".
@@ -339,10 +449,37 @@ module ActiveAgent
         stats["cost"].to_f / priced if stats["cost"] && priced.positive?
       end
 
+      # The denominator a verdict names: "6 scenarios", or "18 replays of 6
+      # scenarios" when the model replayed each more than once.
+      def counted_scenarios(stats)
+        replays = stats["scenarios"].to_i
+        unique = stats["unique_scenarios"].to_i
+        replays > unique ? "#{replays} replays of #{unique} scenarios" : "#{replays} scenarios"
+      end
+
       def criterion_keys
         @results.flat_map { |result| result.scores.keys }.uniq
       end
 
+      # One model's counts over the results of a group, the way
+      # summary_by_model counts a model's whole cohort.
+      def cohort_summary(cohort)
+        scored = cohort.filter_map(&:score)
+        {
+          "scenarios" => cohort.size,
+          "passed" => cohort.count(&:passed?),
+          "errored" => cohort.count(&:errored?),
+          "pass_rate" => cohort.any? ? (cohort.count(&:passed?) * 100.0 / cohort.size).round(1) : 0.0,
+          "avg_score" => scored.any? ? (scored.sum / scored.size).round(3) : nil,
+          "faults" => cohort.filter_map(&:fault).tally
+        }
+      end
+
+      def gate(group, name, label, passed, total, min_pass_rate, gated)
+        Gate.new(group: group, name: name, label: label, passed: passed, total: total,
+                 pass_rate: passed.to_f / total, min_pass_rate: min_pass_rate, gated: gated)
+      end
+
       def stats_for(values)
         scored = values.compact
         return { "skipped" => true, "reason" => "No scorable answers" } if scored.empty?
@@ -628,17 +765,25 @@ module ActiveAgent
         header = [ "| Scenario | #{labels.map { |label| cell("`#{label}`") }.join(' | ')} | Cost |", "|---|#{labels.map { '---' }.join('|')}|---|" ]
         rows = @results.group_by { |result| result.scenario.key }.map do |key, cohort|
           cells = labels.map do |label|
-            result = cohort.find { |candidate| candidate.label == label }
-            next "—" unless result
-
-            mark = result.passed? ? "✅" : (result.errored? ? "⚠️" : "❌")
-            cell([ mark, (Format.score(result.score) if result.score), result.fault&.tr("_", " ") ].compact.join(" "))
+            mine = cohort.select { |candidate| candidate.label == label }.sort_by(&:trial)
+            mine.empty? ? "—" : cell(matrix_cell_text(mine))
           end
           "| #{cell("`#{key}` #{cohort.first.scenario.prompt.truncate(70)}")} | #{cells.join(' | ')} | #{scenario_cost_cell(key)} |"
         end
         header + rows
       end
 
+      # "✅ 100%" or "❌ 50% missing capability" for one result; for a
+      # scenario replayed more than once, every trial's mark run together
+      # ("✅✅❌"), the mean of the trials' scores, and the fault seen most
+      # often among them (a tie goes to the first seen).
+      def matrix_cell_text(results)
+        marks = results.map { |result| status_mark(result) }.join
+        scored = results.filter_map(&:score)
+        fault = results.filter_map(&:fault).tally.max_by(&:last)&.first
+        [ marks, (Format.score(scored.sum.fdiv(scored.size)) if scored.any?), fault&.tr("_", " ") ].compact.join(" ")
+      end
+
       # "~$0.0243 (judge ~$0.0015)" for a scenario's total across models.
       def scenario_cost_cell(key)
         costs = scenario_costs[key] || {}

data/lib/active_agent/evals/report_html.rb

@@ -215,7 +215,13 @@ module ActiveAgent
       # --- page ------------------------------------------------------------
 
       def html_title
-        "Evaluation — #{plural(scenario_cohorts.size, 'scenario')} × #{plural(@models.size, 'model')}"
+        with_trials("Evaluation — #{plural(scenario_cohorts.size, 'scenario')} × #{plural(@models.size, 'model')}")
+      end
+
+      # +text+ with "3 trials per scenario" beside it when each scenario was
+      # replayed more than once; +text+ alone otherwise.
+      def with_trials(text)
+        trials > 1 ? "#{text} · #{trials} trials per scenario" : text
       end
 
       def html_styles
@@ -282,7 +288,7 @@ module ActiveAgent
         passed = @results.count(&:passed?)
         ratio = total.positive? ? passed.to_f / total : 0.0
         tiles = [
-          html_tile("Scenario runs", total, "#{plural(scenario_cohorts.size, 'scenario')} × #{plural(@models.size, 'model')}"),
+          html_tile("Scenario runs", total, with_trials("#{plural(scenario_cohorts.size, 'scenario')} × #{plural(@models.size, 'model')}")),
           html_tile("Pass rate", Format.percent(total.positive? ? ratio : nil), "#{passed}/#{total} passed", tone: tone_for(ratio)),
           html_tile("Open faults", total - passed, plural(fix_items.size, "fix item")),
           html_cost_tile,
@@ -496,29 +502,49 @@ module ActiveAgent
 
         <<~MATRIX
           <section class="section">
-          <div class="section-head"><span class="micro">Scenarios</span><span class="meta">#{h(meta)}</span></div>
+          <div class="section-head"><span class="micro">Scenarios</span><span class="meta">#{h(with_trials(meta))}</span></div>
           <div class="matrix"><div class="inner">#{rows.join}</div></div>
           </section>
         MATRIX
       end
 
+      # A group's row: each model's passes over every replay of the row's
+      # scenarios, toned against the group's bar when the suite declared one —
+      # the row's own pass rate at or above it green, below it red, an ungated
+      # group plain, with the bar as the tooltip — and otherwise green when
+      # every replay passed and red when none did. A row with a group name but
+      # no group key has no bar to read.
       def html_group_row(cohorts)
-        name = group_name(cohorts.first.first.scenario)
+        scenario = cohorts.first.first.scenario
+        name = group_name(scenario)
+        gated = pass_thresholds && scenario.group.present?
+        gates_by_label = gated ? gates.select { |gate| gate.group == scenario.group }.index_by(&:label) : {}
         passes = @models.map do |spec|
-          results = cohorts.filter_map { |cohort| cohort.find { |result| result.label == spec.label } }
+          results = cohorts.flat_map { |cohort| cohort.select { |result| result.label == spec.label } }
+          next %(<span class="group-pass">—</span>) if results.empty?
+
           passed = results.count(&:passed?)
-          tone =
-            if results.empty? then ""
-            elsif passed == results.size then " text-success"
-            elsif passed.zero? then " text-error"
-            else ""
-            end
-          %(<span class="group-pass#{tone}">#{results.empty? ? '—' : "#{h(fmt_passes(passed, results.size))} passed"}</span>)
+          tone, title = html_group_tone(passed, results.size, gates_by_label[spec.label])
+          %(<span class="group-pass#{tone}"#{title}>#{h(fmt_passes(passed, results.size))} passed</span>)
         end
         %(<div class="mx group"><span class="group-name">#{h(name)}</span>) +
           %(<span class="count">#{h(plural(cohorts.size, 'scenario'))}</span>#{passes.join}#{html_group_cost(cohorts)}</div>)
       end
 
+      # [tone class, title attribute] for a group-pass span: the row's own
+      # counts against its gate's bar when there is one, so a group split
+      # across the run tones each row by what that row shows; else from the
+      # counts alone, with no title.
+      def html_group_tone(passed, total, gate)
+        if gate
+          tone = gate.gated ? (passed.to_f / total >= gate.min_pass_rate ? " text-success" : " text-error") : ""
+          [ tone, %( title="#{h(gate.gated ? "pass rate ≥ #{Format.percent(gate.min_pass_rate)}" : 'not gated')}") ]
+        elsif passed == total then [ " text-success", "" ]
+        elsif passed.zero? then [ " text-error", "" ]
+        else [ "", "" ]
+        end
+      end
+
       # The group's subtotal — its scenarios' costs summed, judge apart — in
       # the trailing column, so the grid stays aligned under a group row.
       def html_group_cost(cohorts)
@@ -542,8 +568,11 @@ module ActiveAgent
         scenario = cohort.first.scenario
         expects = scenario.expected_tools.map { |tool| %(<span class="expect">#{h(tool)}</span>) }.join
         cells = @models.map do |spec|
-          result = cohort.find { |candidate| candidate.label == spec.label }
-          result ? html_result_cell(result) : %(<div class="cell"><div class="top"><span class="muted">—</span></div></div>)
+          mine = cohort.select { |candidate| candidate.label == spec.label }.sort_by(&:trial)
+          if mine.empty? then %(<div class="cell"><div class="top"><span class="muted">—</span></div></div>)
+          elsif mine.one? then html_result_cell(mine.first)
+          else html_trials_cell(mine)
+          end
         end
         costs = scenario_costs[scenario.key] || {}
         total_cell = if costs["cost"].nil? && costs["judge_cost"].nil?
@@ -564,6 +593,26 @@ module ActiveAgent
           %(<div class="calls">#{html_calls(result, empty: 'no tools called')}</div>#{html_cell_cost(result)}</div>)
       end
 
+      # The cell of a scenario replayed more than once under one model: how
+      # many replays passed, one line per trial (its number, the glyph, the
+      # score and the fault), and what the replays cost together.
+      def html_trials_cell(results)
+        passed = results.count(&:passed?)
+        tone =
+          if passed == results.size then " tone-success"
+          elsif passed.zero? then " tone-error"
+          else ""
+          end
+        lines = results.map do |result|
+          glyph_tone = result.passed? ? "success" : "error"
+          fault = result.fault ? %(<span class="f">#{h(fault_name(result.fault))}</span>) : ""
+          %(<div class="top trial"><span class="muted">#{result.trial}</span><span class="g tone-#{glyph_tone}">#{result.passed? ? '[+]' : '[!]'}</span>) +
+            %(<span class="s tone-#{glyph_tone}">#{h(fmt_score(result.score))}</span>#{fault}</div>)
+        end
+        %(<div class="cell"><div class="top"><span class="s#{tone}">#{h(fmt_passes(passed, results.size))} passed</span></div>) +
+          %(#{lines.join}#{html_trials_cost(results)}</div>)
+      end
+
       # "~$0.0243 · judge ~$0.0015" under a cell: what this answer cost and
       # what judging it cost. A result with no cost at all shows no line.
       def html_cell_cost(result)
@@ -573,6 +622,25 @@ module ActiveAgent
         %(<div class="cost-line"#{cost_title_attr([ result ], estimated: result.estimated_cost?)}>#{h(text)}</div>)
       end
 
+      # The cost line under a trials cell: the replays' costs summed, the
+      # judge's apart. The sum reads as an estimate when any replay's cost
+      # is one, or when a replay went unpriced (a lower bound), and shows its
+      # working in a tooltip only when a replay was priced from tokens.
+      # Nothing when no replay carried a cost.
+      def html_trials_cost(results)
+        costs = results.filter_map { |result| result.replay.cost }
+        judge_costs = results.filter_map { |result| result.judge_usage&.dig("cost") }
+        return "" if costs.empty? && judge_costs.empty?
+
+        priced_from_tokens = results.any?(&:estimated_cost?)
+        estimated = priced_from_tokens || (costs.any? && costs.size < results.size)
+        parts = [ fmt_cost(costs.any? ? costs.sum : nil, estimated: estimated) ]
+        if judge_costs.any?
+          parts << "judge #{fmt_cost(judge_costs.sum, estimated: results.any? { |result| estimated_usage?(result.judge_usage) })}"
+        end
+        %(<div class="cost-line"#{cost_title_attr(results, estimated: priced_from_tokens)}>#{h(parts.join(' · '))}</div>)
+      end
+
       def result_cost_text(result)
         judge_cost = result.judge_usage&.dig("cost")
         return nil if result.replay.cost.nil? && judge_cost.nil?
@@ -619,7 +687,10 @@ module ActiveAgent
         scenario = cohort.first.scenario
         expects = scenario.expected_tools.any? ? %(<span class="exp">expects <b>#{h(scenario.expected_tools.join(' or '))}</b></span>) : ""
         labels = @models.map(&:label)
-        cards = cohort.sort_by { |result| labels.index(result.label) || labels.size }.map { |result| html_result_card(result) }
+        # A model that replayed the scenario more than once numbers its cards.
+        repeated = cohort.map(&:label).tally.select { |_label, count| count > 1 }
+        cards = cohort.sort_by { |result| [ labels.index(result.label) || labels.size, result.trial ] }
+                      .map { |result| html_result_card(result, numbered: repeated.key?(result.label)) }
         md_id = "md-#{anchor(scenario)}"
 
         <<~BLOCK
@@ -632,8 +703,11 @@ module ActiveAgent
         BLOCK
       end
 
-      def html_result_card(result)
+      # +numbered+ names the card's trial, for a model with several cards on
+      # one scenario.
+      def html_result_card(result, numbered: false)
         short, = split_label(result.spec)
+        trial = numbered ? %(<span class="micro sm">trial #{result.trial}</span>) : ""
         status =
           if result.passed? then %(<span class="badge success">passed · #{h(fmt_score(result.score))}</span>)
           elsif result.errored? then %(<span class="badge error">errored</span>)
@@ -650,7 +724,7 @@ module ActiveAgent
 
         <<~CARD
           <div class="result">
-          <div class="head"><span class="name">#{h(short)}</span>#{status}<span class="meta">#{h(result_meta(result))}</span></div>
+          <div class="head"><span class="name">#{h(short)}</span>#{trial}#{status}<span class="meta">#{h(result_meta(result))}</span></div>
           <div class="body"><div class="tools-line"><span class="micro sm">Tools</span>#{html_calls(result, empty: 'none called')}</div>#{fault}#{error}#{answer}</div>
           </div>
         CARD

data/lib/active_agent/evals/report_markdown.rb

@@ -11,8 +11,9 @@ module ActiveAgent
     #
     # Each piece is public on its own, for the pieces people paste one at a
     # time: #fix_item_markdown, #fix_items_markdown (optionally for one
-    # model), #verdict_markdown and #scenario_markdown. The whole report is
-    # built from them, so a piece copied alone reads as it does in the report.
+    # model), #verdict_markdown, #scenario_markdown, #gates_markdown,
+    # #failures_markdown and #failure_markdown. The whole report is built
+    # from them, so a piece copied alone reads as it does in the report.
     #
     # Table cells escape `|` and fold line breaks; text set inside the HTML
     # of a `<details>` block escapes `<`, `>` and `&`.
@@ -25,9 +26,11 @@ module ActiveAgent
         lines = [ "# #{markdown_title}", "" ]
         lines << judged_by_line
         lines << ""
+        lines << trials_line << "" if trials > 1
         lines.concat(summary_table)
         lines << ""
         lines.concat(total_lines)
+        lines.concat(gates_markdown).push("") if pass_thresholds && gates.any?
         lines.concat(verdict_lines)
         lines.concat(comparison_lines)
         lines.concat(matrix_table)
@@ -84,6 +87,47 @@ module ActiveAgent
         lines.join("\n")
       end
 
+      # Pass rate by group: one row per Report#gates row, in that order —
+      # the group, the model, the passes, the pass rate with its mark (✅
+      # met, ❌ not met, — not gated), the bar and the group's mean score
+      # for that model (the whole report's for an "All" row).
+      def gates_markdown(thresholds = pass_thresholds || PassThresholds.default)
+        lines = [ "## Pass rate by group", "", "| Group | Model | Passed | Pass rate | Bar | Mean score |", "|---|---|---|---|---|---|" ]
+        gates(thresholds).each do |gate|
+          stats = gate.group ? summary_by_group.dig(gate.group, "models", gate.label) : summary_by_model[gate.label]
+          mark = gate.gated ? (gate.met? ? "✅" : "❌") : "—"
+          cells = [ cell(gate.name), cell("`#{gate.label}`"), "#{gate.passed} of #{gate.total}", "#{Format.percent(gate.pass_rate)} #{mark}",
+                    gate.gated ? Format.percent(gate.min_pass_rate) : "not gated", Format.score(stats&.dig("avg_score")) ]
+          lines << "| #{cells.join(' | ')} |"
+        end
+        lines
+      end
+
+      # Every result that did not pass (Report#failures), one
+      # #failure_markdown line each, narrowed to one group when given.
+      def failures_markdown(group: nil)
+        failed = failures(group: group)
+        return "_Nothing failed._" if failed.empty?
+
+        failed.map { |result| failure_markdown(result) }.join("\n")
+      end
+
+      # One result that did not pass, as a line naming its scenario and
+      # model with its fault and summary — without a diagnosis, the replay's
+      # error for an errored result and the status alone otherwise. When the
+      # report holds more than one replay of that scenario × model, the line
+      # names the trial as #scenario_markdown does ("`gpt-5-mini` · trial 2"),
+      # so each line points at one answer.
+      def failure_markdown(result)
+        detail =
+          if result.fault then [ result.fault.tr("_", " "), result.summary.presence ].compact.join(" — ")
+          elsif result.errored? then [ "errored", result.replay.error.presence ].compact.join(" — ")
+          else result.status
+          end
+        trial = " · trial #{result.trial}" if replay_counts[[ result.scenario.key, result.label ]].to_i > 1
+        "- `#{result.scenario.key}` on `#{result.label}`#{trial}: #{detail}"
+      end
+
       # The verdict — the best model, why, and who picked it — then the
       # per-model table and, comparing models, the comparison with its
       # intervals.
@@ -108,7 +152,10 @@ module ActiveAgent
 
         scenario = cohort.first.scenario
         labels = @models.map(&:label)
-        cohort = cohort.sort_by { |result| labels.index(result.label) || labels.size }
+        cohort = cohort.sort_by { |result| [ labels.index(result.label) || labels.size, result.trial ] }
+        # A model that replayed the scenario more than once names each trial.
+        repeated = cohort.map(&:label).tally.select { |_label, count| count > 1 }
+        name = ->(result) { repeated.key?(result.label) ? "`#{result.label}` · trial #{result.trial}" : "`#{result.label}`" }
         body = []
         body << "**Prompt**" << "" << quoted(scenario.prompt).join("\n") << ""
         expectations = expectation_lines(scenario)
@@ -116,12 +163,12 @@ module ActiveAgent
         body << "| Model | Status | Score | Tools | Fault | Cost |" << "|---|---|---|---|---|---|"
         cohort.each do |result|
           tools = result.replay.tool_calls.map { |call| "#{call['name']}#{' ✗' if call['error']}" }.join(", ")
-          body << "| #{[ "`#{result.label}`", status_text(result), (result.score ? Format.score(result.score) : '—'), tools.presence || '—',
+          body << "| #{[ name.call(result), status_text(result), (result.score ? Format.score(result.score) : '—'), tools.presence || '—',
                          result.fault&.tr('_', ' ') || '—', (result.replay.cost ? answer_cost(result) : '—') ].map { |text| cell(text) }.join(' | ')} |"
         end
         body << ""
         cohort.each do |result|
-          body << "**`#{result.label}`** — #{status_text(result)}"
+          body << "**#{name.call(result)}** — #{status_text(result)}"
           body << ""
           body << "Fault: #{result.fault.tr('_', ' ')} — #{[ result.summary, result.recommendation ].compact.join(' ')}" << "" if result.fault
           body << "Error: #{result.replay.error}" << "" if result.replay.error
@@ -132,7 +179,8 @@ module ActiveAgent
         text = body.join("\n").rstrip
         return "### Scenario `#{scenario.key}`\n\n#{text}" unless details
 
-        marks = cohort.map { |result| "#{status_mark(result)} #{result.label}" }.join(" · ")
+        # A model that replayed the scenario shows one mark per trial, run together.
+        marks = cohort.group_by(&:label).map { |label, replays| "#{replays.map { |result| status_mark(result) }.join} #{label}" }.join(" · ")
         summary = "<code>#{html_text(scenario.key)}</code> #{html_text(scenario.prompt.truncate(80))} — #{html_text(marks)}"
         "<details>\n<summary>#{summary}</summary>\n\n#{text}\n\n</details>"
       end
@@ -148,11 +196,20 @@ module ActiveAgent
         @results.map { |result| result.scenario.key }.uniq
       end
 
+      # How many results the report holds per scenario key × model label.
+      def replay_counts
+        @replay_counts ||= @results.map { |result| [ result.scenario.key, result.label ] }.tally
+      end
+
       def judged_by_line
         label = @judge_label || @judge&.label
         label ? "Judged by `#{label}`." : "No judge; scored on rules and expectations alone."
       end
 
+      def trials_line
+        "Each scenario was replayed #{trials} times; a model's pass rate counts every replay."
+      end
+
       # "**Best model: gpt-5-mini**" with the verdict's rationale under it
       # and, when a judge (not the pass-rate ranking) wrote it, who did.
       def verdict_lines

data/lib/active_agent/evals/result.rb

@@ -7,12 +7,19 @@ module ActiveAgent
     #
     # `status` is "errored" when the run raised, "failed" when a fault was
     # assigned, and "passed" otherwise. `diagnosis` is the Diagnosis::Result
-    # hash, with a `"judge"` sub-hash when a Judge refined it.
-    Result = Struct.new(:scenario, :spec, :replay, :scores, :score, :status, :diagnosis, keyword_init: true) do
+    # hash, with a `"judge"` sub-hash when a Judge refined it. `trial` numbers
+    # the replays of one scenario under one model from 1 when the Runner
+    # replays each more than once (Runner.new trials:).
+    Result = Struct.new(:scenario, :spec, :replay, :scores, :score, :status, :diagnosis, :trial, keyword_init: true) do
       def passed?
         status == "passed"
       end
 
+      # A Result built without a trial is the first.
+      def trial
+        self[:trial] || 1
+      end
+
       def failed?
         status == "failed"
       end
@@ -88,6 +95,7 @@ module ActiveAgent
           "label" => label,
           "provider" => provider,
           "model" => model,
+          "trial" => trial,
           "status" => status,
           "score" => score,
           "scores" => scores,