activeagent 1.9.1 → 1.9.3

data/lib/active_agent/evals/report_html.rb

@@ -5,10 +5,15 @@ require "cgi"
 module ActiveAgent
   module Evals
     # Report#to_html: the run as one self-contained page on the dashboard's
-    # design system (DesignTokens) — inline styles, no scripts, no external
-    # assets — so a run's outcome can be archived next to a CI run or handed
-    # to a teammate, and so the dashboard can serve the same page for a
-    # persisted run.
+    # design system (DesignTokens) — inline styles, no external assets — so a
+    # run's outcome can be archived next to a CI run or handed to a teammate,
+    # and so the dashboard can serve the same page for a persisted run.
+    #
+    # Its one inline script (COPY_SCRIPT) backs the "Copy as Markdown"
+    # buttons: the whole report, the verdict, What to fix (for the model the
+    # filter shows), each fix card and each scenario. The Markdown each copies
+    # is ReportMarkdown's, embedded in a hidden textarea, so the page copies
+    # exactly what the dashboard and the CLI write.
     #
     # Same content as Report#to_markdown, laid out the way the dashboard's
     # suite card is: header and stat tiles, the MODELS panel with the judge's
@@ -50,6 +55,8 @@ module ActiveAgent
           #{html_footer}
           </section>
           </main>
+          #{markdown_store("md-report", to_markdown)}
+          <script>#{COPY_SCRIPT}</script>
           </body>
           </html>
         HTML
@@ -151,6 +158,14 @@ module ActiveAgent
         "#{format("%.#{n >= 10_000 ? 1 : 2}f", n / 1_000).sub(/\.?0+\z/, '')}s"
       end
 
+      # "4.02s / 22.2s": a model's P50 and P95 latency; "—" when no replay
+      # was timed.
+      def fmt_percentiles(stats)
+        return "—" if stats["p50_duration_ms"].nil?
+
+        "#{fmt_ms(stats['p50_duration_ms'])} / #{fmt_ms(stats['p95_duration_ms'])}"
+      end
+
       # Money reads "~$0.0243" when any part of it was estimated from tokens
       # × a model rate rather than reported (Format.money); the footer's
       # legend explains the mark once.
@@ -229,7 +244,7 @@ module ActiveAgent
 
         <<~HEADER
           <header>
-          <h1>#{h(title)}</h1>
+          <div class="title-row"><h1>#{h(title)}</h1>#{copy_button('md-report', label: 'Copy as Markdown', title: 'Copy the whole report as GitHub-flavored Markdown')}</div>
           <div class="chips">#{chips.join}</div>
           </header>
         HEADER
@@ -312,8 +327,10 @@ module ActiveAgent
 
         <<~PANEL
           <div class="panel">
-          <div class="panel-head"><span class="micro">Models</span><span class="right">judged by #{h(judged_by)}</span></div>
+          <div class="panel-head"><span class="micro">Models</span><span class="right">judged by #{h(judged_by)}</span>#{copy_button('md-verdict', title: 'Copy the verdict and the model comparison as Markdown')}</div>
+          #{markdown_store('md-verdict', verdict_markdown)}
           #{html_comparison_table if comparing?}
+          #{html_intervals}
           #{blocks.join}
           #{verdict_row}
           </div>
@@ -335,12 +352,49 @@ module ActiveAgent
 
         <<~TABLE
           <div class="compare"><table>
-          <thead><tr><th>Model</th><th class="num">Passed</th><th class="num">Mean score</th><th class="num">Avg latency</th><th class="num" title="Average input + output tokens per scenario">Avg tokens</th><th class="num" title="Cohort spend, and per scenario">Cost</th>#{judge_head}<th class="fault">Typical fault</th></tr></thead>
+          <thead><tr><th>Model</th><th class="num">Passed</th><th class="num">Mean score</th><th class="num">Avg latency</th><th class="num" title="Median and 95th percentile latency, nearest rank">P50 / P95</th><th class="num" title="Average input + output tokens per scenario">Avg tokens</th><th class="num" title="Cohort spend, and per scenario">Cost</th>#{judge_head}<th class="fault">Typical fault</th></tr></thead>
           <tbody>#{rows.map { |label, stats| html_comparison_row(label, stats) }.join}</tbody>
           </table></div>
         TABLE
       end
 
+      # The ranked interval plot under the comparison table (Report#comparison):
+      # one row per model, best first, its pass rate a dot inside its interval
+      # on a fixed 0-100% axis, so a narrow gap is not magnified, with the
+      # figures written out beside it; then the conclusion and how the
+      # intervals were computed. Nothing for a single model. The rows and the
+      # axis share one grid (.iv-row is display: contents), so every track
+      # spans the same width and a tick sits under the dots it names.
+      def html_intervals
+        return "" unless comparison
+
+        method = comparison["method"]
+        confidence = "#{(method['confidence'].to_f * 100).round}% CI"
+        rows = comparison["ranking"].map { |label| html_interval_row(label, comparison["models"][label] || {}) }
+        ticks = [ 0, 25, 50, 75, 100 ].map { |tick| %(<span class="iv-tick" style="left:#{tick}%">#{tick}%</span>) }
+
+        <<~PLOT
+          <div class="intervals">
+          <div class="micro sm">Pass rate · #{h(confidence)}</div>
+          #{rows.join}
+          <div class="iv-row iv-axis" aria-hidden="true"><span></span><span class="iv-track">#{ticks.join}</span><span></span></div>
+          <p class="iv-conclusion">#{h(comparison['conclusion'])}</p>
+          <p class="iv-caption">#{h(confidence)} from #{h(Format.delimited(method['resamples']))} resamples of whole scenarios</p>
+          </div>
+        PLOT
+      end
+
+      def html_interval_row(label, stats)
+        rate = stats["pass_rate"]
+        low, high = stats["interval"]
+        range = high ? %(<span class="iv-range" style="left:#{low}%;width:#{(high - low).round(1)}%"></span>) : ""
+        point = rate ? %(<span class="iv-point" style="left:#{rate}%"></span>) : ""
+        figures = "<b>#{rate ? "#{rate}%" : '—'}</b> #{high ? "#{low} to #{high}" : 'no interval'} · #{h(plural(stats['scenarios'].to_i, 'scenario'))}"
+
+        %(<div class="iv-row"><span class="iv-name">#{h(short_name(model_by_label(label)))}</span>) +
+          %(<span class="iv-track">#{range}#{point}</span><span class="iv-figures">#{figures}</span></div>\n)
+      end
+
       def html_comparison_row(label, stats)
         short, provider = split_label(model_by_label(label))
         total = stats["scenarios"].to_i
@@ -362,6 +416,7 @@ module ActiveAgent
           <td class="num ratio tone-#{tone_for(ratio)}">#{h(fmt_passes(stats['passed'], total))}</td>
           <td class="num">#{h(fmt_score(stats['avg_score']))}</td>
           <td class="num">#{h(fmt_ms(stats['avg_duration_ms']))}</td>
+          <td class="num">#{h(fmt_percentiles(stats))}</td>
           <td class="num"#{tokens_title}>#{tokens_cell}</td>
           <td class="num"#{cost_title_attr(results_for(label), estimated: estimated)}>#{cost_cell}</td>
           #{judge_cell}<td class="fault">#{typical_fault_text(label, stats)}</td>
@@ -417,7 +472,7 @@ module ActiveAgent
         <<~BLOCK
           <div class="model">
           <div class="line"><span class="name">#{h(short)}</span><span class="provider">#{h(provider)}</span>#{pick}<span class="pass"><span class="bar bar-#{tone}"><span style="width:#{(ratio * 100).round}%"></span></span><span class="ratio tone-#{tone}">#{h(fmt_passes(stats['passed'], total))}</span></span></div>
-          <div class="stats-line"><span>score <b>#{h(fmt_score(stats['avg_score']))}</b></span><span>latency <b>#{h(fmt_ms(stats['avg_duration_ms']))}</b></span><span class="tok"><span class="in">in</span> #{h(fmt_k(stats['input_tokens']))} · <span class="out">out</span> #{h(fmt_k(stats['output_tokens']))}</span><span#{cost_title_attr(results_for(label), estimated: estimated)}>cost <b>#{h(fmt_cost(stats['cost'], estimated: estimated))}</b></span>#{judge_line}</div>
+          <div class="stats-line"><span>score <b>#{h(fmt_score(stats['avg_score']))}</b></span><span>latency <b>#{h(fmt_ms(stats['avg_duration_ms']))}</b></span><span>p50 / p95 <b>#{h(fmt_percentiles(stats))}</b></span><span class="tok"><span class="in">in</span> #{h(fmt_k(stats['input_tokens']))} · <span class="out">out</span> #{h(fmt_k(stats['output_tokens']))}</span><span#{cost_title_attr(results_for(label), estimated: estimated)}>cost <b>#{h(fmt_cost(stats['cost'], estimated: estimated))}</b></span>#{judge_line}</div>
           <div class="faults">#{faults_html}</div>
           </div>
         BLOCK
@@ -565,12 +620,13 @@ module ActiveAgent
         expects = scenario.expected_tools.any? ? %(<span class="exp">expects <b>#{h(scenario.expected_tools.join(' or '))}</b></span>) : ""
         labels = @models.map(&:label)
         cards = cohort.sort_by { |result| labels.index(result.label) || labels.size }.map { |result| html_result_card(result) }
+        md_id = "md-#{anchor(scenario)}"
 
         <<~BLOCK
           <div id="#{h(anchor(scenario))}">
           <details>
           <summary><span class="chev">&gt;</span><span class="key">#{h(scenario.key)}</span><span class="prompt">#{h(scenario.prompt)}</span>#{expects}</summary>
-          <div class="drill">#{cards.join}</div>
+          <div class="drill">#{cards.join}<div class="drill-copy">#{copy_button(md_id, title: "Copy #{scenario.key} as Markdown")}#{markdown_store(md_id, scenario_markdown(scenario.key, answer_limit: ANSWER_LIMIT))}</div></div>
           </details>
           </div>
         BLOCK
@@ -623,14 +679,16 @@ module ActiveAgent
                "#{plural(faulted.map { |result| result.scenario.key }.uniq.size, 'scenario')}"
         body =
           if items.any?
-            %(<div class="fixes">#{items.map { |item| html_fix_card(item) }.join}</div>)
+            %(<div class="fixes">#{items.each_with_index.map { |item, index| html_fix_card(item, index) }.join}</div>)
           else
             %(<div class="nothing">[+] nothing to fix</div>)
           end
 
         <<~FIXES
           <section class="section fix-section" aria-label="Recommendations">
-          <div class="section-head"><span class="micro">What to fix</span><span class="meta">#{h(meta)}</span></div>
+          <div class="section-head"><span class="micro">What to fix</span><span class="meta">#{h(meta)}</span>#{copy_button('md-fixes', model_filtered: comparing? && items.any?, title: 'Copy What to fix as Markdown, for the model the filter shows')}</div>
+          #{markdown_store('md-fixes', fix_items_markdown)}
+          #{@models.each_index.map { |index| markdown_store("md-fixes-m#{index}", fix_items_markdown(model: @models[index].label)) }.join if comparing? && items.any?}
           #{html_fix_filter(items) if comparing? && items.any?}
           #{body}
           </section>
@@ -659,7 +717,7 @@ module ActiveAgent
         labels.filter_map { |label| (index = @models.index(model_by_label(label))) && "m#{index}" }.join(" ")
       end
 
-      def html_fix_card(item)
+      def html_fix_card(item, index)
         tone = item["kind"] == "instruction" ? "info" : "error"
         glyph = tone == "info" ? "[i]" : "[!]"
         title = fault_name(item["fault"]) + (item["count"].to_i > 1 ? " ×#{item['count']}" : "")
@@ -672,6 +730,7 @@ module ActiveAgent
         parts << html_fix_server(item["server"]) if item["server"]
         parts << %(<div class="note">#{h(item['note'])}</div>) if item["note"].present?
         parts << html_fix_action(item["action"]) if item["action"]
+        parts << %(<div class="fix-copy">#{copy_button("md-fix-#{index}", title: 'Copy this fix as Markdown')}#{markdown_store("md-fix-#{index}", fix_item_markdown(item))}</div>)
         models = fix_model_tokens(item)
         %(<div class="fix"#{%( data-models="#{models}") if models.present?}>#{parts.join}</div>)
       end
@@ -724,6 +783,19 @@ module ActiveAgent
         "#{scenarios} · #{models}"
       end
 
+      # --- Copy as Markdown -------------------------------------------------
+
+      # A button COPY_SCRIPT answers by copying the textarea +id+ names; with
+      # +model_filtered+, the one for the model the fix filter has checked.
+      def copy_button(id, label: "Copy as Markdown", title: nil, model_filtered: false)
+        %(<button type="button" class="copy-md" data-md="#{h(id)}"#{' data-md-model' if model_filtered}) +
+          %(#{%( title="#{h(title)}") if title}>#{h(label)}</button>)
+      end
+
+      def markdown_store(id, markdown)
+        %(<textarea class="md-store" id="#{h(id)}" hidden readonly>#{h(markdown)}</textarea>)
+      end
+
       # --- footer ----------------------------------------------------------
 
       # The run's terms: the judge, the criteria and the pass mark, what it
@@ -757,6 +829,48 @@ module ActiveAgent
 
       # Colors only through the token variables; radii 4 badges · 6 chips ·
       # 8 controls · 10 nested panels · 12 cards · 999 bars; no shadows.
+      # Copies the Markdown a button names: the async Clipboard API where the
+      # page may use it, else a selected textarea and the copy command, which
+      # still works on an http:// host and in the dashboard's frame.
+      COPY_SCRIPT = <<~JS.gsub(/\n\s*/, "").freeze
+        document.addEventListener("click", function (event) {
+          var button = event.target.closest && event.target.closest("button.copy-md");
+          if (!button) return;
+          event.preventDefault();
+          var id = button.getAttribute("data-md");
+          if (button.hasAttribute("data-md-model")) {
+            var picked = document.querySelector('input[name="fix-model"]:checked');
+            if (picked && picked.value !== "all") id += "-" + picked.value;
+          }
+          var store = document.getElementById(id);
+          if (!store) return;
+          var text = store.value;
+          var label = button.getAttribute("data-label") || button.textContent;
+          button.setAttribute("data-label", label);
+          var done = function (ok) {
+            button.textContent = ok ? "Copied" : "Copy failed";
+            setTimeout(function () { button.textContent = label; }, 1500);
+          };
+          var legacy = function () {
+            var area = document.createElement("textarea");
+            area.value = text;
+            area.style.position = "fixed";
+            area.style.opacity = "0";
+            document.body.appendChild(area);
+            area.select();
+            var ok = false;
+            try { ok = document.execCommand("copy"); } catch (error) {}
+            document.body.removeChild(area);
+            done(ok);
+          };
+          if (navigator.clipboard && navigator.clipboard.writeText) {
+            navigator.clipboard.writeText(text).then(function () { done(true); }, legacy);
+          } else {
+            legacy();
+          }
+        });
+      JS
+
       STYLES = <<~CSS.freeze
         * { box-sizing: border-box; }
         html, body { margin: 0; min-height: 100%; }
@@ -830,6 +944,21 @@ module ActiveAgent
         .pick-model input { position: absolute; opacity: 0; width: 0; height: 0; }
         .pick-model:has(input:checked) { border-color: var(--color-accent-ui); background: var(--color-accent-ui-tint); color: var(--color-accent-ui); }
         .clean { font-family: var(--font-mono); font-size: 11px; color: var(--color-success-text); }
+        .intervals { padding: 10px 12px; border-top: 1px solid var(--color-border-light); display: grid; grid-template-columns: minmax(120px, 200px) minmax(160px, 1fr) minmax(190px, auto); align-items: center; gap: 6px 12px; }
+        .intervals > .micro, .iv-conclusion, .iv-caption { grid-column: 1 / -1; }
+        .iv-row { display: contents; }
+        .iv-name { font-family: var(--font-mono); font-size: 12px; font-weight: 600; color: var(--color-text-primary); overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
+        .iv-track { position: relative; height: 14px; }
+        .iv-track::before { content: ""; position: absolute; left: 0; right: 0; top: 6px; height: 2px; background: var(--color-muted); }
+        .iv-range { position: absolute; top: 3px; height: 8px; box-sizing: border-box; border: 1px solid var(--color-info); border-radius: 999px; background: var(--color-info-soft); }
+        .iv-point { position: absolute; top: 2px; width: 10px; height: 10px; margin-left: -5px; border-radius: 50%; background: var(--color-info); }
+        .iv-figures { font-family: var(--font-mono); font-size: 11px; color: var(--color-text-secondary); white-space: nowrap; }
+        .iv-figures b { color: var(--color-text-primary); }
+        .iv-axis .iv-track::before { display: none; }
+        .iv-tick { position: absolute; top: 0; transform: translateX(-50%); font-family: var(--font-mono); font-size: 10px; color: var(--color-text-muted); }
+        .iv-conclusion { margin: 4px 0 0; font-size: 12px; line-height: 18px; color: var(--color-text-cell); }
+        .iv-caption { margin: 0; font-family: var(--font-mono); font-size: 10px; color: var(--color-text-muted); }
+        @media (max-width: 640px) { .intervals { grid-template-columns: minmax(0, 1fr); row-gap: 4px; } .iv-row + .iv-row .iv-name { margin-top: 6px; } .iv-axis { display: none; } }
         .verdict { padding: 10px 12px; border-top: 1px solid var(--color-border-light); font-size: 12px; line-height: 18px; color: var(--color-text-cell); }
         .verdict .micro { margin-right: 8px; }
         .fixes { display: grid; grid-template-columns: repeat(auto-fit, minmax(320px, 1fr)); gap: 12px; }
@@ -851,6 +980,14 @@ module ActiveAgent
         .action { display: flex; align-items: center; gap: 10px; margin-top: auto; padding-top: 2px; flex-wrap: wrap; }
         .btn { display: inline-block; padding: 6px 12px; border-radius: 8px; font-size: 13px; font-weight: 500; color: var(--color-text-cell); border: 1px solid var(--color-border-strong); background: transparent; white-space: nowrap; }
         .btn:hover { background: var(--color-hover); color: var(--color-text-primary); text-decoration: none; }
+        .copy-md { font-family: var(--font-mono); font-size: 11px; padding: 3px 8px; border-radius: 6px; border: 1px solid var(--color-border); background: transparent; color: var(--color-text-secondary); cursor: pointer; white-space: nowrap; }
+        .copy-md:hover { background: var(--color-hover); color: var(--color-text-primary); }
+        .title-row { display: flex; align-items: center; gap: 12px; flex-wrap: wrap; }
+        .section-head .copy-md, .panel-head .copy-md { margin-left: auto; }
+        .panel-head .right + .copy-md { margin-left: 0; }
+        .drill-copy, .fix-copy { display: flex; justify-content: flex-end; }
+        .drill-copy { grid-column: 1 / -1; }
+        @media print { .copy-md { display: none; } }
         .hint { font-family: var(--font-mono); font-size: 11px; color: var(--color-text-muted); white-space: nowrap; }
         .matrix { border: 1px solid var(--color-border-light); border-radius: 10px; overflow-x: auto; }
         .mx { display: grid; gap: 12px; padding: 10px 12px; border-top: 1px solid var(--color-border-light); }

data/lib/active_agent/evals/report_markdown.rb

@@ -0,0 +1,202 @@
+# frozen_string_literal: true
+
+module ActiveAgent
+  module Evals
+    # Report#to_markdown: the run as GitHub-flavored Markdown with the same
+    # content as Report#to_html — the per-model table and the cost, the
+    # verdict with the model comparison, the scenario matrix, every answer
+    # (one `<details>` block per scenario, since GitHub strips the HTML
+    # page's styles), and What to fix from Report#fix_items — so it can be
+    # pasted into a pull request or an issue.
+    #
+    # Each piece is public on its own, for the pieces people paste one at a
+    # time: #fix_item_markdown, #fix_items_markdown (optionally for one
+    # model), #verdict_markdown and #scenario_markdown. The whole report is
+    # built from them, so a piece copied alone reads as it does in the report.
+    #
+    # Table cells escape `|` and fold line breaks; text set inside the HTML
+    # of a `<details>` block escapes `<`, `>` and `&`.
+    module ReportMarkdown
+      # How much of an answer the whole report quotes; a scenario copied on
+      # its own quotes the whole answer.
+      MARKDOWN_ANSWER_LIMIT = 1_500
+
+      def to_markdown
+        lines = [ "# #{markdown_title}", "" ]
+        lines << judged_by_line
+        lines << ""
+        lines.concat(summary_table)
+        lines << ""
+        lines.concat(total_lines)
+        lines.concat(verdict_lines)
+        lines.concat(comparison_lines)
+        lines.concat(matrix_table)
+        lines.concat([ "", "## Answers", "" ])
+        scenario_keys.each { |key| lines << scenario_markdown(key, details: true, answer_limit: MARKDOWN_ANSWER_LIMIT) << "" }
+        lines << fix_items_markdown(heading: "##")
+        "#{lines.join("\n").rstrip}\n"
+      end
+
+      # One fix item (Report#fix_items) as a heading and its body: the fault
+      # and how often, the scenarios and models it covers, the
+      # recommendation, the judge's proposed instruction change as a quote,
+      # the tools involved, the server that serves them, the note and where
+      # in the dashboard the fix lives.
+      def fix_item_markdown(item, heading: "###")
+        item = item.to_h.stringify_keys
+        title = item["fault"].to_s.tr("_", " ") + (item["count"].to_i > 1 ? " ×#{item['count']}" : "")
+        lines = [ "#{heading} #{item['kind'] == 'instruction' ? 'ℹ️' : '❗'} #{title}", "" ]
+
+        scope = [ "Scenarios: #{Array(item['scenario_keys']).map { |key| "`#{key}`" }.join(', ')}" ]
+        scope << "models: #{Array(item['models']).map { |label| "`#{label}`" }.join(', ')}" if Array(item["models"]).any?
+        scope << "judge suggestion" if item["kind"] == "instruction"
+        lines << scope.join(" · ") << ""
+        lines << item["recommendation"].to_s.strip << "" if item["recommendation"].present?
+        lines.concat(quoted(item["quote"]).push("")) if item["quote"].present?
+        tools = Array(item["tools"])
+        if tools.any?
+          lines << "**#{item['tools_label'].presence || 'tools'}:** " + tools.map { |tool| tool_text(tool) }.join(", ")
+          lines << ""
+        end
+        if (server = item["server"])
+          state = server["status"] == "enabled" ? "enabled for #{@agent_name}" : "#{server['status'].presence || 'unknown'} · not enabled for #{@agent_name}"
+          lines << "Served by **#{server['name'].presence || server['key']}** (#{state})" << ""
+        end
+        lines << "_#{item['note'].to_s.strip}_" << "" if item["note"].present?
+        if (action = item["action"])
+          lines << "Fix: #{[ action['label'], action['hint'] ].compact_blank.join(' — ')}" << ""
+        end
+        lines.join("\n").rstrip
+      end
+
+      # What to fix: every fix item, or with +model+ (a model label) the ones
+      # attributed to that model, as the report's model filter shows them.
+      def fix_items_markdown(model: nil, heading: "##")
+        items = fix_items
+        items = items.select { |item| Array(item["models"]).empty? || item["models"].include?(model) } if model.present?
+        lines = [ "#{heading} What to fix#{" for `#{model}`" if model.present?}", "" ]
+        if items.empty?
+          lines << "_Nothing to fix._"
+        else
+          sub = "#{heading}#"
+          lines << items.map { |item| fix_item_markdown(item, heading: sub) }.join("\n\n")
+        end
+        lines.join("\n")
+      end
+
+      # The verdict — the best model, why, and who picked it — then the
+      # per-model table and, comparing models, the comparison with its
+      # intervals.
+      def verdict_markdown(heading: "##")
+        lines = [ "#{heading} Verdict", "", judged_by_line, "" ]
+        lines.concat(verdict_lines)
+        lines.concat(summary_table)
+        lines << ""
+        lines.concat(total_lines)
+        lines.concat(comparison_lines.map { |line| line.start_with?("## ") ? line.sub("## ", "#{heading}# ") : line })
+        lines.join("\n").rstrip
+      end
+
+      # One scenario: its prompt and expectations, a table of each model's
+      # status, score, tool calls, fault and cost, then each model's answer.
+      # With +details+ it is a `<details>` block whose summary names the key,
+      # the prompt and each model's mark, the way the report folds it.
+      # Raises KeyError for a key the report has no result for.
+      def scenario_markdown(key, details: false, answer_limit: nil)
+        cohort = @results.select { |result| result.scenario.key == key.to_s }
+        raise KeyError, "no results for scenario #{key}" if cohort.empty?
+
+        scenario = cohort.first.scenario
+        labels = @models.map(&:label)
+        cohort = cohort.sort_by { |result| labels.index(result.label) || labels.size }
+        body = []
+        body << "**Prompt**" << "" << quoted(scenario.prompt).join("\n") << ""
+        expectations = expectation_lines(scenario)
+        body.concat(expectations).push("") if expectations.any?
+        body << "| Model | Status | Score | Tools | Fault | Cost |" << "|---|---|---|---|---|---|"
+        cohort.each do |result|
+          tools = result.replay.tool_calls.map { |call| "#{call['name']}#{' ✗' if call['error']}" }.join(", ")
+          body << "| #{[ "`#{result.label}`", status_text(result), (result.score ? Format.score(result.score) : '—'), tools.presence || '—',
+                         result.fault&.tr('_', ' ') || '—', (result.replay.cost ? answer_cost(result) : '—') ].map { |text| cell(text) }.join(' | ')} |"
+        end
+        body << ""
+        cohort.each do |result|
+          body << "**`#{result.label}`** — #{status_text(result)}"
+          body << ""
+          body << "Fault: #{result.fault.tr('_', ' ')} — #{[ result.summary, result.recommendation ].compact.join(' ')}" << "" if result.fault
+          body << "Error: #{result.replay.error}" << "" if result.replay.error
+          answer = result.replay.answer.to_s
+          answer = answer.truncate(answer_limit) if answer_limit
+          body << (answer.present? ? quoted(answer).join("\n") : "_(no answer)_") << ""
+        end
+        text = body.join("\n").rstrip
+        return "### Scenario `#{scenario.key}`\n\n#{text}" unless details
+
+        marks = cohort.map { |result| "#{status_mark(result)} #{result.label}" }.join(" · ")
+        summary = "<code>#{html_text(scenario.key)}</code> #{html_text(scenario.prompt.truncate(80))} — #{html_text(marks)}"
+        "<details>\n<summary>#{summary}</summary>\n\n#{text}\n\n</details>"
+      end
+
+      private
+
+      def markdown_title
+        scenarios = scenario_keys.size
+        "Evaluation — #{scenarios} scenario#{'s' unless scenarios == 1} × #{@models.size} model#{'s' unless @models.size == 1}"
+      end
+
+      def scenario_keys
+        @results.map { |result| result.scenario.key }.uniq
+      end
+
+      def judged_by_line
+        label = @judge_label || @judge&.label
+        label ? "Judged by `#{label}`." : "No judge; scored on rules and expectations alone."
+      end
+
+      # "**Best model: gpt-5-mini**" with the verdict's rationale under it
+      # and, when a judge (not the pass-rate ranking) wrote it, who did.
+      def verdict_lines
+        return [] unless winner
+
+        lines = [ "**Best model: #{winner}**", "" ]
+        rationale = verdict["rationale"].to_s.strip
+        judge = verdict["judge"].presence
+        rationale += " _(#{judge == Report::PASS_RATE_JUDGE ? 'ranked by pass rate' : "judged by #{judge}"})_" if judge && rationale.present?
+        lines << rationale << "" if rationale.present?
+        lines
+      end
+
+      def expectation_lines(scenario)
+        lines = []
+        lines << "Expects tools: #{scenario.expected_tools.map { |name| "`#{name}`" }.join(' or ')}" if scenario.expected_tools.any?
+        lines << "Expects the answer to contain: #{scenario.expected_patterns.map { |text| "`#{text}`" }.join(', ')}" if scenario.expected_patterns.any?
+        lines << "Expects the answer not to contain: #{scenario.forbidden_patterns.map { |text| "`#{text}`" }.join(', ')}" if scenario.forbidden_patterns.any?
+        lines.map { |line| "- #{line}" }
+      end
+
+      def status_mark(result)
+        result.passed? ? "✅" : (result.errored? ? "⚠️" : "❌")
+      end
+
+      def status_text(result)
+        "#{status_mark(result)} #{result.status}"
+      end
+
+      def tool_text(tool)
+        tool = tool.is_a?(Hash) ? tool.stringify_keys : { "name" => tool.to_s }
+        note = tool["note"].presence
+        "`#{tool['name']}`#{" (#{note})" if note}"
+      end
+
+      # Each line of +text+ as a blockquote line, `<` and `>` escaped so an
+      # answer cannot open or close the HTML around it.
+      def quoted(text)
+        text.to_s.strip.split(/\r?\n/).map { |line| line.strip.empty? ? ">" : "> #{line.gsub('<', '&lt;').gsub('>', '&gt;')}" }
+      end
+
+      def html_text(text)
+        text.to_s.gsub("&", "&amp;").gsub("<", "&lt;").gsub(">", "&gt;").gsub(/\s+/, " ")
+      end
+    end
+  end
+end

data/lib/active_agent/evals/runner.rb

@@ -54,10 +54,14 @@ module ActiveAgent
       # @param release [Hash, nil] the release of the agent under evaluation,
       #   `{ "digest", "revision", "label" }`, carried onto the Report so a
       #   published run names the code it scored (see Report.new)
+      # @param refine_gate [#call, nil] asked before each recommendation call,
+      #   after `judge_limit`; false skips it. A caller that scores each
+      #   scenario in its own Runner (a background job per scenario) shares one
+      #   run-wide limit this way, since `judge_limit` counts per instance.
       def initialize(scenarios:, models:, replay:, criteria: [], judge: nil, judge_task: true, available_tools: {},
                      instructions: nil, agent_name: "The agent", threshold: PASS_THRESHOLD,
                      refine_faults: DEFAULT_REFINE_FAULTS, judge_limit: DEFAULT_JUDGE_LIMIT, on_result: nil,
-                     around_evaluation: nil, require_judge_scores: false, metadata: {}, release: nil)
+                     around_evaluation: nil, require_judge_scores: false, metadata: {}, release: nil, refine_gate: nil)
         @scenarios = scenarios
         @models = models
         @replay = replay
@@ -75,6 +79,7 @@ module ActiveAgent
         @require_judge_scores = require_judge_scores
         @metadata = metadata
         @release = release
+        @refine_gate = refine_gate
         @judge_calls = 0
         @scorer = Scorer.new(criteria: criteria, judge: judge)
       end
@@ -187,6 +192,7 @@ module ActiveAgent
       def refine!(diagnosis, scenario, replay, result)
         return unless @judge && @refine_faults.include?(result.fault)
         return if @judge_calls >= @judge_limit
+        return if @refine_gate && !@refine_gate.call
 
         @judge_calls += 1
         refined = @judge.recommend(scenario: scenario, replay: replay, diagnosis: result,

data/lib/active_agent/evals/statistics.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module ActiveAgent
+  module Evals
+    # The summary statistics a report computes over its results.
+    module Statistics
+      module_function
+
+      # Nearest-rank percentile: the value at rank ceil(pct / 100 × n) of the
+      # values sorted ascending, so the result is always one of the values. The
+      # dashboard's Metrics view uses the same rule (ActionAgent::MetricsReport),
+      # so a model's P95 reads the same in both. nil for no values.
+      #
+      # @param values [Array<Numeric>] in any order; nils are ignored
+      # @param pct [Numeric] 0..100
+      # @return [Numeric, nil]
+      def percentile(values, pct)
+        sorted = values.compact.sort
+        return nil if sorted.empty?
+
+        rank = (sorted.size * pct / 100.0).ceil
+        sorted[rank.clamp(1, sorted.size) - 1]
+      end
+    end
+  end
+end

data/lib/active_agent/evals.rb

@@ -19,12 +19,15 @@ require_relative "evals/catalog"
 require_relative "evals/model_spec"
 require_relative "evals/replay"
 require_relative "evals/format"
+require_relative "evals/statistics"
+require_relative "evals/comparison"
 require_relative "evals/scorer"
 require_relative "evals/diagnosis"
 require_relative "evals/judge"
 require_relative "evals/result"
 require_relative "evals/design_tokens"
 require_relative "evals/report_html"
+require_relative "evals/report_markdown"
 require_relative "evals/report"
 require_relative "evals/runner"
 require_relative "evals/correlation"

data/lib/active_agent/providers/_base_provider.rb

@@ -132,6 +132,7 @@ module ActiveAgent
                     :stream_broadcaster, :streaming,   # Callback (Streams)
                     :stream_completion_pending,        # Callback (Streams)
                     :stream_completion_result,         # Callback (Streams)
+                    :stream_finish_reason,             # Callback (Streams)
                     :tools_function,                   # Callback (Tools)
                     :usage_stack,                      # Usage Tracking
                     :stream_usage_index,               # Usage Tracking (Streams)
@@ -607,6 +608,7 @@ module ActiveAgent
           messages:,
           raw_request:  prompt_request_type.serialize(request),
           raw_response: api_response,
+          streamed_finish_reason: stream_finish_reason,
           usages: usage_stack,
           **attributes
         )

data/lib/active_agent/providers/anthropic_provider.rb

@@ -43,7 +43,7 @@ module ActiveAgent
       def initialize(kwargs = {})
         super
 
-        self.json_format_retry_count = kwargs[:max_retries] || ::Anthropic::Client::DEFAULT_MAX_RETRIES
+        self.json_format_retry_count = options.max_retries
       end
 
       # @todo Add support for Anthropic::BedrockClient and Anthropic::VertexClient
@@ -222,6 +222,8 @@ module ActiveAgent
             # No-Op; Wait for Function call to be complete
           when :thinking_delta, :signature_delta
             # TODO: Add with thinking rendering support
+          when :citations_delta
+            # No-Op; the citations arrive with the full block at content_block_stop
           else
             raise "Unexpected delta type: #{api_delta.type}"
           end
@@ -235,6 +237,7 @@ module ActiveAgent
         when :message_delta
           delta = Anthropic::Transforms.gem_to_hash(api_response_chunk.delta)
           self.message_stack.last.merge!(delta)
+          self.stream_finish_reason = delta[:stop_reason] if delta[:stop_reason]
 
         # Message Completed [Full Message]
         when :message_stop

data/lib/active_agent/providers/common/responses/base.rb

@@ -63,6 +63,16 @@ module ActiveAgent
           #   @return [Array<Usage>]
           attribute :usages, default: -> { [] }, writable: false
 
+          # @!attribute [r] streamed_finish_reason
+          #   Why a streamed generation stopped, as the provider reported it.
+          #
+          #   A streamed generation has no response body to read it from, so
+          #   the provider records it from the stream. {#finish_reason} falls
+          #   back to it.
+          #
+          #   @return [String, nil]
+          attribute :streamed_finish_reason, writable: false
+
           # Initializes response with deep-duplicated attributes.
           #
           # Deep duplication prevents external modifications from affecting internal state.
@@ -180,14 +190,17 @@ module ActiveAgent
           #   response.stop_reason    #=> "stop" (alias)
           def finish_reason
             @finish_reason ||= begin
-              return nil unless raw_response
+              return streamed_finish_reason unless raw_response
 
               if raw_response.is_a?(Hash)
                 # OpenAI format: choices[0].finish_reason or choices[0].message.finish_reason
                 raw_response.dig(:choices, 0, :finish_reason) ||
                   raw_response.dig(:choices, 0, :message, :finish_reason) ||
                   # Anthropic format: stop_reason
-                  raw_response[:stop_reason]
+                  raw_response[:stop_reason] ||
+                  # OpenAI Responses format: why it was cut short, else its status
+                  raw_response.dig(:incomplete_details, :reason) ||
+                  raw_response[:status]
               end
             end
           end