activeagent 1.7.2 → 1.8.1

data/lib/active_agent/evals/report_html.rb

@@ -12,8 +12,8 @@ module ActiveAgent
     #
     # Same content as Report#to_markdown, laid out the way the dashboard's
     # suite card is: header and stat tiles, the MODELS panel with the judge's
-    # pick and verdict, WHAT TO FIX cards from Report#fix_items, the
-    # SCENARIOS matrix, and a per-scenario disclosure with every answer.
+    # pick and verdict, the SCENARIOS matrix, a per-scenario disclosure with
+    # every answer, and WHAT TO FIX cards from Report#fix_items.
     module ReportHtml
       THEMES = %w[light dark].freeze
       ANSWER_LIMIT = 3_000
@@ -44,9 +44,9 @@ module ActiveAgent
           #{html_stat_tiles}
           <section class="card">
           #{html_models_panel}
-          #{html_fixes}
           #{html_matrix}
           #{html_details}
+          #{html_fixes}
           #{html_footer}
           </section>
           </main>
@@ -151,16 +151,50 @@ module ActiveAgent
         "#{format("%.#{n >= 10_000 ? 1 : 2}f", n / 1_000).sub(/\.?0+\z/, '')}s"
       end
 
-      def fmt_cost(value)
-        value.nil? ? "—" : format("$%.4f", value)
+      # Money reads "~$0.0243" when any part of it was estimated from tokens
+      # × a model rate rather than reported (Format.money); the footer's
+      # legend explains the mark once.
+      def fmt_cost(value, estimated: false)
+        Format.money(value, estimated: estimated)
       end
 
+      # A 0..1 score as a whole percent, "—" when there is none.
       def fmt_score(value)
-        value.nil? ? "—" : format("%.2f", value)
+        Format.score(value)
+      end
+
+      # "14/16 · 88%" — a fraction always carries its percentage.
+      def fmt_passes(passed, total)
+        Format.passes(passed, total)
       end
 
-      def fmt_mean_score(value)
-        value.nil? ? "—" : format("%.3f", value)
+      # Whether the page shows a "~" anywhere, so the footer carries the legend.
+      def estimated_anywhere?
+        run_costs["estimated"] || summary_by_model.values.any? { |stats| estimated_cost?(stats) }
+      end
+
+      # The tooltip of an estimated figure, worked out from the results it
+      # sums: their tokens at the rate the first estimated one recorded.
+      # Nothing for a reported figure.
+      def cost_title_attr(results, estimated:)
+        return "" unless estimated
+
+        priced = results.select(&:estimated_cost?)
+        rate = priced.filter_map(&:cost_rate).first
+        title = Format.cost_title(
+          input_tokens: priced.sum { |result| result.replay.input_tokens.to_i },
+          output_tokens: priced.sum { |result| result.replay.output_tokens.to_i },
+          rate: rate
+        )
+        %( title="#{h(title)}")
+      end
+
+      def results_for(label)
+        @results.select { |result| result.label == label }
+      end
+
+      def judge_estimated?
+        judge_usage&.dig("estimated") == true
       end
 
       # --- page ------------------------------------------------------------
@@ -177,14 +211,21 @@ module ActiveAgent
           "}",
           DesignTokens.css(scope: ":root.theme-dark", tokens: DesignTokens::DARK, color_scheme: "dark"),
           STYLES,
-          ".mx { grid-template-columns: minmax(240px, 1.6fr) 150px repeat(#{@models.size}, minmax(170px, 1fr)); }",
-          ".matrix .inner { min-width: #{390 + 185 * @models.size}px; }"
+          ".mx { grid-template-columns: minmax(240px, 1.6fr) 150px repeat(#{@models.size}, minmax(170px, 1fr)) 120px; }",
+          # One rule per model: with that chip checked, hide every fix card
+          # attributed to other models (cards attributed to none stay).
+          *@models.each_index.map { |i| ".fix-section:has(input[value=\"m#{i}\"]:checked) .fix[data-models]:not([data-models~=\"m#{i}\"]) { display: none; }" },
+          ".matrix .inner { min-width: #{520 + 185 * @models.size}px; }"
         ].join("\n")
       end
 
+      # One chip per scalar metadata value — an array or a hash (the judge's
+      # trace ids, say) is a record, not a label — then the release when the
+      # report names one, and the judge with its calls and spend.
       def html_header(title)
-        chips = @metadata.to_h.map { |key, value| html_chip(key, value) }
-        chips << html_chip("judge", judge_name)
+        chips = metadata_chips.map { |key, value| html_chip(key, value) }
+        chips << html_chip("release", release_label) if release_label
+        chips << html_chip("judge", judge_chip_text)
 
         <<~HEADER
           <header>
@@ -198,24 +239,63 @@ module ActiveAgent
         %(<span class="chip"><b>#{h(key)}</b>#{h(value)}</span>)
       end
 
+      # The metadata worth a chip: scalar values, minus the judge's trace
+      # ids, which the dashboard follows but nobody reads.
+      def metadata_chips
+        @metadata.to_h.reject { |key, value| key.to_s == "judge_trace_ids" || value.is_a?(Hash) || value.is_a?(Array) || value.nil? }
+      end
+
+      # "1a2b3c4d5e6f · abc1234", or the label the caller gave the release.
+      def release_label
+        return nil unless release
+
+        release["label"].presence || [ release["digest"], release["revision"] ].compact_blank.join(" · ").presence
+      end
+
+      # "gpt-5 · 12 calls · ~$0.0315" when the judge spent anything.
+      def judge_chip_text
+        usage = judge_usage
+        return judge_name unless usage
+
+        parts = [ judge_name, plural(usage["calls"].to_i, "call") ]
+        parts << fmt_cost(usage["cost"], estimated: judge_estimated?) if usage["cost"]
+        parts.join(" · ")
+      end
+
       def html_stat_tiles
         total = @results.size
         passed = @results.count(&:passed?)
         ratio = total.positive? ? passed.to_f / total : 0.0
         tiles = [
           html_tile("Scenario runs", total, "#{plural(scenario_cohorts.size, 'scenario')} × #{plural(@models.size, 'model')}"),
-          html_tile("Pass rate", "#{(ratio * 100).round}%", "#{passed} / #{total} passed", tone: tone_for(ratio)),
+          html_tile("Pass rate", Format.percent(total.positive? ? ratio : nil), "#{passed}/#{total} passed", tone: tone_for(ratio)),
           html_tile("Open faults", total - passed, plural(fix_items.size, "fix item")),
+          html_cost_tile,
           html_tile("Models", @models.size, models_subline)
         ]
         %(<section class="stats">#{tiles.join}</section>)
       end
 
-      def html_tile(label, value, sub, tone: nil)
-        %(<div class="tile"><div class="micro">#{h(label)}</div>) +
+      def html_tile(label, value, sub, tone: nil, title: nil)
+        %(<div class="tile"#{%( title="#{h(title)}") if title}><div class="micro">#{h(label)}</div>) +
           %(<div class="value#{" tone-#{tone}" if tone}">#{h(value)}</div><div class="sub">#{h(sub)}</div></div>)
       end
 
+      # The run's spend: agent plus judge, with the two apart underneath.
+      # A run with nothing priced says so rather than showing a blank.
+      def html_cost_tile
+        costs = run_costs
+        sub =
+          if costs["total"].nil?
+            "nothing priced"
+          elsif judge_usage
+            "agent #{fmt_cost(costs['cost'], estimated: costs['estimated'])} · judge #{fmt_cost(costs['judge_cost'], estimated: judge_estimated?)}"
+          else
+            "agent only · #{plural(costs['priced'], 'scenario run')} priced"
+          end
+        html_tile("Cost", fmt_cost(costs["total"], estimated: costs["estimated"]), sub)
+      end
+
       def models_subline
         if comparing? && winner
           "judge's pick · #{short_name(model_by_label(winner))}"
@@ -233,12 +313,92 @@ module ActiveAgent
         <<~PANEL
           <div class="panel">
           <div class="panel-head"><span class="micro">Models</span><span class="right">judged by #{h(judged_by)}</span></div>
+          #{html_comparison_table if comparing?}
           #{blocks.join}
           #{verdict_row}
           </div>
         PANEL
       end
 
+      # The comparison read across: one row per model, best first (pass rate,
+      # then mean score) — passed, mean score, average latency, average
+      # tokens per scenario, cost (and per scenario), the judge's spend on
+      # the cohort when a judge was asked, and the model's typical fault.
+      # The blocks under it carry the same figures per model with bars and
+      # every fault.
+      def html_comparison_table
+        rows = summary_by_model.sort_by do |label, stats|
+          total = stats["scenarios"].to_i
+          [ total.positive? ? -stats["passed"].to_f / total : 0.0, -(stats["avg_score"] || -1).to_f, @models.index(model_by_label(label)).to_i ]
+        end
+        judge_head = judge_usage ? %(<th class="num" title="What the judge spent scoring this model's answers">Judge</th>) : ""
+
+        <<~TABLE
+          <div class="compare"><table>
+          <thead><tr><th>Model</th><th class="num">Passed</th><th class="num">Mean score</th><th class="num">Avg latency</th><th class="num" title="Average input + output tokens per scenario">Avg tokens</th><th class="num" title="Cohort spend, and per scenario">Cost</th>#{judge_head}<th class="fault">Typical fault</th></tr></thead>
+          <tbody>#{rows.map { |label, stats| html_comparison_row(label, stats) }.join}</tbody>
+          </table></div>
+        TABLE
+      end
+
+      def html_comparison_row(label, stats)
+        short, provider = split_label(model_by_label(label))
+        total = stats["scenarios"].to_i
+        ratio = total.positive? ? stats["passed"].to_f / total : 0.0
+        pick = comparing? && winner == label ? %(<span class="pick" title="picked by the judge">★ pick</span>) : ""
+        per = ->(value) { value.nil? || total.zero? ? nil : value.to_f / total }
+        avg_tokens = per.call(stats["input_tokens"].to_i + stats["output_tokens"].to_i)
+        tokens_cell = avg_tokens ? h(fmt_k(avg_tokens.round)) : "—"
+        tokens_title = avg_tokens ? %( title="#{per.call(stats['input_tokens']).to_f.round} in · #{per.call(stats['output_tokens']).to_f.round} out per scenario") : ""
+        per_cost = cost_per_priced(stats)
+        estimated = estimated_cost?(stats)
+        cost_cell = h(fmt_cost(stats["cost"], estimated: estimated))
+        cost_cell += "<span class=\"per\">#{h(fmt_cost(per_cost, estimated: estimated))}/scenario</span>" if per_cost
+        judge_cell = judge_usage ? %(<td class="num">#{html_judge_cost(stats)}</td>) : ""
+
+        <<~ROW
+          <tr>
+          <td class="model-cell"><span class="name">#{h(short)}</span>#{pick}<span class="provider">#{h(provider)}</span></td>
+          <td class="num ratio tone-#{tone_for(ratio)}">#{h(fmt_passes(stats['passed'], total))}</td>
+          <td class="num">#{h(fmt_score(stats['avg_score']))}</td>
+          <td class="num">#{h(fmt_ms(stats['avg_duration_ms']))}</td>
+          <td class="num"#{tokens_title}>#{tokens_cell}</td>
+          <td class="num"#{cost_title_attr(results_for(label), estimated: estimated)}>#{cost_cell}</td>
+          #{judge_cell}<td class="fault">#{typical_fault_text(label, stats)}</td>
+          </tr>
+        ROW
+      end
+
+      # "~$0.0030<span class="per">3 calls</span>" — the judge's spend on a
+      # model's answers; "—" when it was not asked about them.
+      def html_judge_cost(stats)
+        calls = stats["judge_calls"].to_i
+        return "—" if stats["judge_cost"].nil? && calls.zero?
+
+        h(fmt_cost(stats["judge_cost"], estimated: judge_estimated?)) + %(<span class="per">#{h(plural(calls, 'call'))}</span>)
+      end
+
+      # "missing content ×2 · refund_window: The answer is missing expected
+      # content: 30." — the model's most frequent fault, and the diagnosis of
+      # the first result that carries it; "no faults" for a clean cohort.
+      def typical_fault_text(label, stats)
+        tally = stats["faults"] || {}
+        return %(<span class="clean">no faults</span>) if tally.empty?
+
+        mine = @results.select { |result| result.label == label }
+        example_of = ->(fault) { mine.find { |result| result.fault == fault } }
+        # Most frequent first; between equals, a fault a result can explain,
+        # then a specific fault over the judge's catch-all, then the name.
+        fault, count = tally.min_by { |name, n| [ -n, example_of.call(name) ? 0 : 1, name == "low_quality" ? 1 : 0, name ] }
+        example = example_of.call(fault)
+        head = "#{fault_name(fault)} ×#{count}"
+        return h(head) unless example&.summary.present?
+
+        detail = "#{example.scenario.key}: #{example.summary}"
+        detail = "#{detail[0, 119]}…" if detail.length > 120
+        "#{h(head)} <span class=\"detail\">· #{h(detail)}</span>"
+      end
+
       def html_model_block(label, stats)
         short, provider = split_label(model_by_label(label))
         total = stats["scenarios"]
@@ -248,106 +408,21 @@ module ActiveAgent
         faults = stats["faults"].map { |fault, count| %(<span class="badge error">#{h(fault_name(fault))} ×#{count}</span>) }
         faults_html = faults.any? ? faults.join : %(<span class="clean">[+] no faults</span>)
 
+        estimated = estimated_cost?(stats)
+        judge_line = ""
+        if stats["judge_cost"] || stats["judge_calls"].to_i.positive?
+          judge_line = %(<span>judge <b>#{h(fmt_cost(stats['judge_cost'], estimated: judge_estimated?))}</b> · #{h(plural(stats['judge_calls'].to_i, 'call'))}</span>)
+        end
+
         <<~BLOCK
           <div class="model">
-          <div class="line"><span class="name">#{h(short)}</span><span class="provider">#{h(provider)}</span>#{pick}<span class="pass"><span class="bar bar-#{tone}"><span style="width:#{(ratio * 100).round}%"></span></span><span class="ratio tone-#{tone}">#{stats['passed']}/#{total}</span></span></div>
-          <div class="stats-line"><span>score <b>#{h(fmt_mean_score(stats['avg_score']))}</b></span><span>latency <b>#{h(fmt_ms(stats['avg_duration_ms']))}</b></span><span class="tok"><span class="in">in</span> #{h(fmt_k(stats['input_tokens']))} · <span class="out">out</span> #{h(fmt_k(stats['output_tokens']))}</span><span>cost <b>#{h(fmt_cost(stats['cost']))}</b></span></div>
+          <div class="line"><span class="name">#{h(short)}</span><span class="provider">#{h(provider)}</span>#{pick}<span class="pass"><span class="bar bar-#{tone}"><span style="width:#{(ratio * 100).round}%"></span></span><span class="ratio tone-#{tone}">#{h(fmt_passes(stats['passed'], total))}</span></span></div>
+          <div class="stats-line"><span>score <b>#{h(fmt_score(stats['avg_score']))}</b></span><span>latency <b>#{h(fmt_ms(stats['avg_duration_ms']))}</b></span><span class="tok"><span class="in">in</span> #{h(fmt_k(stats['input_tokens']))} · <span class="out">out</span> #{h(fmt_k(stats['output_tokens']))}</span><span#{cost_title_attr(results_for(label), estimated: estimated)}>cost <b>#{h(fmt_cost(stats['cost'], estimated: estimated))}</b></span>#{judge_line}</div>
           <div class="faults">#{faults_html}</div>
           </div>
         BLOCK
       end
 
-      # --- WHAT TO FIX -----------------------------------------------------
-
-      # The section stands even for a run with nothing to fix — the dashboard
-      # keeps it too, so a clean run reads as clean rather than as a page
-      # missing a section. A report over no results at all has nothing to say.
-      def html_fixes
-        return "" if @results.empty?
-
-        items = fix_items
-        faulted = @results.reject(&:passed?)
-        meta = "#{plural(items.size, 'item')} · #{plural(faulted.size, 'fault')} across " \
-               "#{plural(faulted.map { |result| result.scenario.key }.uniq.size, 'scenario')}"
-        body =
-          if items.any?
-            %(<div class="fixes">#{items.map { |item| html_fix_card(item) }.join}</div>)
-          else
-            %(<div class="nothing">[+] nothing to fix</div>)
-          end
-
-        <<~FIXES
-          <section class="section" aria-label="Recommendations">
-          <div class="section-head"><span class="micro">What to fix</span><span class="meta">#{h(meta)}</span></div>
-          #{body}
-          </section>
-        FIXES
-      end
-
-      def html_fix_card(item)
-        tone = item["kind"] == "instruction" ? "info" : "error"
-        glyph = tone == "info" ? "[i]" : "[!]"
-        title = fault_name(item["fault"]) + (item["count"].to_i > 1 ? " ×#{item['count']}" : "")
-
-        parts = [ %(<div class="head"><span class="glyph tone-#{tone}">#{glyph}</span>) +
-                  %(<span class="badge #{tone}">#{h(title)}</span><span class="scope">#{h(fix_scope(item))}</span></div>) ]
-        parts << %(<p>#{h(item['recommendation'])}</p>) if item["recommendation"].present?
-        parts << %(<div class="quote">“#{h(item['quote'])}”</div>) if item["quote"].present?
-        parts << html_fix_tools(item) if item["tools"].any?
-        parts << html_fix_server(item["server"]) if item["server"]
-        parts << %(<div class="note">#{h(item['note'])}</div>) if item["note"].present?
-        parts << html_fix_action(item["action"]) if item["action"]
-        %(<div class="fix">#{parts.join}</div>)
-      end
-
-      def html_fix_tools(item)
-        chips = item["tools"].map do |tool|
-          note = tool["note"].presence
-          %(<span class="tool"><b>#{h(tool['name'])}</b>#{%(<span class="note">#{h(note)}</span>) if note}</span>)
-        end
-        %(<div class="tools"><span class="micro sm">#{h(item['tools_label'])}</span><div class="list">#{chips.join}</div></div>)
-      end
-
-      # "available · not enabled for Assistant", "unknown · not enabled for
-      # Assistant" — every status but "enabled" leads with the status word, the
-      # way the dashboard's fix list reads it.
-      def html_fix_server(server)
-        badge =
-          if server["status"] == "enabled"
-            %(<span class="badge success xs">enabled for #{h(@agent_name)}</span>)
-          else
-            %(<span class="badge warning xs">#{h(server['status'].presence || 'unknown')} · not enabled for #{h(@agent_name)}</span>)
-          end
-        %(<div class="served"><span>served by</span><b>#{h(server['name'].presence || server['key'])}</b>#{badge}</div>)
-      end
-
-      # With a route the action is a button; without one, the page can only
-      # say where in the dashboard the fix lives. The link targets the top
-      # window: served in the dashboard's report iframe it would otherwise
-      # open the whole dashboard inside the frame.
-      def html_fix_action(action)
-        button = action["path"].present? ? %(<a class="btn" target="_top" href="#{h(action['path'])}">#{h(action['label'])}</a>) : ""
-        %(<div class="action">#{button}<span class="hint">#{h(action['hint'])}</span></div>)
-      end
-
-      # "3 scenarios · both models" — the models are worth naming only on a
-      # comparison run; on a single-model run the count says it all.
-      def fix_scope(item)
-        return "#{item['scenario_keys'].join(', ')} · judge suggestion" if item["kind"] == "instruction"
-
-        scenarios = plural(item["scenario_keys"].size, "scenario")
-        labels = Array(item["models"])
-        return scenarios unless comparing? && labels.any?
-
-        models =
-          if labels.size >= @models.size
-            @models.size == 2 ? "both models" : "all models"
-          else
-            labels.map { |label| short_name(model_by_label(label)) }.join(", ")
-          end
-        "#{scenarios} · #{models}"
-      end
-
       # --- SCENARIOS matrix ------------------------------------------------
 
       def html_matrix
@@ -355,7 +430,7 @@ module ActiveAgent
           short, provider = split_label(spec)
           %(<span class="col"><span class="name">#{h(short)}</span><span class="provider">#{h(provider)}</span></span>)
         end
-        rows = [ %(<div class="mx head"><span class="micro sm">Scenario</span><span class="micro sm">Expects</span>#{columns.join}</div>) ]
+        rows = [ %(<div class="mx head"><span class="micro sm">Scenario</span><span class="micro sm">Expects</span>#{columns.join}<span class="micro sm cost-head" title="What the scenario cost across every model, and what judging it cost">Cost</span></div>) ]
         scenario_groups.each do |cohorts|
           rows << html_group_row(cohorts) if group_name(cohorts.first.first.scenario)
           cohorts.each { |cohort| rows << html_scenario_row(cohort) }
@@ -383,10 +458,29 @@ module ActiveAgent
             elsif passed.zero? then " text-error"
             else ""
             end
-          %(<span class="group-pass#{tone}">#{passed}/#{results.size} passed</span>)
+          %(<span class="group-pass#{tone}">#{results.empty? ? '—' : "#{h(fmt_passes(passed, results.size))} passed"}</span>)
         end
         %(<div class="mx group"><span class="group-name">#{h(name)}</span>) +
-          %(<span class="count">#{h(plural(cohorts.size, 'scenario'))}</span>#{passes.join}</div>)
+          %(<span class="count">#{h(plural(cohorts.size, 'scenario'))}</span>#{passes.join}#{html_group_cost(cohorts)}</div>)
+      end
+
+      # The group's subtotal — its scenarios' costs summed, judge apart — in
+      # the trailing column, so the grid stays aligned under a group row.
+      def html_group_cost(cohorts)
+        costs = cohorts.map { |cohort| scenario_costs[cohort.first.scenario.key] }.compact
+        agent = costs.filter_map { |entry| entry["cost"] }
+        judge = costs.filter_map { |entry| entry["judge_cost"] }
+        estimated = costs.any? { |entry| entry["estimated"] }
+        return %(<span class="cost"><span class="muted">—</span></span>) if agent.empty? && judge.empty?
+
+        html_cost_block(agent.any? ? agent.sum : nil, judge.any? ? judge.sum : nil, estimated: estimated)
+      end
+
+      # "<b>~$0.0243</b><span class="judge">judge ~$0.0015</span>" — a
+      # scenario's (or group's) spend across every model.
+      def html_cost_block(cost, judge_cost, estimated:)
+        judge = judge_cost ? %(<span class="judge">judge #{h(fmt_cost(judge_cost, estimated: estimated))}</span>) : ""
+        %(<span class="cost"><b>#{h(fmt_cost(cost, estimated: estimated))}</b>#{judge}</span>)
       end
 
       def html_scenario_row(cohort)
@@ -396,8 +490,14 @@ module ActiveAgent
           result = cohort.find { |candidate| candidate.label == spec.label }
           result ? html_result_cell(result) : %(<div class="cell"><div class="top"><span class="muted">—</span></div></div>)
         end
+        costs = scenario_costs[scenario.key] || {}
+        total_cell = if costs["cost"].nil? && costs["judge_cost"].nil?
+          %(<span class="cost"><span class="muted">—</span></span>)
+        else
+          html_cost_block(costs["cost"], costs["judge_cost"], estimated: costs["estimated"])
+        end
         %(<div class="mx"><div><div class="key"><a href="##{h(anchor(scenario))}">#{h(scenario.key)}</a></div>) +
-          %(<div class="prompt">#{h(scenario.prompt)}</div></div><div class="expects">#{expects}</div>#{cells.join}</div>)
+          %(<div class="prompt">#{h(scenario.prompt)}</div></div><div class="expects">#{expects}</div>#{cells.join}#{total_cell}</div>)
       end
 
       def html_result_cell(result)
@@ -406,7 +506,25 @@ module ActiveAgent
         fault = result.fault ? %(<span class="f">#{h(fault_name(result.fault))}</span>) : ""
         %(<div class="cell"><div class="top"><span class="g tone-#{tone}">#{glyph}</span>) +
           %(<span class="s tone-#{tone}">#{h(fmt_score(result.score))}</span>#{fault}</div>) +
-          %(<div class="calls">#{html_calls(result, empty: 'no tools called')}</div></div>)
+          %(<div class="calls">#{html_calls(result, empty: 'no tools called')}</div>#{html_cell_cost(result)}</div>)
+      end
+
+      # "~$0.0243 · judge ~$0.0015" under a cell: what this answer cost and
+      # what judging it cost. A result with no cost at all shows no line.
+      def html_cell_cost(result)
+        text = result_cost_text(result)
+        return "" if text.nil?
+
+        %(<div class="cost-line"#{cost_title_attr([ result ], estimated: result.estimated_cost?)}>#{h(text)}</div>)
+      end
+
+      def result_cost_text(result)
+        judge_cost = result.judge_usage&.dig("cost")
+        return nil if result.replay.cost.nil? && judge_cost.nil?
+
+        parts = [ fmt_cost(result.replay.cost, estimated: result.estimated_cost?) ]
+        parts << "judge #{fmt_cost(judge_cost, estimated: estimated_usage?(result.judge_usage))}" if judge_cost
+        parts.join(" · ")
       end
 
       def html_calls(result, empty:)
@@ -487,20 +605,156 @@ module ActiveAgent
         [
           replay.duration_ms && fmt_ms(replay.duration_ms),
           replay.total_tokens.positive? ? "#{fmt_k(replay.total_tokens)} tokens" : nil,
-          replay.cost && fmt_cost(replay.cost)
+          result_cost_text(result)
         ].compact.join(" · ")
       end
 
+      # --- WHAT TO FIX -----------------------------------------------------
+
+      # The section stands even for a run with nothing to fix — the dashboard
+      # keeps it too, so a clean run reads as clean rather than as a page
+      # missing a section. A report over no results at all has nothing to say.
+      def html_fixes
+        return "" if @results.empty?
+
+        items = fix_items
+        faulted = @results.reject(&:passed?)
+        meta = "#{plural(items.size, 'item')} · #{plural(faulted.size, 'fault')} across " \
+               "#{plural(faulted.map { |result| result.scenario.key }.uniq.size, 'scenario')}"
+        body =
+          if items.any?
+            %(<div class="fixes">#{items.map { |item| html_fix_card(item) }.join}</div>)
+          else
+            %(<div class="nothing">[+] nothing to fix</div>)
+          end
+
+        <<~FIXES
+          <section class="section fix-section" aria-label="Recommendations">
+          <div class="section-head"><span class="micro">What to fix</span><span class="meta">#{h(meta)}</span></div>
+          #{html_fix_filter(items) if comparing? && items.any?}
+          #{body}
+          </section>
+        FIXES
+      end
+
+      # A model filter for the fix cards — a fault one model keeps making is
+      # that model's to fix, so the list narrows to what was attributed to
+      # it. Radio chips and stylesheet rules alone (the page carries no
+      # script): each card names its models in data-models, and a checked
+      # model hides every card that does not name it. Cards attributed to no
+      # model (an older run) stay under every filter.
+      def html_fix_filter(items)
+        chips = [ %(<label class="chip pick-model"><input type="radio" name="fix-model" value="all" checked><span>all models #{items.size}</span></label>) ]
+        @models.each_with_index do |spec, index|
+          count = items.count { |item| Array(item["models"]).empty? || item["models"].include?(spec.label) }
+          chips << %(<label class="chip pick-model"><input type="radio" name="fix-model" value="m#{index}"><span>#{h(short_name(spec))} #{count}</span></label>)
+        end
+        %(<div class="fix-filter"><span class="micro sm">for</span>#{chips.join}</div>)
+      end
+
+      def fix_model_tokens(item)
+        labels = Array(item["models"])
+        return "" if labels.empty?
+
+        labels.filter_map { |label| (index = @models.index(model_by_label(label))) && "m#{index}" }.join(" ")
+      end
+
+      def html_fix_card(item)
+        tone = item["kind"] == "instruction" ? "info" : "error"
+        glyph = tone == "info" ? "[i]" : "[!]"
+        title = fault_name(item["fault"]) + (item["count"].to_i > 1 ? " ×#{item['count']}" : "")
+
+        parts = [ %(<div class="head"><span class="glyph tone-#{tone}">#{glyph}</span>) +
+                  %(<span class="badge #{tone}">#{h(title)}</span><span class="scope">#{h(fix_scope(item))}</span></div>) ]
+        parts << %(<p>#{h(item['recommendation'])}</p>) if item["recommendation"].present?
+        parts << %(<div class="quote">“#{h(item['quote'])}”</div>) if item["quote"].present?
+        parts << html_fix_tools(item) if item["tools"].any?
+        parts << html_fix_server(item["server"]) if item["server"]
+        parts << %(<div class="note">#{h(item['note'])}</div>) if item["note"].present?
+        parts << html_fix_action(item["action"]) if item["action"]
+        models = fix_model_tokens(item)
+        %(<div class="fix"#{%( data-models="#{models}") if models.present?}>#{parts.join}</div>)
+      end
+
+      def html_fix_tools(item)
+        chips = item["tools"].map do |tool|
+          note = tool["note"].presence
+          %(<span class="tool"><b>#{h(tool['name'])}</b>#{%(<span class="note">#{h(note)}</span>) if note}</span>)
+        end
+        %(<div class="tools"><span class="micro sm">#{h(item['tools_label'])}</span><div class="list">#{chips.join}</div></div>)
+      end
+
+      # "available · not enabled for Assistant", "unknown · not enabled for
+      # Assistant" — every status but "enabled" leads with the status word, the
+      # way the dashboard's fix list reads it.
+      def html_fix_server(server)
+        badge =
+          if server["status"] == "enabled"
+            %(<span class="badge success xs">enabled for #{h(@agent_name)}</span>)
+          else
+            %(<span class="badge warning xs">#{h(server['status'].presence || 'unknown')} · not enabled for #{h(@agent_name)}</span>)
+          end
+        %(<div class="served"><span>served by</span><b>#{h(server['name'].presence || server['key'])}</b>#{badge}</div>)
+      end
+
+      # With a route the action is a button; without one, the page can only
+      # say where in the dashboard the fix lives. The link targets the top
+      # window: served in the dashboard's report iframe it would otherwise
+      # open the whole dashboard inside the frame.
+      def html_fix_action(action)
+        button = action["path"].present? ? %(<a class="btn" target="_top" href="#{h(action['path'])}">#{h(action['label'])}</a>) : ""
+        %(<div class="action">#{button}<span class="hint">#{h(action['hint'])}</span></div>)
+      end
+
+      # "3 scenarios · both models" — the models are worth naming only on a
+      # comparison run; on a single-model run the count says it all.
+      def fix_scope(item)
+        return "#{item['scenario_keys'].join(', ')} · judge suggestion" if item["kind"] == "instruction"
+
+        scenarios = plural(item["scenario_keys"].size, "scenario")
+        labels = Array(item["models"])
+        return scenarios unless comparing? && labels.any?
+
+        models =
+          if labels.size >= @models.size
+            @models.size == 2 ? "both models" : "all models"
+          else
+            labels.map { |label| short_name(model_by_label(label)) }.join(", ")
+          end
+        "#{scenarios} · #{models}"
+      end
+
       # --- footer ----------------------------------------------------------
 
+      # The run's terms: the judge, the criteria and the pass mark, what it
+      # cost on each side, the metadata, and — once, when any figure on the
+      # page carries a "~" — what the mark means.
       def html_footer
         criteria = criterion_keys.map { |key| key.to_s.tr("_", " ") }.join(" · ")
         spans = [ %(<span class="nowrap">judge #{h(judge_name)}</span>) ]
         spans << %(<span class="criteria">criteria #{h(criteria)}</span>) if criteria.present?
-        spans.concat(@metadata.to_h.map { |key, value| %(<span class="nowrap">#{h(key)} #{h(value)}</span>) })
+        spans << %(<span class="nowrap">#{h(Format.threshold(@threshold))}</span>)
+        spans << %(<span class="nowrap">#{h(footer_cost_text)}</span>) if footer_cost_text
+        spans << %(<span class="nowrap">release #{h(release_label)}</span>) if release_label
+        spans.concat(metadata_chips.map { |key, value| %(<span class="nowrap">#{h(key)} #{h(value)}</span>) })
+        spans << %(<span class="legend">#{h(Format::LEGEND)}</span>) if estimated_anywhere?
         %(<footer>#{spans.join}</footer>)
       end
 
+      # "cost ~$0.0412 · agent ~$0.0397 · judge ~$0.0015", or nil when
+      # nothing was priced.
+      def footer_cost_text
+        costs = run_costs
+        return nil if costs["total"].nil?
+
+        parts = [ "cost #{fmt_cost(costs['total'], estimated: costs['estimated'])}" ]
+        if judge_usage
+          parts << "agent #{fmt_cost(costs['cost'], estimated: costs['estimated'])}"
+          parts << "judge #{fmt_cost(costs['judge_cost'], estimated: judge_estimated?)}"
+        end
+        parts.join(" · ")
+      end
+
       # Colors only through the token variables; radii 4 badges · 6 chips ·
       # 8 controls · 10 nested panels · 12 cards · 999 bars; no shadows.
       STYLES = <<~CSS.freeze
@@ -557,6 +811,24 @@ module ActiveAgent
         .tok .in { color: var(--color-token-in); }
         .tok .out { color: var(--color-token-out); }
         .faults { display: flex; gap: 6px; flex-wrap: wrap; }
+        .compare { overflow-x: auto; border-top: 1px solid var(--color-border-light); }
+        .compare table { width: 100%; border-collapse: collapse; }
+        .compare th { padding: 8px 12px; text-align: left; vertical-align: bottom; white-space: nowrap; font-family: var(--font-mono); font-size: 10px; font-weight: 600; letter-spacing: 0.05em; text-transform: uppercase; color: var(--color-text-muted); background: var(--color-muted); }
+        .compare td { padding: 9px 12px; vertical-align: top; border-top: 1px solid var(--color-border-light); font-size: 13px; color: var(--color-text-cell); }
+        .compare th.num, .compare td.num { text-align: right; }
+        .compare td.num { font-family: var(--font-mono); font-size: 12px; white-space: nowrap; }
+        .compare td.ratio { font-weight: 600; }
+        .compare .per { display: block; font-weight: 400; color: var(--color-text-muted); }
+        .compare .model-cell { white-space: nowrap; }
+        .compare .model-cell .name { font-family: var(--font-mono); font-size: 12px; font-weight: 600; color: var(--color-text-primary); }
+        .compare .model-cell .provider { display: block; font-family: var(--font-mono); font-size: 11px; color: var(--color-text-muted); }
+        .compare .pick { margin-left: 6px; font-family: var(--font-mono); font-size: 10px; font-weight: 700; color: var(--color-warning-text); }
+        .compare th.fault, .compare td.fault { width: 34%; }
+        .compare td.fault .detail { color: var(--color-text-secondary); }
+        .fix-filter { display: flex; align-items: center; gap: 6px; flex-wrap: wrap; }
+        .pick-model { cursor: pointer; border: 1px solid var(--color-border); background: var(--color-card); }
+        .pick-model input { position: absolute; opacity: 0; width: 0; height: 0; }
+        .pick-model:has(input:checked) { border-color: var(--color-accent-ui); background: var(--color-accent-ui-tint); color: var(--color-accent-ui); }
         .clean { font-family: var(--font-mono); font-size: 11px; color: var(--color-success-text); }
         .verdict { padding: 10px 12px; border-top: 1px solid var(--color-border-light); font-size: 12px; line-height: 18px; color: var(--color-text-cell); }
         .verdict .micro { margin-right: 8px; }
@@ -590,6 +862,10 @@ module ActiveAgent
         .group-name { font-size: 12px; font-weight: 600; }
         .count { font-family: var(--font-mono); font-size: 11px; color: var(--color-text-muted); }
         .group-pass { font-family: var(--font-mono); font-size: 11px; font-weight: 600; color: var(--color-text-cell); }
+        .mx .cost { display: flex; flex-direction: column; gap: 1px; min-width: 0; font-family: var(--font-mono); font-size: 11px; color: var(--color-text-secondary); text-align: right; }
+        .mx .cost b { font-weight: 600; color: var(--color-text-primary); }
+        .mx .cost .judge { color: var(--color-text-muted); white-space: nowrap; }
+        .mx .cost-head { text-align: right; }
         .key { font-family: var(--font-mono); font-size: 11px; color: var(--color-text-muted); margin-bottom: 2px; }
         .key a { color: inherit; }
         .prompt { font-size: 13px; line-height: 18px; color: var(--color-text-primary); }
@@ -603,6 +879,7 @@ module ActiveAgent
         .calls { display: flex; flex-wrap: wrap; gap: 2px 8px; font-family: var(--font-mono); font-size: 11px; color: var(--color-text-muted); }
         .call-hit { color: var(--color-success-text); font-weight: 600; }
         .call-err { color: var(--color-error); font-weight: 600; }
+        .cost-line { font-family: var(--font-mono); font-size: 11px; color: var(--color-text-secondary); white-space: nowrap; }
         .details { display: flex; flex-direction: column; gap: 8px; }
         details { border: 1px solid var(--color-border-light); border-radius: 10px; overflow: hidden; }
         summary { display: flex; align-items: center; gap: 10px; padding: 10px 12px; cursor: pointer; list-style: none; flex-wrap: wrap; }
@@ -628,6 +905,7 @@ module ActiveAgent
         footer { display: flex; align-items: center; gap: 16px; flex-wrap: wrap; padding-top: 12px; border-top: 1px solid var(--color-border-light); font-family: var(--font-mono); font-size: 11px; color: var(--color-text-muted); }
         footer .criteria { min-width: 0; }
         footer .nowrap { white-space: nowrap; }
+        footer .legend { margin-left: auto; white-space: nowrap; color: var(--color-text-muted); }
       CSS
     end
   end