@post-print/agent-test 1.1.1 → 1.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. package/README.md +4 -2
  2. package/dist/cli.d.ts.map +1 -1
  3. package/dist/cli.js +34 -6
  4. package/dist/cli.js.map +1 -1
  5. package/dist/compare-scenario.d.ts +10 -0
  6. package/dist/compare-scenario.d.ts.map +1 -1
  7. package/dist/compare-scenario.js +48 -3
  8. package/dist/compare-scenario.js.map +1 -1
  9. package/dist/expect.d.ts +1 -0
  10. package/dist/expect.d.ts.map +1 -1
  11. package/dist/expect.js +24 -0
  12. package/dist/expect.js.map +1 -1
  13. package/dist/html-report.d.ts +4 -1
  14. package/dist/html-report.d.ts.map +1 -1
  15. package/dist/html-report.js +223 -142
  16. package/dist/html-report.js.map +1 -1
  17. package/dist/index.d.ts +3 -2
  18. package/dist/index.d.ts.map +1 -1
  19. package/dist/index.js +3 -2
  20. package/dist/index.js.map +1 -1
  21. package/dist/live-isolation.d.ts +2 -0
  22. package/dist/live-isolation.d.ts.map +1 -1
  23. package/dist/live-isolation.js +8 -2
  24. package/dist/live-isolation.js.map +1 -1
  25. package/dist/report-preview.d.ts +24 -0
  26. package/dist/report-preview.d.ts.map +1 -1
  27. package/dist/report-preview.js +46 -2
  28. package/dist/report-preview.js.map +1 -1
  29. package/dist/run-suite.d.ts +10 -0
  30. package/dist/run-suite.d.ts.map +1 -1
  31. package/dist/run-suite.js +147 -126
  32. package/dist/run-suite.js.map +1 -1
  33. package/dist/scenario-finalizer.d.ts +27 -0
  34. package/dist/scenario-finalizer.d.ts.map +1 -0
  35. package/dist/scenario-finalizer.js +39 -0
  36. package/dist/scenario-finalizer.js.map +1 -0
  37. package/dist/scenario-story.d.ts.map +1 -1
  38. package/dist/scenario-story.js +14 -8
  39. package/dist/scenario-story.js.map +1 -1
  40. package/dist/types.d.ts +9 -0
  41. package/dist/types.d.ts.map +1 -1
  42. package/dist/validate-suite.d.ts.map +1 -1
  43. package/dist/validate-suite.js +4 -0
  44. package/dist/validate-suite.js.map +1 -1
  45. package/dist/viewer/catalog.d.ts +14 -3
  46. package/dist/viewer/catalog.d.ts.map +1 -1
  47. package/dist/viewer/catalog.js +37 -6
  48. package/dist/viewer/catalog.js.map +1 -1
  49. package/dist/viewer/compare-metrics.d.ts.map +1 -1
  50. package/dist/viewer/compare-metrics.js +8 -9
  51. package/dist/viewer/compare-metrics.js.map +1 -1
  52. package/dist/viewer/events.d.ts +22 -1
  53. package/dist/viewer/events.d.ts.map +1 -1
  54. package/dist/viewer/events.js +1 -0
  55. package/dist/viewer/events.js.map +1 -1
  56. package/dist/viewer/live-runner.d.ts +7 -0
  57. package/dist/viewer/live-runner.d.ts.map +1 -1
  58. package/dist/viewer/live-runner.js +36 -23
  59. package/dist/viewer/live-runner.js.map +1 -1
  60. package/dist/viewer/page.d.ts +14 -1
  61. package/dist/viewer/page.d.ts.map +1 -1
  62. package/dist/viewer/page.js +848 -141
  63. package/dist/viewer/page.js.map +1 -1
  64. package/dist/viewer/run-controller.d.ts +13 -6
  65. package/dist/viewer/run-controller.d.ts.map +1 -1
  66. package/dist/viewer/run-controller.js +246 -74
  67. package/dist/viewer/run-controller.js.map +1 -1
  68. package/dist/viewer/server.d.ts +5 -0
  69. package/dist/viewer/server.d.ts.map +1 -1
  70. package/dist/viewer/server.js +72 -17
  71. package/dist/viewer/server.js.map +1 -1
  72. package/dist/viewer/session-server.d.ts +2 -0
  73. package/dist/viewer/session-server.d.ts.map +1 -0
  74. package/dist/viewer/session-server.js +71 -0
  75. package/dist/viewer/session-server.js.map +1 -0
  76. package/package.json +2 -2
@@ -5,7 +5,7 @@ import { compareArmTokens, compareArmTurns, compareResultArms, describeCompareOu
5
5
  import { renderContextPanel } from "./context-panel.js";
6
6
  import { displayToolPath } from "./scenario-story.js";
7
7
  import { summarizeReports } from "./suite-summary.js";
8
- import { renderViewerCompareBoard, summarizeViewerCompare } from "./viewer/compare-metrics.js";
8
+ import { renderViewerPage } from "./viewer/page.js";
9
9
  function escapeHtml(value) {
10
10
  return value
11
11
  .replaceAll("&", "&")
@@ -97,6 +97,11 @@ const MATCHER_LABELS = {
97
97
  mustNotInclude: FORBIDDEN_BEHAVIOR,
98
98
  mustRun: MISSING_COMMAND,
99
99
  toHaveRunCommand: MISSING_COMMAND,
100
+ mustRunSuccessfully: MISSING_COMMAND,
101
+ toHaveRunCommandSuccessfully: {
102
+ label: "Command did not pass",
103
+ hint: "The required command was missing, failed, or did not report a structured execution status.",
104
+ },
100
105
  allowedCommands: FORBIDDEN_COMMAND,
101
106
  toHaveAllowedCommands: FORBIDDEN_COMMAND,
102
107
  mustCallTool: MISSING_TOOL,
@@ -283,10 +288,21 @@ function renderJudgeVerdicts(result) {
283
288
  .map((verdict) => {
284
289
  const badge = verdict.pass ? "pass" : "fail";
285
290
  const icon = verdict.pass ? "✓" : "✗";
291
+ const evidence = verdict.evidence?.length
292
+ ? `<ul class="judge-evidence">${verdict.evidence.map((line) => `<li>${escapeHtml(line)}</li>`).join("")}</ul>`
293
+ : "";
294
+ const exchange = verdict.prompt || verdict.response
295
+ ? `<details class="judge-exchange"><summary>Judge conversation</summary><div class="judge-chat">
296
+ ${verdict.prompt ? `<div class="judge-turn judge-input"><span>Input</span><pre>${escapeHtml(verdict.prompt)}</pre></div>` : ""}
297
+ ${verdict.response ? `<div class="judge-turn judge-response"><span>Response</span><pre>${escapeHtml(verdict.response)}</pre></div>` : ""}
298
+ </div></details>`
299
+ : "";
286
300
  return `
287
301
  <article class="verdict verdict-${badge}">
288
302
  <p class="question"><span class="verdict-icon">${icon}</span>${escapeHtml(verdict.question)}</p>
289
303
  <p class="rationale">${escapeHtml(verdict.rationale)}</p>
304
+ ${evidence}
305
+ ${exchange}
290
306
  </article>`;
291
307
  })
292
308
  .join("\n");
@@ -467,13 +483,45 @@ function renderCompareMetricRow(label, aText, bText, delta, deltaText) {
467
483
  <td class="${deltaClass(delta)}">${escapeHtml(deltaText)}</td>
468
484
  </tr>`;
469
485
  }
470
- function renderNamedCompareMetricRow(label, cells) {
486
+ function renderNamedCompareMetricRow(label, cells, values) {
487
+ const numeric = (values ?? []).filter((value) => value !== undefined);
488
+ const low = numeric.length > 1 ? Math.min(...numeric) : undefined;
489
+ const high = numeric.length > 1 ? Math.max(...numeric) : undefined;
471
490
  return `<tr>
472
491
  <th scope="row">${escapeHtml(label)}</th>
473
- ${cells.map((cell) => `<td>${escapeHtml(cell)}</td>`).join("")}
492
+ ${cells
493
+ .map((cell, index) => {
494
+ const value = values?.[index];
495
+ const state = value !== undefined && low !== high
496
+ ? value === low
497
+ ? "is-better"
498
+ : value === high
499
+ ? "is-worse"
500
+ : ""
501
+ : "";
502
+ return `<td class="${state}">${escapeHtml(cell)}</td>`;
503
+ })
504
+ .join("")}
474
505
  </tr>`;
475
506
  }
476
- function renderTwoArmCompareMetrics(arms) {
507
+ function outcomeCell(arm, gates) {
508
+ if (arm.passed === undefined)
509
+ return "<td>n/a</td>";
510
+ if (arm.passed)
511
+ return '<td class="is-outcome-pass">pass</td>';
512
+ const expected = gates?.some((gate) => !("winner" in gate) &&
513
+ gate.arm === arm.id &&
514
+ gate.metric === "outcome" &&
515
+ gate.operator === "equal" &&
516
+ gate.value === "fail");
517
+ return expected
518
+ ? '<td class="is-expected-failure">expected fail</td>'
519
+ : '<td class="is-outcome-fail">fail</td>';
520
+ }
521
+ function renderOutcomeRow(arms, gates, includeDelta = false) {
522
+ return `<tr><th scope="row">Outcome</th>${arms.map((arm) => outcomeCell(arm, gates)).join("")}${includeDelta ? '<td class="delta-flat">n/a</td>' : ""}</tr>`;
523
+ }
524
+ function renderTwoArmCompareMetrics(arms, gates) {
477
525
  const [left, right] = arms;
478
526
  const aTurns = compareArmTurns(left);
479
527
  const bTurns = compareArmTurns(right);
@@ -482,9 +530,6 @@ function renderTwoArmCompareMetrics(arms) {
482
530
  const bTools = armToolCount(right);
483
531
  const toolDelta = aTools !== undefined && bTools !== undefined ? bTools - aTools : undefined;
484
532
  const totalDelta = tokenDelta(left, right, "total");
485
- const durationDelta = left.durationMs !== undefined && right.durationMs !== undefined
486
- ? right.durationMs - left.durationMs
487
- : undefined;
488
533
  return `
489
534
  <div class="compare-table-wrap">
490
535
  <table class="compare-table">
@@ -497,18 +542,17 @@ function renderTwoArmCompareMetrics(arms) {
497
542
  </tr>
498
543
  </thead>
499
544
  <tbody>
500
- ${renderCompareMetricRow("Outcome", left.passed === undefined ? "n/a" : left.passed ? "pass" : "fail", right.passed === undefined ? "n/a" : right.passed ? "pass" : "fail", undefined, "n/a")}
545
+ ${renderOutcomeRow(arms, gates, true)}
501
546
  ${renderCompareMetricRow("Turns", formatArmTurns(left), formatArmTurns(right), turnDelta, formatSigned(turnDelta))}
502
547
  ${renderCompareMetricRow("Tokens", formatArmTokens(left, "total"), formatArmTokens(right, "total"), totalDelta, formatSigned(totalDelta))}
503
548
  ${renderCompareMetricRow("In", formatArmTokens(left, "input"), formatArmTokens(right, "input"), tokenDelta(left, right, "input"), formatSigned(tokenDelta(left, right, "input")))}
504
549
  ${renderCompareMetricRow("Out", formatArmTokens(left, "output"), formatArmTokens(right, "output"), tokenDelta(left, right, "output"), formatSigned(tokenDelta(left, right, "output")))}
505
550
  ${renderCompareMetricRow("Tools", aTools === undefined ? "n/a" : formatInteger(aTools), bTools === undefined ? "n/a" : formatInteger(bTools), toolDelta, formatSigned(toolDelta))}
506
- ${renderCompareMetricRow("Duration", left.durationMs === undefined ? "n/a" : formatDuration(left.durationMs), right.durationMs === undefined ? "n/a" : formatDuration(right.durationMs), durationDelta, formatSigned(durationDelta))}
507
551
  </tbody>
508
552
  </table>
509
553
  </div>`;
510
554
  }
511
- function renderNamedCompareMetrics(arms) {
555
+ function renderNamedCompareMetrics(arms, gates) {
512
556
  return `
513
557
  <div class="compare-table-wrap">
514
558
  <table class="compare-table">
@@ -519,16 +563,15 @@ function renderNamedCompareMetrics(arms) {
519
563
  </tr>
520
564
  </thead>
521
565
  <tbody>
522
- ${renderNamedCompareMetricRow("Outcome", arms.map((arm) => (arm.passed === undefined ? "n/a" : arm.passed ? "pass" : "fail")))}
523
- ${renderNamedCompareMetricRow("Turns", arms.map((arm) => formatArmTurns(arm)))}
524
- ${renderNamedCompareMetricRow("Tokens", arms.map((arm) => formatArmTokens(arm, "total")))}
566
+ ${renderOutcomeRow(arms, gates)}
567
+ ${renderNamedCompareMetricRow("Turns", arms.map((arm) => formatArmTurns(arm)), arms.map(compareArmTurns))}
568
+ ${renderNamedCompareMetricRow("Tokens", arms.map((arm) => formatArmTokens(arm, "total")), arms.map(compareArmTokens))}
525
569
  ${renderNamedCompareMetricRow("In", arms.map((arm) => formatArmTokens(arm, "input")))}
526
570
  ${renderNamedCompareMetricRow("Out", arms.map((arm) => formatArmTokens(arm, "output")))}
527
571
  ${renderNamedCompareMetricRow("Tools", arms.map((arm) => {
528
572
  const tools = armToolCount(arm);
529
573
  return tools === undefined ? "n/a" : formatInteger(tools);
530
- }))}
531
- ${renderNamedCompareMetricRow("Duration", arms.map((arm) => (arm.durationMs === undefined ? "n/a" : formatDuration(arm.durationMs))))}
574
+ }), arms.map(armToolCount))}
532
575
  </tbody>
533
576
  </table>
534
577
  </div>`;
@@ -542,31 +585,22 @@ function renderCompareMetrics(result) {
542
585
  if (arms.length === 0) {
543
586
  return "";
544
587
  }
588
+ const twoArm = arms.length === 2 && arms[0] && arms[1];
545
589
  const callouts = describeCompareOutcome(compare);
546
- const calloutList = callouts.length > 0
547
- ? `<ul class="compare-callouts">${callouts.map((line) => `<li>${escapeHtml(line)}</li>`).join("")}</ul>`
590
+ const calloutList = callouts.length
591
+ ? `<details class="compare-details"><summary>Arm details</summary><ul class="compare-callouts">${callouts.map((line) => `<li>${escapeHtml(line)}</li>`).join("")}</ul></details>`
548
592
  : "";
549
- const twoArm = arms.length === 2 && arms[0] && arms[1];
550
593
  const lede = twoArm
551
- ? `${arms[0].label} vs ${arms[1].label}. Δ is B minus A. A lower turn count and a lower token count is better.`
552
- : `${arms.map((arm) => arm.label).join(", ")}. A lower turn count and a lower token count is better. Named pairs do not pick one winner.`;
553
- const winners = renderViewerCompareBoard(summarizeViewerCompare(arms.map((arm) => ({
554
- id: arm.id,
555
- label: arm.label,
556
- turns: compareArmTurns(arm),
557
- tokens: compareArmTokens(arm),
558
- tools: arm.trace ? arm.trace.toolCalls.length : undefined,
559
- durationMs: arm.durationMs,
560
- })), compare.gateResults));
594
+ ? `${arms[0].label} vs ${arms[1].label}. Lower is better: green is lowest and red is highest.`
595
+ : `${arms.map((arm) => arm.label).join(", ")}. Lower is better: green is lowest and red is highest. Ties are neutral.`;
561
596
  return `
562
- <section class="compare">
597
+ <section class="compare comparison-metrics">
563
598
  <header class="compare-header">
564
- <h3>Comparison</h3>
599
+ <h4>Comparison metrics</h4>
565
600
  <p class="muted">${escapeHtml(lede)}</p>
566
601
  </header>
567
- ${winners}
602
+ ${twoArm ? renderTwoArmCompareMetrics([arms[0], arms[1]], compare.gates) : renderNamedCompareMetrics(arms, compare.gates)}
568
603
  ${calloutList}
569
- ${twoArm ? renderTwoArmCompareMetrics([arms[0], arms[1]]) : renderNamedCompareMetrics(arms)}
570
604
  </section>`;
571
605
  }
572
606
  function renderArmMetrics(arm) {
@@ -617,28 +651,26 @@ function renderArmColumn(arm, index) {
617
651
  ${renderArmFailures(arm)}
618
652
  </header>
619
653
  ${arm.contextMode || arm.contextFiles || arm.hostInput ? renderContextPanel(arm.contextFiles, arm.contextMode, arm.hostInput) : ""}
620
- ${renderChat(arm.trace, arm.prompt)}
654
+ ${renderTraceDetails(arm.trace, arm.prompt)}
621
655
  </article>`;
622
656
  }
657
+ function renderTraceDetails(trace, prompt) {
658
+ const tools = trace?.toolCalls.length ?? 0;
659
+ const shell = trace?.shellCommands.length ?? 0;
660
+ const messages = trace?.messages.length ?? 0;
661
+ const label = `${messages} messages · ${tools} tools${shell ? ` · ${shell} shell command${shell === 1 ? "" : "s"}` : ""}`;
662
+ return `<details class="trace-details" open><summary>Conversation & evidence <span>${escapeHtml(label)}</span></summary>${renderChat(trace, prompt)}</details>`;
663
+ }
623
664
  function renderCompareConversations(result, host) {
624
665
  const compare = result.compare;
625
666
  if (!compare) {
626
667
  return `<section class="conversation">
627
668
  <h3>Conversation</h3>
628
669
  ${result.contextMode || result.contextFiles || result.hostInput ? renderContextPanel(result.contextFiles, result.contextMode, result.hostInput) : ""}
629
- ${renderChat(result.trace, result.prompt)}
670
+ ${renderTraceDetails(result.trace, result.prompt)}
630
671
  </section>`;
631
672
  }
632
673
  const arms = compareResultArms(compare);
633
- if (arms.length <= 2) {
634
- return `
635
- <div class="compare-layout">
636
- <div class="compare-arms" data-arm-count="${arms.length}">
637
- ${arms.map((arm, index) => renderArmColumn(arm, index)).join("")}
638
- </div>
639
- ${renderCompareMetrics(result)}
640
- </div>`;
641
- }
642
674
  const group = compareTabGroupId(result, host);
643
675
  const tablist = arms
644
676
  .map((arm, index) => {
@@ -657,69 +689,58 @@ function renderCompareConversations(result, host) {
657
689
  <div class="compare-arms" data-arm-count="${arms.length}">
658
690
  ${arms.map((arm, index) => renderArmColumn(arm, index)).join("")}
659
691
  </div>
660
- ${renderCompareMetrics(result)}
661
692
  </div>`;
662
693
  }
663
- function renderStoryList(title, lines) {
664
- if (!lines || lines.length === 0) {
665
- return "";
666
- }
667
- const items = lines.map((line) => `<li>${escapeHtml(line)}</li>`).join("");
668
- return `<section class="story-block"><h3>${escapeHtml(title)}</h3><ul class="story-list">${items}</ul></section>`;
669
- }
670
694
  function renderStoryCheck(check) {
671
695
  const icon = check.status === "pass" ? "✓" : check.status === "fail" ? "✗" : "·";
672
696
  return `<li class="story-check story-check-${check.status}"><span class="story-check-icon">${icon}</span>${escapeHtml(check.text)}</li>`;
673
697
  }
674
698
  function renderStorySection(section) {
675
699
  const titled = Boolean(section.title);
676
- const title = section.title ? `<h3>${escapeHtml(section.title)}</h3>` : `<h3>Criteria</h3>`;
700
+ const title = section.title ? `<h4>${escapeHtml(section.title)}</h4>` : "";
677
701
  const description = section.description
678
702
  ? `<p class="story-section-description">${escapeHtml(section.description)}</p>`
679
703
  : "";
680
704
  const checks = section.checks.length > 0
681
705
  ? `<ul class="story-checks">${section.checks.map(renderStoryCheck).join("")}</ul>`
682
706
  : "";
683
- const notes = section.notes && section.notes.length > 0
684
- ? `<ul class="story-notes">${section.notes.map((note) => `<li>${escapeHtml(note)}</li>`).join("")}</ul>`
685
- : "";
686
707
  const titledClass = titled ? " story-section-titled" : "";
687
- return `<section class="story-section${titledClass}">${title}${description}${checks}${notes}</section>`;
708
+ return `<div class="story-section${titledClass}">${title}${description}${checks}</div>`;
688
709
  }
689
- function renderStory(result) {
710
+ function renderStory(result, comparison = "") {
690
711
  const story = result.story;
691
- if (!story) {
712
+ if (!story && !comparison) {
692
713
  return "";
693
714
  }
694
- if (story.sections && story.sections.length > 0) {
695
- const titled = story.sections.some((section) => section.title);
696
- if (!titled && story.sections.length === 1 && story.sections[0]) {
697
- const section = story.sections[0];
698
- const criteria = section.checks.length > 0
699
- ? `<section class="story-block"><h3>Criteria</h3><ul class="story-checks">${section.checks.map(renderStoryCheck).join("")}</ul></section>`
700
- : "";
701
- return `<div class="story">${criteria}${renderStoryList("Result", [
702
- ...(section.notes ?? []),
703
- ...story.verdict,
704
- ])}</div>`;
705
- }
706
- const sections = story.sections.map(renderStorySection).join("");
707
- const leftover = story.verdict.length > 0 ? renderStoryList("Result", story.verdict) : "";
708
- return `<div class="story story-sections">${sections}${leftover}</div>`;
709
- }
710
- return `<div class="story">
711
- ${renderStoryList("Criteria", story.criteria)}
712
- ${renderStoryList("Result", [...story.result, ...story.verdict])}
713
- </div>`;
715
+ const sections = story?.sections?.filter((section) => section.checks.length > 0) ?? [];
716
+ const storyCriteria = sections.length
717
+ ? sections.map(renderStorySection).join("")
718
+ : story?.criteria.length
719
+ ? `<ul class="story-checks">${story.criteria
720
+ .map((text) => renderStoryCheck({ text, status: result.passed ? "pass" : "fail" }))
721
+ .join("")}</ul>`
722
+ : "";
723
+ const fallbackCompareCriteria = result.compare?.gateResults?.length
724
+ ? `<div class="story-section story-section-titled"><h4>Compare</h4><ul class="story-checks">${result.compare.gateResults
725
+ .map((gate) => renderStoryCheck({
726
+ text: gate.message,
727
+ status: gate.passed ? "pass" : "fail",
728
+ }))
729
+ .join("")}</ul></div>`
730
+ : "";
731
+ const criteria = storyCriteria || fallbackCompareCriteria;
732
+ if (!criteria && !comparison)
733
+ return "";
734
+ return `<div class="story"><section class="story-criteria"><h3>Pass criteria</h3>${criteria}${comparison}</section></div>`;
714
735
  }
715
- function renderScenario(result, host) {
716
- const open = result.skipped ? "" : !result.passed || result.compare ? " open" : "";
717
- const failures = result.story ? "" : renderFailures(result);
736
+ export function renderScenarioResult(result, host) {
737
+ const open = result.skipped ? "" : " open";
738
+ const failures = renderFailures(result);
718
739
  const judgeVerdicts = renderJudgeVerdicts(result);
719
740
  const tokens = formatTokensBadge(result);
720
741
  const usageDetail = result.compare ? "" : renderUsageDetail(usageOf(result));
721
742
  const traceMeta = result.compare ? "" : renderTraceMeta(result);
722
- const story = renderStory(result);
743
+ const story = renderStory(result, renderCompareMetrics(result));
723
744
  const diagnostics = story || failures || judgeVerdicts
724
745
  ? `<div class="diagnostics">
725
746
  ${story}
@@ -752,20 +773,6 @@ function renderScenario(result, host) {
752
773
  </div>
753
774
  </details>`;
754
775
  }
755
- function renderSuite(report) {
756
- const scenarios = report.results.map((result) => renderScenario(result, report.host)).join("\n");
757
- return `
758
- <section class="suite">
759
- <header class="suite-header">
760
- <div class="suite-title">
761
- <h2>${escapeHtml(report.suite)}</h2>
762
- <span class="host">${escapeHtml(report.host)}</span>
763
- </div>
764
- <p class="suite-counts"><span>${formatInteger(report.passed)} passed</span><span>${formatInteger(report.failed)} failed</span><span>${formatInteger(report.skipped)} skipped</span></p>
765
- </header>
766
- ${scenarios}
767
- </section>`;
768
- }
769
776
  export function reportCss() {
770
777
  return sharedReportCss();
771
778
  }
@@ -921,10 +928,12 @@ function sharedReportCss() {
921
928
  .diagnostics section { background: var(--panel-2); border: 1px solid var(--border); border-radius: 8px; padding: 0.65rem 0.7rem; min-width: 0; }
922
929
  .conversation { min-width: 0; }
923
930
 
924
- .story { display: grid; grid-template-columns: repeat(auto-fit, minmax(14rem, 1fr)); gap: 0.85rem; margin-bottom: 0.85rem; }
925
- .story-sections { display: flex; flex-direction: column; gap: 0.85rem; }
926
- .story-block h3, .story-section h3 { margin: 0 0 0.35rem; font-size: 0.78rem; text-transform: uppercase; letter-spacing: 0.04em; color: var(--muted); }
927
- .story-section-titled h3 { text-transform: none; letter-spacing: 0; font-size: 0.9rem; color: var(--text); }
931
+ .story { margin-bottom: 0.85rem; }
932
+ .story-criteria { display: grid; gap: 0.55rem; }
933
+ .story-criteria > h3 { margin: 0; font-size: 0.78rem; text-transform: uppercase; letter-spacing: 0.04em; color: var(--muted); }
934
+ .story-section { display: grid; gap: 0.3rem; }
935
+ .story-section + .story-section { padding-block-start: 0.55rem; border-block-start: 1px solid var(--border); }
936
+ .story-section h4 { margin: 0; font-size: 0.84rem; color: var(--text); }
928
937
  .story-section-description { margin: 0 0 0.4rem; color: var(--muted); font-size: 0.82rem; }
929
938
  .story-list, .story-checks, .story-notes { margin: 0; padding-left: 1.1rem; display: flex; flex-direction: column; gap: 0.25rem; color: var(--text); }
930
939
  .story-checks { list-style: none; padding-left: 0; }
@@ -949,6 +958,15 @@ function sharedReportCss() {
949
958
  .verdict-fail .verdict-icon { color: var(--fail); }
950
959
  .question { margin: 0; font-size: 0.85rem; font-weight: 600; }
951
960
  .rationale { margin: 0.3rem 0 0; font-size: 0.82rem; color: var(--muted); }
961
+ .judge-evidence { margin: 0.35rem 0 0; padding-inline-start: 1.1rem; color: var(--muted); font-size: 0.78rem; }
962
+ .judge-exchange { margin-block-start: 0.5rem; border: 1px solid var(--border); border-radius: 7px; overflow: hidden; }
963
+ .judge-exchange > summary { cursor: pointer; padding: 0.4rem 0.55rem; color: var(--muted); font-size: 0.75rem; font-weight: 650; }
964
+ .judge-exchange[open] > summary { border-block-end: 1px solid var(--border); background: var(--panel); }
965
+ .judge-chat { display: grid; gap: 0.55rem; padding: 0.55rem; }
966
+ .judge-turn { display: grid; gap: 0.25rem; min-width: 0; }
967
+ .judge-turn > span { color: var(--muted); font-size: 0.66rem; font-weight: 700; letter-spacing: 0.04em; text-transform: uppercase; }
968
+ .judge-turn pre { max-height: 22rem; margin: 0; padding: 0.5rem; overflow: auto; border: 1px solid var(--border); border-radius: 6px; background: #0b1017; color: var(--text); font: 0.72rem/1.45 ui-monospace, SFMono-Regular, Menlo, monospace; white-space: pre-wrap; overflow-wrap: anywhere; }
969
+ .judge-response pre { border-inline-start: 3px solid var(--pass); }
952
970
 
953
971
  .chat {
954
972
  display: flex;
@@ -1042,9 +1060,14 @@ function sharedReportCss() {
1042
1060
  .bubble.role-system, .bubble.role-tool { background: var(--system-bubble); font-size: 0.88rem; max-width: min(42rem, 94%); }
1043
1061
 
1044
1062
  .tool-card {
1045
- max-width: min(44rem, 100%);
1046
- border-radius: 10px;
1047
- padding: 0.65rem 0.75rem;
1063
+ display: flex;
1064
+ flex-wrap: wrap;
1065
+ align-items: baseline;
1066
+ gap: 0.3rem 0.55rem;
1067
+ width: fit-content;
1068
+ max-width: min(40rem, 100%);
1069
+ border-radius: 7px;
1070
+ padding: 0.38rem 0.5rem;
1048
1071
  border: 1px solid color-mix(in srgb, var(--tool) 35%, var(--border));
1049
1072
  background: var(--tool-bubble);
1050
1073
  word-spacing: normal;
@@ -1053,25 +1076,25 @@ function sharedReportCss() {
1053
1076
  display: flex;
1054
1077
  align-items: center;
1055
1078
  gap: 0.4rem;
1056
- font-size: 0.82rem;
1079
+ font-size: 0.74rem;
1057
1080
  font-weight: 700;
1058
1081
  color: var(--tool);
1059
1082
  }
1060
- .tool-icon { font-size: 0.85rem; }
1083
+ .tool-icon { display: none; }
1061
1084
  .tool-name { font-weight: 700; }
1062
1085
  .tool-args {
1063
- margin-top: 0.45rem;
1064
- display: grid;
1065
- gap: 0.35rem;
1066
- padding-top: 0.45rem;
1067
- border-top: 1px solid color-mix(in srgb, var(--tool) 20%, var(--border));
1086
+ margin-top: 0;
1087
+ display: grid;
1088
+ gap: 0.22rem;
1089
+ padding-top: 0;
1090
+ border-top: 0;
1068
1091
  }
1069
1092
  .tool-arg {
1070
1093
  display: grid;
1071
- grid-template-columns: 6.25rem minmax(0, 1fr);
1072
- gap: 0.4rem 0.65rem;
1094
+ grid-template-columns: max-content minmax(0, 1fr);
1095
+ gap: 0.3rem 0.45rem;
1073
1096
  align-items: start;
1074
- font-size: 0.8rem;
1097
+ font-size: 0.72rem;
1075
1098
  }
1076
1099
  .tool-arg-key { color: var(--muted); padding-top: 0.2rem; }
1077
1100
  .tool-arg code {
@@ -1079,16 +1102,23 @@ function sharedReportCss() {
1079
1102
  color: var(--text);
1080
1103
  background: #0b1017;
1081
1104
  border-radius: 6px;
1082
- padding: 0.28rem 0.45rem;
1105
+ padding: 0.18rem 0.35rem;
1083
1106
  overflow-wrap: anywhere;
1084
1107
  word-break: break-word;
1085
1108
  white-space: pre-wrap;
1086
- line-height: 1.4;
1109
+ line-height: 1.3;
1087
1110
  }
1088
1111
 
1089
1112
  .shell-commands { margin: 0; padding-left: 0; list-style: none; display: flex; flex-direction: column; gap: 0.3rem; }
1090
1113
  .shell-commands li { font-size: 0.8rem; background: #0b1017; border: 1px solid var(--border); border-radius: 6px; padding: 0.3rem 0.5rem; }
1091
1114
  code { font-family: ui-monospace, SFMono-Regular, Menlo, monospace; font-size: 0.85em; }
1115
+ .trace-details { margin-top: 0.75rem; border: 1px solid var(--border); border-radius: 8px; overflow: hidden; }
1116
+ .trace-details summary { cursor: pointer; display: flex; justify-content: space-between; gap: 0.75rem; padding: 0.55rem 0.7rem; font-size: 0.82rem; font-weight: 650; }
1117
+ .trace-details summary span { color: var(--muted); font-weight: 400; font-variant-numeric: tabular-nums; }
1118
+ .trace-details[open] summary { border-bottom: 1px solid var(--border); background: var(--panel-2); }
1119
+ .trace-details > .chat, .trace-details > h4, .trace-details > .shell-commands, .trace-details > .empty { margin-inline: 0.7rem; }
1120
+ .trace-details > .chat { margin-block: 0.7rem; }
1121
+ .trace-details > .shell-commands { margin-bottom: 0.7rem; }
1092
1122
 
1093
1123
  .compare-layout {
1094
1124
  container-type: inline-size;
@@ -1178,7 +1208,8 @@ function sharedReportCss() {
1178
1208
  padding: 0.85rem 0.9rem 1rem;
1179
1209
  }
1180
1210
  .compare-header { margin-bottom: 0.65rem; }
1181
- .compare-header h3 { margin-bottom: 0.2rem; }
1211
+ .compare-header h3, .compare-header h4 { margin-bottom: 0.2rem; }
1212
+ .story-criteria > .comparison-metrics { margin-top: 0.85rem; padding: 0.8rem 0 0; background: transparent; border: 0; border-top: 1px solid var(--border); border-radius: 0; }
1182
1213
  .compare-winners { margin: 0 0 0.75rem; }
1183
1214
  .compare-winners h3 { margin-bottom: 0.35rem; }
1184
1215
  .compare-winners ul {
@@ -1191,12 +1222,13 @@ function sharedReportCss() {
1191
1222
  .compare-winner-pass { color: var(--pass); }
1192
1223
  .compare-winner-fail { color: var(--fail); }
1193
1224
  .compare-callouts {
1194
- margin: 0 0 0.75rem;
1225
+ margin: 0.5rem 0 0;
1195
1226
  padding-left: 1.1rem;
1196
1227
  display: grid;
1197
1228
  gap: 0.25rem;
1198
1229
  font-size: 0.85rem;
1199
1230
  }
1231
+ .compare-details { margin-top: 0.65rem; color: var(--muted); font-size: 0.78rem; }
1200
1232
  .compare-table-wrap { overflow-x: auto; }
1201
1233
  .compare-table {
1202
1234
  width: 100%;
@@ -1214,8 +1246,13 @@ function sharedReportCss() {
1214
1246
  .delta-up { color: var(--fail); }
1215
1247
  .delta-down { color: var(--pass); }
1216
1248
  .delta-flat { color: var(--muted); }
1249
+ .compare-table td.is-better { color: var(--pass); font-weight: 650; background: color-mix(in srgb, var(--pass) 12%, transparent); }
1250
+ .compare-table td.is-worse { color: var(--fail); font-weight: 650; background: color-mix(in srgb, var(--fail) 12%, transparent); }
1251
+ .compare-table td.is-outcome-pass { color: var(--pass); font-weight: 650; }
1252
+ .compare-table td.is-outcome-fail { color: var(--fail); font-weight: 650; background: color-mix(in srgb, var(--fail) 12%, transparent); }
1253
+ .compare-table td.is-expected-failure { color: var(--muted); font-weight: 650; background: color-mix(in srgb, var(--muted) 9%, transparent); }
1217
1254
  .is-better { color: var(--pass); font-weight: 650; }
1218
- .is-worse { color: var(--muted); }
1255
+ .is-worse { color: var(--fail); font-weight: 650; }
1219
1256
 
1220
1257
  @container (max-width: 44rem) {
1221
1258
  .compare-arms { grid-template-columns: minmax(0, 1fr); }
@@ -1240,30 +1277,14 @@ export function renderHtmlReport(reports, meta = {}) {
1240
1277
  const hostNames = [...new Set(reports.map((report) => report.host).filter(Boolean))];
1241
1278
  const host = meta.host ?? (hostNames.length > 0 ? hostNames.join(", ") : "unknown");
1242
1279
  const runUsage = summarizeReports(reports).usage;
1243
- const suitesHtml = reports.map(renderSuite).join("\n");
1244
1280
  const suiteWord = reports.length === 1 ? "suite" : "suites";
1245
- return `<!DOCTYPE html>
1246
- <html lang="en">
1247
- <head>
1248
- <meta charset="utf-8" />
1249
- <meta name="viewport" content="width=device-width, initial-scale=1" />
1250
- <title>agent-test report</title>
1251
- <style>${sharedReportCss()}</style>
1252
- </head>
1253
- <body>
1254
- <main>
1255
- <header class="report-header">
1256
- <p class="brand">agent-test</p>
1257
- <h1>Run report</h1>
1258
- <p class="lede">${escapeHtml(String(host))} · ${formatInteger(reports.length)} ${suiteWord}${meta.suitesDir ? ` · ${escapeHtml(meta.suitesDir)}` : ""}</p>
1259
- </header>
1260
- <section class="summary" aria-label="Run verdict">
1281
+ const overviewHtml = `<section class="summary" aria-label="Run verdict">
1261
1282
  <div class="stats">
1262
1283
  <span class="stat stat-pass"><strong>${formatInteger(totalPassed)}</strong><span>passed</span></span>
1263
1284
  <span class="stat stat-fail"><strong>${formatInteger(totalFailed)}</strong><span>failed</span></span>
1264
1285
  <span class="stat stat-skip"><strong>${formatInteger(totalSkipped)}</strong><span>skipped</span></span>
1265
1286
  </div>
1266
- <p class="when">Generated ${escapeHtml(generatedAt.toISOString())}</p>
1287
+ <p class="when">${escapeHtml(String(host))} · ${formatInteger(reports.length)} ${suiteWord}${meta.suitesDir ? ` · ${escapeHtml(meta.suitesDir)}` : ""} · Generated ${escapeHtml(generatedAt.toISOString())}</p>
1267
1288
  </section>
1268
1289
  <section class="guide" aria-labelledby="guide-heading">
1269
1290
  <h2 id="guide-heading">How to read this report</h2>
@@ -1275,12 +1296,72 @@ export function renderHtmlReport(reports, meta = {}) {
1275
1296
  <li>Context delivery shows the exact submitted user input and any harness preamble files.</li>
1276
1297
  </ol>
1277
1298
  </section>
1278
- ${renderCostSection(runUsage)}
1279
- ${suitesHtml}
1280
- </main>
1281
- </body>
1282
- </html>
1283
- `;
1299
+ ${renderCostSection(runUsage)}`;
1300
+ const catalog = meta.catalog ?? viewerCatalogFromReports(reports, meta.suitesDir);
1301
+ const runId = "report";
1302
+ const bootstrap = {
1303
+ catalog,
1304
+ runs: [
1305
+ {
1306
+ id: runId,
1307
+ request: {},
1308
+ status: "completed",
1309
+ startedAt: generatedAt.toISOString(),
1310
+ finishedAt: generatedAt.toISOString(),
1311
+ reports: [],
1312
+ },
1313
+ ],
1314
+ selectedRunId: runId,
1315
+ capabilities: { canRun: false },
1316
+ };
1317
+ const results = reports.flatMap((report) => report.results.map((result) => ({
1318
+ suite: report.suite,
1319
+ scenario: result.scenario,
1320
+ host: report.host,
1321
+ html: renderScenarioResult(result, report.host),
1322
+ })));
1323
+ return renderViewerPage(bootstrap, {
1324
+ baseCss: sharedReportCss(),
1325
+ headerLede: `${host} · ${meta.suitesDir ?? catalog.suitesDir}`,
1326
+ overviewHtml,
1327
+ results,
1328
+ });
1329
+ }
1330
+ function viewerCatalogFromReports(reports, suitesDir = "agent-suites") {
1331
+ const suites = new Map();
1332
+ for (const report of reports) {
1333
+ let suite = suites.get(report.suite);
1334
+ if (!suite) {
1335
+ suite = { name: report.suite, hosts: [], scenarios: [] };
1336
+ suites.set(report.suite, suite);
1337
+ }
1338
+ if (!suite.hosts.includes(report.host))
1339
+ suite.hosts.push(report.host);
1340
+ for (const result of report.results) {
1341
+ if (suite.scenarios.some((scenario) => scenario.name === result.scenario))
1342
+ continue;
1343
+ suite.scenarios.push({
1344
+ name: result.scenario,
1345
+ description: result.description,
1346
+ prompt: "",
1347
+ rubric: {},
1348
+ ...(result.skipped ? { skip: true } : {}),
1349
+ ...(result.compare
1350
+ ? {
1351
+ compare: compareResultArms(result.compare).map((arm) => ({
1352
+ id: arm.id,
1353
+ label: arm.label,
1354
+ description: arm.description,
1355
+ prompt: arm.prompt ?? result.prompt ?? "",
1356
+ rubric: {},
1357
+ })),
1358
+ gates: result.compare.gates,
1359
+ }
1360
+ : {}),
1361
+ });
1362
+ }
1363
+ }
1364
+ return { suitesDir, defaultSelectedHosts: [], suites: [...suites.values()] };
1284
1365
  }
1285
1366
  /**
1286
1367
  * Write an HTML report. Writes to `outPath` when given (parent dirs created),