@post-print/agent-test 1.1.1 → 1.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -2
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +34 -6
- package/dist/cli.js.map +1 -1
- package/dist/compare-scenario.d.ts +10 -0
- package/dist/compare-scenario.d.ts.map +1 -1
- package/dist/compare-scenario.js +48 -3
- package/dist/compare-scenario.js.map +1 -1
- package/dist/expect.d.ts +1 -0
- package/dist/expect.d.ts.map +1 -1
- package/dist/expect.js +24 -0
- package/dist/expect.js.map +1 -1
- package/dist/html-report.d.ts +4 -1
- package/dist/html-report.d.ts.map +1 -1
- package/dist/html-report.js +223 -142
- package/dist/html-report.js.map +1 -1
- package/dist/index.d.ts +3 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -2
- package/dist/index.js.map +1 -1
- package/dist/live-isolation.d.ts +2 -0
- package/dist/live-isolation.d.ts.map +1 -1
- package/dist/live-isolation.js +8 -2
- package/dist/live-isolation.js.map +1 -1
- package/dist/report-preview.d.ts +24 -0
- package/dist/report-preview.d.ts.map +1 -1
- package/dist/report-preview.js +46 -2
- package/dist/report-preview.js.map +1 -1
- package/dist/run-suite.d.ts +10 -0
- package/dist/run-suite.d.ts.map +1 -1
- package/dist/run-suite.js +147 -126
- package/dist/run-suite.js.map +1 -1
- package/dist/scenario-finalizer.d.ts +27 -0
- package/dist/scenario-finalizer.d.ts.map +1 -0
- package/dist/scenario-finalizer.js +39 -0
- package/dist/scenario-finalizer.js.map +1 -0
- package/dist/scenario-story.d.ts.map +1 -1
- package/dist/scenario-story.js +14 -8
- package/dist/scenario-story.js.map +1 -1
- package/dist/types.d.ts +9 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/validate-suite.d.ts.map +1 -1
- package/dist/validate-suite.js +4 -0
- package/dist/validate-suite.js.map +1 -1
- package/dist/viewer/catalog.d.ts +14 -3
- package/dist/viewer/catalog.d.ts.map +1 -1
- package/dist/viewer/catalog.js +37 -6
- package/dist/viewer/catalog.js.map +1 -1
- package/dist/viewer/compare-metrics.d.ts.map +1 -1
- package/dist/viewer/compare-metrics.js +8 -9
- package/dist/viewer/compare-metrics.js.map +1 -1
- package/dist/viewer/events.d.ts +22 -1
- package/dist/viewer/events.d.ts.map +1 -1
- package/dist/viewer/events.js +1 -0
- package/dist/viewer/events.js.map +1 -1
- package/dist/viewer/live-runner.d.ts +7 -0
- package/dist/viewer/live-runner.d.ts.map +1 -1
- package/dist/viewer/live-runner.js +36 -23
- package/dist/viewer/live-runner.js.map +1 -1
- package/dist/viewer/page.d.ts +14 -1
- package/dist/viewer/page.d.ts.map +1 -1
- package/dist/viewer/page.js +848 -141
- package/dist/viewer/page.js.map +1 -1
- package/dist/viewer/run-controller.d.ts +13 -6
- package/dist/viewer/run-controller.d.ts.map +1 -1
- package/dist/viewer/run-controller.js +246 -74
- package/dist/viewer/run-controller.js.map +1 -1
- package/dist/viewer/server.d.ts +5 -0
- package/dist/viewer/server.d.ts.map +1 -1
- package/dist/viewer/server.js +72 -17
- package/dist/viewer/server.js.map +1 -1
- package/dist/viewer/session-server.d.ts +2 -0
- package/dist/viewer/session-server.d.ts.map +1 -0
- package/dist/viewer/session-server.js +71 -0
- package/dist/viewer/session-server.js.map +1 -0
- package/package.json +2 -2
package/dist/html-report.js
CHANGED
|
@@ -5,7 +5,7 @@ import { compareArmTokens, compareArmTurns, compareResultArms, describeCompareOu
|
|
|
5
5
|
import { renderContextPanel } from "./context-panel.js";
|
|
6
6
|
import { displayToolPath } from "./scenario-story.js";
|
|
7
7
|
import { summarizeReports } from "./suite-summary.js";
|
|
8
|
-
import {
|
|
8
|
+
import { renderViewerPage } from "./viewer/page.js";
|
|
9
9
|
function escapeHtml(value) {
|
|
10
10
|
return value
|
|
11
11
|
.replaceAll("&", "&")
|
|
@@ -97,6 +97,11 @@ const MATCHER_LABELS = {
|
|
|
97
97
|
mustNotInclude: FORBIDDEN_BEHAVIOR,
|
|
98
98
|
mustRun: MISSING_COMMAND,
|
|
99
99
|
toHaveRunCommand: MISSING_COMMAND,
|
|
100
|
+
mustRunSuccessfully: MISSING_COMMAND,
|
|
101
|
+
toHaveRunCommandSuccessfully: {
|
|
102
|
+
label: "Command did not pass",
|
|
103
|
+
hint: "The required command was missing, failed, or did not report a structured execution status.",
|
|
104
|
+
},
|
|
100
105
|
allowedCommands: FORBIDDEN_COMMAND,
|
|
101
106
|
toHaveAllowedCommands: FORBIDDEN_COMMAND,
|
|
102
107
|
mustCallTool: MISSING_TOOL,
|
|
@@ -283,10 +288,21 @@ function renderJudgeVerdicts(result) {
|
|
|
283
288
|
.map((verdict) => {
|
|
284
289
|
const badge = verdict.pass ? "pass" : "fail";
|
|
285
290
|
const icon = verdict.pass ? "✓" : "✗";
|
|
291
|
+
const evidence = verdict.evidence?.length
|
|
292
|
+
? `<ul class="judge-evidence">${verdict.evidence.map((line) => `<li>${escapeHtml(line)}</li>`).join("")}</ul>`
|
|
293
|
+
: "";
|
|
294
|
+
const exchange = verdict.prompt || verdict.response
|
|
295
|
+
? `<details class="judge-exchange"><summary>Judge conversation</summary><div class="judge-chat">
|
|
296
|
+
${verdict.prompt ? `<div class="judge-turn judge-input"><span>Input</span><pre>${escapeHtml(verdict.prompt)}</pre></div>` : ""}
|
|
297
|
+
${verdict.response ? `<div class="judge-turn judge-response"><span>Response</span><pre>${escapeHtml(verdict.response)}</pre></div>` : ""}
|
|
298
|
+
</div></details>`
|
|
299
|
+
: "";
|
|
286
300
|
return `
|
|
287
301
|
<article class="verdict verdict-${badge}">
|
|
288
302
|
<p class="question"><span class="verdict-icon">${icon}</span>${escapeHtml(verdict.question)}</p>
|
|
289
303
|
<p class="rationale">${escapeHtml(verdict.rationale)}</p>
|
|
304
|
+
${evidence}
|
|
305
|
+
${exchange}
|
|
290
306
|
</article>`;
|
|
291
307
|
})
|
|
292
308
|
.join("\n");
|
|
@@ -467,13 +483,45 @@ function renderCompareMetricRow(label, aText, bText, delta, deltaText) {
|
|
|
467
483
|
<td class="${deltaClass(delta)}">${escapeHtml(deltaText)}</td>
|
|
468
484
|
</tr>`;
|
|
469
485
|
}
|
|
470
|
-
function renderNamedCompareMetricRow(label, cells) {
|
|
486
|
+
function renderNamedCompareMetricRow(label, cells, values) {
|
|
487
|
+
const numeric = (values ?? []).filter((value) => value !== undefined);
|
|
488
|
+
const low = numeric.length > 1 ? Math.min(...numeric) : undefined;
|
|
489
|
+
const high = numeric.length > 1 ? Math.max(...numeric) : undefined;
|
|
471
490
|
return `<tr>
|
|
472
491
|
<th scope="row">${escapeHtml(label)}</th>
|
|
473
|
-
${cells
|
|
492
|
+
${cells
|
|
493
|
+
.map((cell, index) => {
|
|
494
|
+
const value = values?.[index];
|
|
495
|
+
const state = value !== undefined && low !== high
|
|
496
|
+
? value === low
|
|
497
|
+
? "is-better"
|
|
498
|
+
: value === high
|
|
499
|
+
? "is-worse"
|
|
500
|
+
: ""
|
|
501
|
+
: "";
|
|
502
|
+
return `<td class="${state}">${escapeHtml(cell)}</td>`;
|
|
503
|
+
})
|
|
504
|
+
.join("")}
|
|
474
505
|
</tr>`;
|
|
475
506
|
}
|
|
476
|
-
function
|
|
507
|
+
function outcomeCell(arm, gates) {
|
|
508
|
+
if (arm.passed === undefined)
|
|
509
|
+
return "<td>n/a</td>";
|
|
510
|
+
if (arm.passed)
|
|
511
|
+
return '<td class="is-outcome-pass">pass</td>';
|
|
512
|
+
const expected = gates?.some((gate) => !("winner" in gate) &&
|
|
513
|
+
gate.arm === arm.id &&
|
|
514
|
+
gate.metric === "outcome" &&
|
|
515
|
+
gate.operator === "equal" &&
|
|
516
|
+
gate.value === "fail");
|
|
517
|
+
return expected
|
|
518
|
+
? '<td class="is-expected-failure">expected fail</td>'
|
|
519
|
+
: '<td class="is-outcome-fail">fail</td>';
|
|
520
|
+
}
|
|
521
|
+
function renderOutcomeRow(arms, gates, includeDelta = false) {
|
|
522
|
+
return `<tr><th scope="row">Outcome</th>${arms.map((arm) => outcomeCell(arm, gates)).join("")}${includeDelta ? '<td class="delta-flat">n/a</td>' : ""}</tr>`;
|
|
523
|
+
}
|
|
524
|
+
function renderTwoArmCompareMetrics(arms, gates) {
|
|
477
525
|
const [left, right] = arms;
|
|
478
526
|
const aTurns = compareArmTurns(left);
|
|
479
527
|
const bTurns = compareArmTurns(right);
|
|
@@ -482,9 +530,6 @@ function renderTwoArmCompareMetrics(arms) {
|
|
|
482
530
|
const bTools = armToolCount(right);
|
|
483
531
|
const toolDelta = aTools !== undefined && bTools !== undefined ? bTools - aTools : undefined;
|
|
484
532
|
const totalDelta = tokenDelta(left, right, "total");
|
|
485
|
-
const durationDelta = left.durationMs !== undefined && right.durationMs !== undefined
|
|
486
|
-
? right.durationMs - left.durationMs
|
|
487
|
-
: undefined;
|
|
488
533
|
return `
|
|
489
534
|
<div class="compare-table-wrap">
|
|
490
535
|
<table class="compare-table">
|
|
@@ -497,18 +542,17 @@ function renderTwoArmCompareMetrics(arms) {
|
|
|
497
542
|
</tr>
|
|
498
543
|
</thead>
|
|
499
544
|
<tbody>
|
|
500
|
-
${
|
|
545
|
+
${renderOutcomeRow(arms, gates, true)}
|
|
501
546
|
${renderCompareMetricRow("Turns", formatArmTurns(left), formatArmTurns(right), turnDelta, formatSigned(turnDelta))}
|
|
502
547
|
${renderCompareMetricRow("Tokens", formatArmTokens(left, "total"), formatArmTokens(right, "total"), totalDelta, formatSigned(totalDelta))}
|
|
503
548
|
${renderCompareMetricRow("In", formatArmTokens(left, "input"), formatArmTokens(right, "input"), tokenDelta(left, right, "input"), formatSigned(tokenDelta(left, right, "input")))}
|
|
504
549
|
${renderCompareMetricRow("Out", formatArmTokens(left, "output"), formatArmTokens(right, "output"), tokenDelta(left, right, "output"), formatSigned(tokenDelta(left, right, "output")))}
|
|
505
550
|
${renderCompareMetricRow("Tools", aTools === undefined ? "n/a" : formatInteger(aTools), bTools === undefined ? "n/a" : formatInteger(bTools), toolDelta, formatSigned(toolDelta))}
|
|
506
|
-
${renderCompareMetricRow("Duration", left.durationMs === undefined ? "n/a" : formatDuration(left.durationMs), right.durationMs === undefined ? "n/a" : formatDuration(right.durationMs), durationDelta, formatSigned(durationDelta))}
|
|
507
551
|
</tbody>
|
|
508
552
|
</table>
|
|
509
553
|
</div>`;
|
|
510
554
|
}
|
|
511
|
-
function renderNamedCompareMetrics(arms) {
|
|
555
|
+
function renderNamedCompareMetrics(arms, gates) {
|
|
512
556
|
return `
|
|
513
557
|
<div class="compare-table-wrap">
|
|
514
558
|
<table class="compare-table">
|
|
@@ -519,16 +563,15 @@ function renderNamedCompareMetrics(arms) {
|
|
|
519
563
|
</tr>
|
|
520
564
|
</thead>
|
|
521
565
|
<tbody>
|
|
522
|
-
${
|
|
523
|
-
${renderNamedCompareMetricRow("Turns", arms.map((arm) => formatArmTurns(arm)))}
|
|
524
|
-
${renderNamedCompareMetricRow("Tokens", arms.map((arm) => formatArmTokens(arm, "total")))}
|
|
566
|
+
${renderOutcomeRow(arms, gates)}
|
|
567
|
+
${renderNamedCompareMetricRow("Turns", arms.map((arm) => formatArmTurns(arm)), arms.map(compareArmTurns))}
|
|
568
|
+
${renderNamedCompareMetricRow("Tokens", arms.map((arm) => formatArmTokens(arm, "total")), arms.map(compareArmTokens))}
|
|
525
569
|
${renderNamedCompareMetricRow("In", arms.map((arm) => formatArmTokens(arm, "input")))}
|
|
526
570
|
${renderNamedCompareMetricRow("Out", arms.map((arm) => formatArmTokens(arm, "output")))}
|
|
527
571
|
${renderNamedCompareMetricRow("Tools", arms.map((arm) => {
|
|
528
572
|
const tools = armToolCount(arm);
|
|
529
573
|
return tools === undefined ? "n/a" : formatInteger(tools);
|
|
530
|
-
}))}
|
|
531
|
-
${renderNamedCompareMetricRow("Duration", arms.map((arm) => (arm.durationMs === undefined ? "n/a" : formatDuration(arm.durationMs))))}
|
|
574
|
+
}), arms.map(armToolCount))}
|
|
532
575
|
</tbody>
|
|
533
576
|
</table>
|
|
534
577
|
</div>`;
|
|
@@ -542,31 +585,22 @@ function renderCompareMetrics(result) {
|
|
|
542
585
|
if (arms.length === 0) {
|
|
543
586
|
return "";
|
|
544
587
|
}
|
|
588
|
+
const twoArm = arms.length === 2 && arms[0] && arms[1];
|
|
545
589
|
const callouts = describeCompareOutcome(compare);
|
|
546
|
-
const calloutList = callouts.length
|
|
547
|
-
? `<ul class="compare-callouts">${callouts.map((line) => `<li>${escapeHtml(line)}</li>`).join("")}</ul>`
|
|
590
|
+
const calloutList = callouts.length
|
|
591
|
+
? `<details class="compare-details"><summary>Arm details</summary><ul class="compare-callouts">${callouts.map((line) => `<li>${escapeHtml(line)}</li>`).join("")}</ul></details>`
|
|
548
592
|
: "";
|
|
549
|
-
const twoArm = arms.length === 2 && arms[0] && arms[1];
|
|
550
593
|
const lede = twoArm
|
|
551
|
-
? `${arms[0].label} vs ${arms[1].label}.
|
|
552
|
-
: `${arms.map((arm) => arm.label).join(", ")}.
|
|
553
|
-
const winners = renderViewerCompareBoard(summarizeViewerCompare(arms.map((arm) => ({
|
|
554
|
-
id: arm.id,
|
|
555
|
-
label: arm.label,
|
|
556
|
-
turns: compareArmTurns(arm),
|
|
557
|
-
tokens: compareArmTokens(arm),
|
|
558
|
-
tools: arm.trace ? arm.trace.toolCalls.length : undefined,
|
|
559
|
-
durationMs: arm.durationMs,
|
|
560
|
-
})), compare.gateResults));
|
|
594
|
+
? `${arms[0].label} vs ${arms[1].label}. Lower is better: green is lowest and red is highest.`
|
|
595
|
+
: `${arms.map((arm) => arm.label).join(", ")}. Lower is better: green is lowest and red is highest. Ties are neutral.`;
|
|
561
596
|
return `
|
|
562
|
-
<section class="compare">
|
|
597
|
+
<section class="compare comparison-metrics">
|
|
563
598
|
<header class="compare-header">
|
|
564
|
-
<
|
|
599
|
+
<h4>Comparison metrics</h4>
|
|
565
600
|
<p class="muted">${escapeHtml(lede)}</p>
|
|
566
601
|
</header>
|
|
567
|
-
|
|
602
|
+
${twoArm ? renderTwoArmCompareMetrics([arms[0], arms[1]], compare.gates) : renderNamedCompareMetrics(arms, compare.gates)}
|
|
568
603
|
${calloutList}
|
|
569
|
-
${twoArm ? renderTwoArmCompareMetrics([arms[0], arms[1]]) : renderNamedCompareMetrics(arms)}
|
|
570
604
|
</section>`;
|
|
571
605
|
}
|
|
572
606
|
function renderArmMetrics(arm) {
|
|
@@ -617,28 +651,26 @@ function renderArmColumn(arm, index) {
|
|
|
617
651
|
${renderArmFailures(arm)}
|
|
618
652
|
</header>
|
|
619
653
|
${arm.contextMode || arm.contextFiles || arm.hostInput ? renderContextPanel(arm.contextFiles, arm.contextMode, arm.hostInput) : ""}
|
|
620
|
-
${
|
|
654
|
+
${renderTraceDetails(arm.trace, arm.prompt)}
|
|
621
655
|
</article>`;
|
|
622
656
|
}
|
|
657
|
+
function renderTraceDetails(trace, prompt) {
|
|
658
|
+
const tools = trace?.toolCalls.length ?? 0;
|
|
659
|
+
const shell = trace?.shellCommands.length ?? 0;
|
|
660
|
+
const messages = trace?.messages.length ?? 0;
|
|
661
|
+
const label = `${messages} messages · ${tools} tools${shell ? ` · ${shell} shell command${shell === 1 ? "" : "s"}` : ""}`;
|
|
662
|
+
return `<details class="trace-details" open><summary>Conversation & evidence <span>${escapeHtml(label)}</span></summary>${renderChat(trace, prompt)}</details>`;
|
|
663
|
+
}
|
|
623
664
|
function renderCompareConversations(result, host) {
|
|
624
665
|
const compare = result.compare;
|
|
625
666
|
if (!compare) {
|
|
626
667
|
return `<section class="conversation">
|
|
627
668
|
<h3>Conversation</h3>
|
|
628
669
|
${result.contextMode || result.contextFiles || result.hostInput ? renderContextPanel(result.contextFiles, result.contextMode, result.hostInput) : ""}
|
|
629
|
-
|
|
670
|
+
${renderTraceDetails(result.trace, result.prompt)}
|
|
630
671
|
</section>`;
|
|
631
672
|
}
|
|
632
673
|
const arms = compareResultArms(compare);
|
|
633
|
-
if (arms.length <= 2) {
|
|
634
|
-
return `
|
|
635
|
-
<div class="compare-layout">
|
|
636
|
-
<div class="compare-arms" data-arm-count="${arms.length}">
|
|
637
|
-
${arms.map((arm, index) => renderArmColumn(arm, index)).join("")}
|
|
638
|
-
</div>
|
|
639
|
-
${renderCompareMetrics(result)}
|
|
640
|
-
</div>`;
|
|
641
|
-
}
|
|
642
674
|
const group = compareTabGroupId(result, host);
|
|
643
675
|
const tablist = arms
|
|
644
676
|
.map((arm, index) => {
|
|
@@ -657,69 +689,58 @@ function renderCompareConversations(result, host) {
|
|
|
657
689
|
<div class="compare-arms" data-arm-count="${arms.length}">
|
|
658
690
|
${arms.map((arm, index) => renderArmColumn(arm, index)).join("")}
|
|
659
691
|
</div>
|
|
660
|
-
${renderCompareMetrics(result)}
|
|
661
692
|
</div>`;
|
|
662
693
|
}
|
|
663
|
-
function renderStoryList(title, lines) {
|
|
664
|
-
if (!lines || lines.length === 0) {
|
|
665
|
-
return "";
|
|
666
|
-
}
|
|
667
|
-
const items = lines.map((line) => `<li>${escapeHtml(line)}</li>`).join("");
|
|
668
|
-
return `<section class="story-block"><h3>${escapeHtml(title)}</h3><ul class="story-list">${items}</ul></section>`;
|
|
669
|
-
}
|
|
670
694
|
function renderStoryCheck(check) {
|
|
671
695
|
const icon = check.status === "pass" ? "✓" : check.status === "fail" ? "✗" : "·";
|
|
672
696
|
return `<li class="story-check story-check-${check.status}"><span class="story-check-icon">${icon}</span>${escapeHtml(check.text)}</li>`;
|
|
673
697
|
}
|
|
674
698
|
function renderStorySection(section) {
|
|
675
699
|
const titled = Boolean(section.title);
|
|
676
|
-
const title = section.title ? `<
|
|
700
|
+
const title = section.title ? `<h4>${escapeHtml(section.title)}</h4>` : "";
|
|
677
701
|
const description = section.description
|
|
678
702
|
? `<p class="story-section-description">${escapeHtml(section.description)}</p>`
|
|
679
703
|
: "";
|
|
680
704
|
const checks = section.checks.length > 0
|
|
681
705
|
? `<ul class="story-checks">${section.checks.map(renderStoryCheck).join("")}</ul>`
|
|
682
706
|
: "";
|
|
683
|
-
const notes = section.notes && section.notes.length > 0
|
|
684
|
-
? `<ul class="story-notes">${section.notes.map((note) => `<li>${escapeHtml(note)}</li>`).join("")}</ul>`
|
|
685
|
-
: "";
|
|
686
707
|
const titledClass = titled ? " story-section-titled" : "";
|
|
687
|
-
return `<
|
|
708
|
+
return `<div class="story-section${titledClass}">${title}${description}${checks}</div>`;
|
|
688
709
|
}
|
|
689
|
-
function renderStory(result) {
|
|
710
|
+
function renderStory(result, comparison = "") {
|
|
690
711
|
const story = result.story;
|
|
691
|
-
if (!story) {
|
|
712
|
+
if (!story && !comparison) {
|
|
692
713
|
return "";
|
|
693
714
|
}
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
?
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
715
|
+
const sections = story?.sections?.filter((section) => section.checks.length > 0) ?? [];
|
|
716
|
+
const storyCriteria = sections.length
|
|
717
|
+
? sections.map(renderStorySection).join("")
|
|
718
|
+
: story?.criteria.length
|
|
719
|
+
? `<ul class="story-checks">${story.criteria
|
|
720
|
+
.map((text) => renderStoryCheck({ text, status: result.passed ? "pass" : "fail" }))
|
|
721
|
+
.join("")}</ul>`
|
|
722
|
+
: "";
|
|
723
|
+
const fallbackCompareCriteria = result.compare?.gateResults?.length
|
|
724
|
+
? `<div class="story-section story-section-titled"><h4>Compare</h4><ul class="story-checks">${result.compare.gateResults
|
|
725
|
+
.map((gate) => renderStoryCheck({
|
|
726
|
+
text: gate.message,
|
|
727
|
+
status: gate.passed ? "pass" : "fail",
|
|
728
|
+
}))
|
|
729
|
+
.join("")}</ul></div>`
|
|
730
|
+
: "";
|
|
731
|
+
const criteria = storyCriteria || fallbackCompareCriteria;
|
|
732
|
+
if (!criteria && !comparison)
|
|
733
|
+
return "";
|
|
734
|
+
return `<div class="story"><section class="story-criteria"><h3>Pass criteria</h3>${criteria}${comparison}</section></div>`;
|
|
714
735
|
}
|
|
715
|
-
function
|
|
716
|
-
const open = result.skipped ? "" :
|
|
717
|
-
const failures =
|
|
736
|
+
export function renderScenarioResult(result, host) {
|
|
737
|
+
const open = result.skipped ? "" : " open";
|
|
738
|
+
const failures = renderFailures(result);
|
|
718
739
|
const judgeVerdicts = renderJudgeVerdicts(result);
|
|
719
740
|
const tokens = formatTokensBadge(result);
|
|
720
741
|
const usageDetail = result.compare ? "" : renderUsageDetail(usageOf(result));
|
|
721
742
|
const traceMeta = result.compare ? "" : renderTraceMeta(result);
|
|
722
|
-
const story = renderStory(result);
|
|
743
|
+
const story = renderStory(result, renderCompareMetrics(result));
|
|
723
744
|
const diagnostics = story || failures || judgeVerdicts
|
|
724
745
|
? `<div class="diagnostics">
|
|
725
746
|
${story}
|
|
@@ -752,20 +773,6 @@ function renderScenario(result, host) {
|
|
|
752
773
|
</div>
|
|
753
774
|
</details>`;
|
|
754
775
|
}
|
|
755
|
-
function renderSuite(report) {
|
|
756
|
-
const scenarios = report.results.map((result) => renderScenario(result, report.host)).join("\n");
|
|
757
|
-
return `
|
|
758
|
-
<section class="suite">
|
|
759
|
-
<header class="suite-header">
|
|
760
|
-
<div class="suite-title">
|
|
761
|
-
<h2>${escapeHtml(report.suite)}</h2>
|
|
762
|
-
<span class="host">${escapeHtml(report.host)}</span>
|
|
763
|
-
</div>
|
|
764
|
-
<p class="suite-counts"><span>${formatInteger(report.passed)} passed</span><span>${formatInteger(report.failed)} failed</span><span>${formatInteger(report.skipped)} skipped</span></p>
|
|
765
|
-
</header>
|
|
766
|
-
${scenarios}
|
|
767
|
-
</section>`;
|
|
768
|
-
}
|
|
769
776
|
export function reportCss() {
|
|
770
777
|
return sharedReportCss();
|
|
771
778
|
}
|
|
@@ -921,10 +928,12 @@ function sharedReportCss() {
|
|
|
921
928
|
.diagnostics section { background: var(--panel-2); border: 1px solid var(--border); border-radius: 8px; padding: 0.65rem 0.7rem; min-width: 0; }
|
|
922
929
|
.conversation { min-width: 0; }
|
|
923
930
|
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
931
|
+
.story { margin-bottom: 0.85rem; }
|
|
932
|
+
.story-criteria { display: grid; gap: 0.55rem; }
|
|
933
|
+
.story-criteria > h3 { margin: 0; font-size: 0.78rem; text-transform: uppercase; letter-spacing: 0.04em; color: var(--muted); }
|
|
934
|
+
.story-section { display: grid; gap: 0.3rem; }
|
|
935
|
+
.story-section + .story-section { padding-block-start: 0.55rem; border-block-start: 1px solid var(--border); }
|
|
936
|
+
.story-section h4 { margin: 0; font-size: 0.84rem; color: var(--text); }
|
|
928
937
|
.story-section-description { margin: 0 0 0.4rem; color: var(--muted); font-size: 0.82rem; }
|
|
929
938
|
.story-list, .story-checks, .story-notes { margin: 0; padding-left: 1.1rem; display: flex; flex-direction: column; gap: 0.25rem; color: var(--text); }
|
|
930
939
|
.story-checks { list-style: none; padding-left: 0; }
|
|
@@ -949,6 +958,15 @@ function sharedReportCss() {
|
|
|
949
958
|
.verdict-fail .verdict-icon { color: var(--fail); }
|
|
950
959
|
.question { margin: 0; font-size: 0.85rem; font-weight: 600; }
|
|
951
960
|
.rationale { margin: 0.3rem 0 0; font-size: 0.82rem; color: var(--muted); }
|
|
961
|
+
.judge-evidence { margin: 0.35rem 0 0; padding-inline-start: 1.1rem; color: var(--muted); font-size: 0.78rem; }
|
|
962
|
+
.judge-exchange { margin-block-start: 0.5rem; border: 1px solid var(--border); border-radius: 7px; overflow: hidden; }
|
|
963
|
+
.judge-exchange > summary { cursor: pointer; padding: 0.4rem 0.55rem; color: var(--muted); font-size: 0.75rem; font-weight: 650; }
|
|
964
|
+
.judge-exchange[open] > summary { border-block-end: 1px solid var(--border); background: var(--panel); }
|
|
965
|
+
.judge-chat { display: grid; gap: 0.55rem; padding: 0.55rem; }
|
|
966
|
+
.judge-turn { display: grid; gap: 0.25rem; min-width: 0; }
|
|
967
|
+
.judge-turn > span { color: var(--muted); font-size: 0.66rem; font-weight: 700; letter-spacing: 0.04em; text-transform: uppercase; }
|
|
968
|
+
.judge-turn pre { max-height: 22rem; margin: 0; padding: 0.5rem; overflow: auto; border: 1px solid var(--border); border-radius: 6px; background: #0b1017; color: var(--text); font: 0.72rem/1.45 ui-monospace, SFMono-Regular, Menlo, monospace; white-space: pre-wrap; overflow-wrap: anywhere; }
|
|
969
|
+
.judge-response pre { border-inline-start: 3px solid var(--pass); }
|
|
952
970
|
|
|
953
971
|
.chat {
|
|
954
972
|
display: flex;
|
|
@@ -1042,9 +1060,14 @@ function sharedReportCss() {
|
|
|
1042
1060
|
.bubble.role-system, .bubble.role-tool { background: var(--system-bubble); font-size: 0.88rem; max-width: min(42rem, 94%); }
|
|
1043
1061
|
|
|
1044
1062
|
.tool-card {
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1063
|
+
display: flex;
|
|
1064
|
+
flex-wrap: wrap;
|
|
1065
|
+
align-items: baseline;
|
|
1066
|
+
gap: 0.3rem 0.55rem;
|
|
1067
|
+
width: fit-content;
|
|
1068
|
+
max-width: min(40rem, 100%);
|
|
1069
|
+
border-radius: 7px;
|
|
1070
|
+
padding: 0.38rem 0.5rem;
|
|
1048
1071
|
border: 1px solid color-mix(in srgb, var(--tool) 35%, var(--border));
|
|
1049
1072
|
background: var(--tool-bubble);
|
|
1050
1073
|
word-spacing: normal;
|
|
@@ -1053,25 +1076,25 @@ function sharedReportCss() {
|
|
|
1053
1076
|
display: flex;
|
|
1054
1077
|
align-items: center;
|
|
1055
1078
|
gap: 0.4rem;
|
|
1056
|
-
|
|
1079
|
+
font-size: 0.74rem;
|
|
1057
1080
|
font-weight: 700;
|
|
1058
1081
|
color: var(--tool);
|
|
1059
1082
|
}
|
|
1060
|
-
|
|
1083
|
+
.tool-icon { display: none; }
|
|
1061
1084
|
.tool-name { font-weight: 700; }
|
|
1062
1085
|
.tool-args {
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1086
|
+
margin-top: 0;
|
|
1087
|
+
display: grid;
|
|
1088
|
+
gap: 0.22rem;
|
|
1089
|
+
padding-top: 0;
|
|
1090
|
+
border-top: 0;
|
|
1068
1091
|
}
|
|
1069
1092
|
.tool-arg {
|
|
1070
1093
|
display: grid;
|
|
1071
|
-
|
|
1072
|
-
|
|
1094
|
+
grid-template-columns: max-content minmax(0, 1fr);
|
|
1095
|
+
gap: 0.3rem 0.45rem;
|
|
1073
1096
|
align-items: start;
|
|
1074
|
-
|
|
1097
|
+
font-size: 0.72rem;
|
|
1075
1098
|
}
|
|
1076
1099
|
.tool-arg-key { color: var(--muted); padding-top: 0.2rem; }
|
|
1077
1100
|
.tool-arg code {
|
|
@@ -1079,16 +1102,23 @@ function sharedReportCss() {
|
|
|
1079
1102
|
color: var(--text);
|
|
1080
1103
|
background: #0b1017;
|
|
1081
1104
|
border-radius: 6px;
|
|
1082
|
-
|
|
1105
|
+
padding: 0.18rem 0.35rem;
|
|
1083
1106
|
overflow-wrap: anywhere;
|
|
1084
1107
|
word-break: break-word;
|
|
1085
1108
|
white-space: pre-wrap;
|
|
1086
|
-
|
|
1109
|
+
line-height: 1.3;
|
|
1087
1110
|
}
|
|
1088
1111
|
|
|
1089
1112
|
.shell-commands { margin: 0; padding-left: 0; list-style: none; display: flex; flex-direction: column; gap: 0.3rem; }
|
|
1090
1113
|
.shell-commands li { font-size: 0.8rem; background: #0b1017; border: 1px solid var(--border); border-radius: 6px; padding: 0.3rem 0.5rem; }
|
|
1091
1114
|
code { font-family: ui-monospace, SFMono-Regular, Menlo, monospace; font-size: 0.85em; }
|
|
1115
|
+
.trace-details { margin-top: 0.75rem; border: 1px solid var(--border); border-radius: 8px; overflow: hidden; }
|
|
1116
|
+
.trace-details summary { cursor: pointer; display: flex; justify-content: space-between; gap: 0.75rem; padding: 0.55rem 0.7rem; font-size: 0.82rem; font-weight: 650; }
|
|
1117
|
+
.trace-details summary span { color: var(--muted); font-weight: 400; font-variant-numeric: tabular-nums; }
|
|
1118
|
+
.trace-details[open] summary { border-bottom: 1px solid var(--border); background: var(--panel-2); }
|
|
1119
|
+
.trace-details > .chat, .trace-details > h4, .trace-details > .shell-commands, .trace-details > .empty { margin-inline: 0.7rem; }
|
|
1120
|
+
.trace-details > .chat { margin-block: 0.7rem; }
|
|
1121
|
+
.trace-details > .shell-commands { margin-bottom: 0.7rem; }
|
|
1092
1122
|
|
|
1093
1123
|
.compare-layout {
|
|
1094
1124
|
container-type: inline-size;
|
|
@@ -1178,7 +1208,8 @@ function sharedReportCss() {
|
|
|
1178
1208
|
padding: 0.85rem 0.9rem 1rem;
|
|
1179
1209
|
}
|
|
1180
1210
|
.compare-header { margin-bottom: 0.65rem; }
|
|
1181
|
-
.compare-header h3 { margin-bottom: 0.2rem; }
|
|
1211
|
+
.compare-header h3, .compare-header h4 { margin-bottom: 0.2rem; }
|
|
1212
|
+
.story-criteria > .comparison-metrics { margin-top: 0.85rem; padding: 0.8rem 0 0; background: transparent; border: 0; border-top: 1px solid var(--border); border-radius: 0; }
|
|
1182
1213
|
.compare-winners { margin: 0 0 0.75rem; }
|
|
1183
1214
|
.compare-winners h3 { margin-bottom: 0.35rem; }
|
|
1184
1215
|
.compare-winners ul {
|
|
@@ -1191,12 +1222,13 @@ function sharedReportCss() {
|
|
|
1191
1222
|
.compare-winner-pass { color: var(--pass); }
|
|
1192
1223
|
.compare-winner-fail { color: var(--fail); }
|
|
1193
1224
|
.compare-callouts {
|
|
1194
|
-
|
|
1225
|
+
margin: 0.5rem 0 0;
|
|
1195
1226
|
padding-left: 1.1rem;
|
|
1196
1227
|
display: grid;
|
|
1197
1228
|
gap: 0.25rem;
|
|
1198
1229
|
font-size: 0.85rem;
|
|
1199
1230
|
}
|
|
1231
|
+
.compare-details { margin-top: 0.65rem; color: var(--muted); font-size: 0.78rem; }
|
|
1200
1232
|
.compare-table-wrap { overflow-x: auto; }
|
|
1201
1233
|
.compare-table {
|
|
1202
1234
|
width: 100%;
|
|
@@ -1214,8 +1246,13 @@ function sharedReportCss() {
|
|
|
1214
1246
|
.delta-up { color: var(--fail); }
|
|
1215
1247
|
.delta-down { color: var(--pass); }
|
|
1216
1248
|
.delta-flat { color: var(--muted); }
|
|
1249
|
+
.compare-table td.is-better { color: var(--pass); font-weight: 650; background: color-mix(in srgb, var(--pass) 12%, transparent); }
|
|
1250
|
+
.compare-table td.is-worse { color: var(--fail); font-weight: 650; background: color-mix(in srgb, var(--fail) 12%, transparent); }
|
|
1251
|
+
.compare-table td.is-outcome-pass { color: var(--pass); font-weight: 650; }
|
|
1252
|
+
.compare-table td.is-outcome-fail { color: var(--fail); font-weight: 650; background: color-mix(in srgb, var(--fail) 12%, transparent); }
|
|
1253
|
+
.compare-table td.is-expected-failure { color: var(--muted); font-weight: 650; background: color-mix(in srgb, var(--muted) 9%, transparent); }
|
|
1217
1254
|
.is-better { color: var(--pass); font-weight: 650; }
|
|
1218
|
-
.is-worse { color: var(--
|
|
1255
|
+
.is-worse { color: var(--fail); font-weight: 650; }
|
|
1219
1256
|
|
|
1220
1257
|
@container (max-width: 44rem) {
|
|
1221
1258
|
.compare-arms { grid-template-columns: minmax(0, 1fr); }
|
|
@@ -1240,30 +1277,14 @@ export function renderHtmlReport(reports, meta = {}) {
|
|
|
1240
1277
|
const hostNames = [...new Set(reports.map((report) => report.host).filter(Boolean))];
|
|
1241
1278
|
const host = meta.host ?? (hostNames.length > 0 ? hostNames.join(", ") : "unknown");
|
|
1242
1279
|
const runUsage = summarizeReports(reports).usage;
|
|
1243
|
-
const suitesHtml = reports.map(renderSuite).join("\n");
|
|
1244
1280
|
const suiteWord = reports.length === 1 ? "suite" : "suites";
|
|
1245
|
-
|
|
1246
|
-
<html lang="en">
|
|
1247
|
-
<head>
|
|
1248
|
-
<meta charset="utf-8" />
|
|
1249
|
-
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
|
1250
|
-
<title>agent-test report</title>
|
|
1251
|
-
<style>${sharedReportCss()}</style>
|
|
1252
|
-
</head>
|
|
1253
|
-
<body>
|
|
1254
|
-
<main>
|
|
1255
|
-
<header class="report-header">
|
|
1256
|
-
<p class="brand">agent-test</p>
|
|
1257
|
-
<h1>Run report</h1>
|
|
1258
|
-
<p class="lede">${escapeHtml(String(host))} · ${formatInteger(reports.length)} ${suiteWord}${meta.suitesDir ? ` · ${escapeHtml(meta.suitesDir)}` : ""}</p>
|
|
1259
|
-
</header>
|
|
1260
|
-
<section class="summary" aria-label="Run verdict">
|
|
1281
|
+
const overviewHtml = `<section class="summary" aria-label="Run verdict">
|
|
1261
1282
|
<div class="stats">
|
|
1262
1283
|
<span class="stat stat-pass"><strong>${formatInteger(totalPassed)}</strong><span>passed</span></span>
|
|
1263
1284
|
<span class="stat stat-fail"><strong>${formatInteger(totalFailed)}</strong><span>failed</span></span>
|
|
1264
1285
|
<span class="stat stat-skip"><strong>${formatInteger(totalSkipped)}</strong><span>skipped</span></span>
|
|
1265
1286
|
</div>
|
|
1266
|
-
<p class="when"
|
|
1287
|
+
<p class="when">${escapeHtml(String(host))} · ${formatInteger(reports.length)} ${suiteWord}${meta.suitesDir ? ` · ${escapeHtml(meta.suitesDir)}` : ""} · Generated ${escapeHtml(generatedAt.toISOString())}</p>
|
|
1267
1288
|
</section>
|
|
1268
1289
|
<section class="guide" aria-labelledby="guide-heading">
|
|
1269
1290
|
<h2 id="guide-heading">How to read this report</h2>
|
|
@@ -1275,12 +1296,72 @@ export function renderHtmlReport(reports, meta = {}) {
|
|
|
1275
1296
|
<li>Context delivery shows the exact submitted user input and any harness preamble files.</li>
|
|
1276
1297
|
</ol>
|
|
1277
1298
|
</section>
|
|
1278
|
-
${renderCostSection(runUsage)}
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1299
|
+
${renderCostSection(runUsage)}`;
|
|
1300
|
+
const catalog = meta.catalog ?? viewerCatalogFromReports(reports, meta.suitesDir);
|
|
1301
|
+
const runId = "report";
|
|
1302
|
+
const bootstrap = {
|
|
1303
|
+
catalog,
|
|
1304
|
+
runs: [
|
|
1305
|
+
{
|
|
1306
|
+
id: runId,
|
|
1307
|
+
request: {},
|
|
1308
|
+
status: "completed",
|
|
1309
|
+
startedAt: generatedAt.toISOString(),
|
|
1310
|
+
finishedAt: generatedAt.toISOString(),
|
|
1311
|
+
reports: [],
|
|
1312
|
+
},
|
|
1313
|
+
],
|
|
1314
|
+
selectedRunId: runId,
|
|
1315
|
+
capabilities: { canRun: false },
|
|
1316
|
+
};
|
|
1317
|
+
const results = reports.flatMap((report) => report.results.map((result) => ({
|
|
1318
|
+
suite: report.suite,
|
|
1319
|
+
scenario: result.scenario,
|
|
1320
|
+
host: report.host,
|
|
1321
|
+
html: renderScenarioResult(result, report.host),
|
|
1322
|
+
})));
|
|
1323
|
+
return renderViewerPage(bootstrap, {
|
|
1324
|
+
baseCss: sharedReportCss(),
|
|
1325
|
+
headerLede: `${host} · ${meta.suitesDir ?? catalog.suitesDir}`,
|
|
1326
|
+
overviewHtml,
|
|
1327
|
+
results,
|
|
1328
|
+
});
|
|
1329
|
+
}
|
|
1330
|
+
function viewerCatalogFromReports(reports, suitesDir = "agent-suites") {
|
|
1331
|
+
const suites = new Map();
|
|
1332
|
+
for (const report of reports) {
|
|
1333
|
+
let suite = suites.get(report.suite);
|
|
1334
|
+
if (!suite) {
|
|
1335
|
+
suite = { name: report.suite, hosts: [], scenarios: [] };
|
|
1336
|
+
suites.set(report.suite, suite);
|
|
1337
|
+
}
|
|
1338
|
+
if (!suite.hosts.includes(report.host))
|
|
1339
|
+
suite.hosts.push(report.host);
|
|
1340
|
+
for (const result of report.results) {
|
|
1341
|
+
if (suite.scenarios.some((scenario) => scenario.name === result.scenario))
|
|
1342
|
+
continue;
|
|
1343
|
+
suite.scenarios.push({
|
|
1344
|
+
name: result.scenario,
|
|
1345
|
+
description: result.description,
|
|
1346
|
+
prompt: "",
|
|
1347
|
+
rubric: {},
|
|
1348
|
+
...(result.skipped ? { skip: true } : {}),
|
|
1349
|
+
...(result.compare
|
|
1350
|
+
? {
|
|
1351
|
+
compare: compareResultArms(result.compare).map((arm) => ({
|
|
1352
|
+
id: arm.id,
|
|
1353
|
+
label: arm.label,
|
|
1354
|
+
description: arm.description,
|
|
1355
|
+
prompt: arm.prompt ?? result.prompt ?? "",
|
|
1356
|
+
rubric: {},
|
|
1357
|
+
})),
|
|
1358
|
+
gates: result.compare.gates,
|
|
1359
|
+
}
|
|
1360
|
+
: {}),
|
|
1361
|
+
});
|
|
1362
|
+
}
|
|
1363
|
+
}
|
|
1364
|
+
return { suitesDir, defaultSelectedHosts: [], suites: [...suites.values()] };
|
|
1284
1365
|
}
|
|
1285
1366
|
/**
|
|
1286
1367
|
* Write an HTML report. Writes to `outPath` when given (parent dirs created),
|