@post-print/agent-test 1.1.8 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -66
- package/dist/cli.d.ts +2 -64
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +26 -691
- package/dist/cli.js.map +1 -1
- package/dist/compare-scenario.d.ts +1 -1
- package/dist/compare-scenario.d.ts.map +1 -1
- package/dist/compare-scenario.js +72 -75
- package/dist/compare-scenario.js.map +1 -1
- package/dist/failures.d.ts +5 -1
- package/dist/failures.d.ts.map +1 -1
- package/dist/failures.js +1 -1
- package/dist/failures.js.map +1 -1
- package/dist/host-login.d.ts +1 -26
- package/dist/host-login.d.ts.map +1 -1
- package/dist/host-login.js +1 -92
- package/dist/host-login.js.map +1 -1
- package/dist/index.d.ts +6 -28
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -26
- package/dist/index.js.map +1 -1
- package/dist/scenario-story.d.ts.map +1 -1
- package/dist/scenario-story.js +57 -39
- package/dist/scenario-story.js.map +1 -1
- package/dist/sdk/cli.d.ts +3 -0
- package/dist/sdk/cli.d.ts.map +1 -0
- package/dist/sdk/cli.js +20 -0
- package/dist/sdk/cli.js.map +1 -0
- package/dist/sdk/definitions.d.ts +22 -0
- package/dist/sdk/definitions.d.ts.map +1 -0
- package/dist/sdk/definitions.js +58 -0
- package/dist/sdk/definitions.js.map +1 -0
- package/dist/sdk/expect.d.ts +31 -0
- package/dist/sdk/expect.d.ts.map +1 -0
- package/dist/sdk/expect.js +68 -0
- package/dist/sdk/expect.js.map +1 -0
- package/dist/sdk/judge.d.ts +16 -0
- package/dist/sdk/judge.d.ts.map +1 -0
- package/dist/sdk/judge.js +82 -0
- package/dist/sdk/judge.js.map +1 -0
- package/dist/sdk/metrics.d.ts +3 -0
- package/dist/sdk/metrics.d.ts.map +1 -0
- package/dist/sdk/metrics.js +24 -0
- package/dist/sdk/metrics.js.map +1 -0
- package/dist/sdk/reporter.d.ts +12 -0
- package/dist/sdk/reporter.d.ts.map +1 -0
- package/dist/sdk/reporter.js +58 -0
- package/dist/sdk/reporter.js.map +1 -0
- package/dist/sdk/runtime.d.ts +33 -0
- package/dist/sdk/runtime.d.ts.map +1 -0
- package/dist/sdk/runtime.js +188 -0
- package/dist/sdk/runtime.js.map +1 -0
- package/dist/sdk/test.d.ts +27 -0
- package/dist/sdk/test.d.ts.map +1 -0
- package/dist/sdk/test.js +92 -0
- package/dist/sdk/test.js.map +1 -0
- package/dist/sdk/types.d.ts +75 -0
- package/dist/sdk/types.d.ts.map +1 -0
- package/dist/sdk/types.js +2 -0
- package/dist/sdk/types.js.map +1 -0
- package/dist/sdk/usage.d.ts +10 -0
- package/dist/sdk/usage.d.ts.map +1 -0
- package/dist/sdk/usage.js +13 -0
- package/dist/sdk/usage.js.map +1 -0
- package/dist/sdk/viewer-results.d.ts +43 -0
- package/dist/sdk/viewer-results.d.ts.map +1 -0
- package/dist/sdk/viewer-results.js +126 -0
- package/dist/sdk/viewer-results.js.map +1 -0
- package/dist/sdk/viewer.d.ts +11 -0
- package/dist/sdk/viewer.d.ts.map +1 -0
- package/dist/sdk/viewer.js +160 -0
- package/dist/sdk/viewer.js.map +1 -0
- package/dist/sdk/workspace.d.ts +33 -0
- package/dist/sdk/workspace.d.ts.map +1 -0
- package/dist/sdk/workspace.js +100 -0
- package/dist/sdk/workspace.js.map +1 -0
- package/dist/types.d.ts +4 -40
- package/dist/types.d.ts.map +1 -1
- package/dist/viewer/catalog.d.ts +2 -8
- package/dist/viewer/catalog.d.ts.map +1 -1
- package/dist/viewer/catalog.js +23 -128
- package/dist/viewer/catalog.js.map +1 -1
- package/dist/viewer/client.d.ts.map +1 -1
- package/dist/viewer/client.js +12 -12
- package/dist/viewer/client.js.map +1 -1
- package/dist/viewer/run-controller.d.ts +4 -2
- package/dist/viewer/run-controller.d.ts.map +1 -1
- package/dist/viewer/run-controller.js +297 -248
- package/dist/viewer/run-controller.js.map +1 -1
- package/dist/viewer/server.d.ts +4 -3
- package/dist/viewer/server.d.ts.map +1 -1
- package/dist/viewer/server.js +149 -112
- package/dist/viewer/server.js.map +1 -1
- package/dist/viewer/styles.d.ts +2 -0
- package/dist/viewer/styles.d.ts.map +1 -0
- package/dist/viewer/styles.js +444 -0
- package/dist/viewer/styles.js.map +1 -0
- package/dist/worker-pool.d.ts +5 -1
- package/dist/worker-pool.d.ts.map +1 -1
- package/dist/worker-pool.js +1 -1
- package/dist/worker-pool.js.map +1 -1
- package/package.json +7 -7
- package/dist/check.d.ts +0 -38
- package/dist/check.d.ts.map +0 -1
- package/dist/check.js +0 -169
- package/dist/check.js.map +0 -1
- package/dist/command-allowlist.d.ts +0 -9
- package/dist/command-allowlist.d.ts.map +0 -1
- package/dist/command-allowlist.js +0 -29
- package/dist/command-allowlist.js.map +0 -1
- package/dist/debug-bundle.d.ts +0 -81
- package/dist/debug-bundle.d.ts.map +0 -1
- package/dist/debug-bundle.js +0 -392
- package/dist/debug-bundle.js.map +0 -1
- package/dist/discover-suites.d.ts +0 -2
- package/dist/discover-suites.d.ts.map +0 -1
- package/dist/discover-suites.js +0 -19
- package/dist/discover-suites.js.map +0 -1
- package/dist/doctor.d.ts +0 -24
- package/dist/doctor.d.ts.map +0 -1
- package/dist/doctor.js +0 -272
- package/dist/doctor.js.map +0 -1
- package/dist/expect.d.ts +0 -59
- package/dist/expect.d.ts.map +0 -1
- package/dist/expect.js +0 -518
- package/dist/expect.js.map +0 -1
- package/dist/host-log.d.ts +0 -17
- package/dist/host-log.d.ts.map +0 -1
- package/dist/host-log.js +0 -154
- package/dist/host-log.js.map +0 -1
- package/dist/hosts.d.ts +0 -22
- package/dist/hosts.d.ts.map +0 -1
- package/dist/hosts.js +0 -84
- package/dist/hosts.js.map +0 -1
- package/dist/html-report.d.ts +0 -18
- package/dist/html-report.d.ts.map +0 -1
- package/dist/html-report.js +0 -1276
- package/dist/html-report.js.map +0 -1
- package/dist/live-isolation.d.ts +0 -82
- package/dist/live-isolation.d.ts.map +0 -1
- package/dist/live-isolation.js +0 -399
- package/dist/live-isolation.js.map +0 -1
- package/dist/live-timeout.d.ts +0 -11
- package/dist/live-timeout.d.ts.map +0 -1
- package/dist/live-timeout.js +0 -29
- package/dist/live-timeout.js.map +0 -1
- package/dist/load-adapters.d.ts +0 -13
- package/dist/load-adapters.d.ts.map +0 -1
- package/dist/load-adapters.js +0 -76
- package/dist/load-adapters.js.map +0 -1
- package/dist/load-suite.d.ts +0 -22
- package/dist/load-suite.d.ts.map +0 -1
- package/dist/load-suite.js +0 -154
- package/dist/load-suite.js.map +0 -1
- package/dist/mcp-config.d.ts +0 -13
- package/dist/mcp-config.d.ts.map +0 -1
- package/dist/mcp-config.js +0 -121
- package/dist/mcp-config.js.map +0 -1
- package/dist/preflight.d.ts +0 -3
- package/dist/preflight.d.ts.map +0 -1
- package/dist/preflight.js +0 -17
- package/dist/preflight.js.map +0 -1
- package/dist/progress.d.ts +0 -24
- package/dist/progress.d.ts.map +0 -1
- package/dist/progress.js +0 -111
- package/dist/progress.js.map +0 -1
- package/dist/record-trace.d.ts +0 -64
- package/dist/record-trace.d.ts.map +0 -1
- package/dist/record-trace.js +0 -124
- package/dist/record-trace.js.map +0 -1
- package/dist/report-preview-server.d.ts +0 -2
- package/dist/report-preview-server.d.ts.map +0 -1
- package/dist/report-preview-server.js +0 -24
- package/dist/report-preview-server.js.map +0 -1
- package/dist/report-preview.d.ts +0 -35
- package/dist/report-preview.d.ts.map +0 -1
- package/dist/report-preview.js +0 -205
- package/dist/report-preview.js.map +0 -1
- package/dist/run-suite.d.ts +0 -139
- package/dist/run-suite.d.ts.map +0 -1
- package/dist/run-suite.js +0 -1669
- package/dist/run-suite.js.map +0 -1
- package/dist/scenario-finalizer.d.ts +0 -27
- package/dist/scenario-finalizer.d.ts.map +0 -1
- package/dist/scenario-finalizer.js +0 -39
- package/dist/scenario-finalizer.js.map +0 -1
- package/dist/scenario-retry.d.ts +0 -15
- package/dist/scenario-retry.d.ts.map +0 -1
- package/dist/scenario-retry.js +0 -52
- package/dist/scenario-retry.js.map +0 -1
- package/dist/scenario-seed.d.ts +0 -13
- package/dist/scenario-seed.d.ts.map +0 -1
- package/dist/scenario-seed.js +0 -72
- package/dist/scenario-seed.js.map +0 -1
- package/dist/suite-summary.d.ts +0 -16
- package/dist/suite-summary.d.ts.map +0 -1
- package/dist/suite-summary.js +0 -218
- package/dist/suite-summary.js.map +0 -1
- package/dist/theme.d.ts +0 -96
- package/dist/theme.d.ts.map +0 -1
- package/dist/theme.js +0 -446
- package/dist/theme.js.map +0 -1
- package/dist/validate-seeds.d.ts +0 -20
- package/dist/validate-seeds.d.ts.map +0 -1
- package/dist/validate-seeds.js +0 -72
- package/dist/validate-seeds.js.map +0 -1
- package/dist/validate-suite.d.ts +0 -24
- package/dist/validate-suite.d.ts.map +0 -1
- package/dist/validate-suite.js +0 -542
- package/dist/validate-suite.js.map +0 -1
- package/dist/viewer/cli.d.ts +0 -17
- package/dist/viewer/cli.d.ts.map +0 -1
- package/dist/viewer/cli.js +0 -38
- package/dist/viewer/cli.js.map +0 -1
- package/dist/viewer/compare-metrics.d.ts +0 -30
- package/dist/viewer/compare-metrics.d.ts.map +0 -1
- package/dist/viewer/compare-metrics.js +0 -101
- package/dist/viewer/compare-metrics.js.map +0 -1
- package/dist/viewer/context-files.d.ts +0 -8
- package/dist/viewer/context-files.d.ts.map +0 -1
- package/dist/viewer/context-files.js +0 -57
- package/dist/viewer/context-files.js.map +0 -1
- package/dist/viewer/emit.d.ts +0 -7
- package/dist/viewer/emit.d.ts.map +0 -1
- package/dist/viewer/emit.js +0 -25
- package/dist/viewer/emit.js.map +0 -1
- package/dist/viewer/live-runner.d.ts +0 -27
- package/dist/viewer/live-runner.d.ts.map +0 -1
- package/dist/viewer/live-runner.js +0 -166
- package/dist/viewer/live-runner.js.map +0 -1
- package/dist/viewer/session-server.d.ts +0 -2
- package/dist/viewer/session-server.d.ts.map +0 -1
- package/dist/viewer/session-server.js +0 -71
- package/dist/viewer/session-server.js.map +0 -1
- package/dist/warnings.d.ts +0 -3
- package/dist/warnings.d.ts.map +0 -1
- package/dist/warnings.js +0 -13
- package/dist/warnings.js.map +0 -1
package/dist/html-report.js
DELETED
|
@@ -1,1276 +0,0 @@
|
|
|
1
|
-
import { mkdir, mkdtemp, writeFile } from "node:fs/promises";
|
|
2
|
-
import { tmpdir } from "node:os";
|
|
3
|
-
import { dirname, join } from "node:path";
|
|
4
|
-
import { compareArmTokens, compareArmTurns, compareResultArms, describeCompareOutcome, formatCompareTurns, } from "./compare-scenario.js";
|
|
5
|
-
import { displayToolPath } from "./scenario-story.js";
|
|
6
|
-
import { renderViewerPage } from "./viewer/page.js";
|
|
7
|
-
function escapeHtml(value) {
|
|
8
|
-
return value
|
|
9
|
-
.replaceAll("&", "&")
|
|
10
|
-
.replaceAll("<", "<")
|
|
11
|
-
.replaceAll(">", ">")
|
|
12
|
-
.replaceAll('"', """)
|
|
13
|
-
.replaceAll("'", "'");
|
|
14
|
-
}
|
|
15
|
-
const INTEGER_FORMAT = new Intl.NumberFormat("en-US", { maximumFractionDigits: 0 });
|
|
16
|
-
const DECIMAL_FORMAT = new Intl.NumberFormat("en-US", {
|
|
17
|
-
minimumFractionDigits: 1,
|
|
18
|
-
maximumFractionDigits: 1,
|
|
19
|
-
});
|
|
20
|
-
function formatInteger(value) {
|
|
21
|
-
return INTEGER_FORMAT.format(Math.round(value));
|
|
22
|
-
}
|
|
23
|
-
function formatDuration(ms) {
|
|
24
|
-
if (ms < 1000) {
|
|
25
|
-
return `${formatInteger(ms)}ms`;
|
|
26
|
-
}
|
|
27
|
-
return `${DECIMAL_FORMAT.format(ms / 1000)}s`;
|
|
28
|
-
}
|
|
29
|
-
function statusLabel(result) {
|
|
30
|
-
if (result.skipped) {
|
|
31
|
-
return "skipped";
|
|
32
|
-
}
|
|
33
|
-
return result.passed ? "passed" : "failed";
|
|
34
|
-
}
|
|
35
|
-
function statusClass(result) {
|
|
36
|
-
if (result.skipped) {
|
|
37
|
-
return "status-skipped";
|
|
38
|
-
}
|
|
39
|
-
return result.passed ? "status-passed" : "status-failed";
|
|
40
|
-
}
|
|
41
|
-
const MISSING_REQUIREMENT = {
|
|
42
|
-
label: "Missing requirement",
|
|
43
|
-
hint: "A required string did not appear in assistant text, commands, or tool results.",
|
|
44
|
-
};
|
|
45
|
-
const FORBIDDEN_BEHAVIOR = {
|
|
46
|
-
label: "Forbidden behavior",
|
|
47
|
-
hint: "The agent did something it was told not to do.",
|
|
48
|
-
};
|
|
49
|
-
const MISSING_COMMAND = {
|
|
50
|
-
label: "Missing command",
|
|
51
|
-
hint: "An expected command never ran.",
|
|
52
|
-
};
|
|
53
|
-
const FORBIDDEN_COMMAND = {
|
|
54
|
-
label: "Forbidden command",
|
|
55
|
-
hint: "A shell command ran that is not on the allowedCommands list.",
|
|
56
|
-
};
|
|
57
|
-
const MISSING_TOOL = {
|
|
58
|
-
label: "Missing tool call",
|
|
59
|
-
hint: "An expected tool was never invoked.",
|
|
60
|
-
};
|
|
61
|
-
const FORBIDDEN_TOOL = {
|
|
62
|
-
label: "Forbidden tool call",
|
|
63
|
-
hint: "A tool the scenario forbids was invoked.",
|
|
64
|
-
};
|
|
65
|
-
const MISSING_SKILL = {
|
|
66
|
-
label: "Missing skill",
|
|
67
|
-
hint: "The agent did not read a skill it was expected to use.",
|
|
68
|
-
};
|
|
69
|
-
const UNEXPECTED_SKILL = {
|
|
70
|
-
label: "Unexpected skill",
|
|
71
|
-
hint: "The agent read a skill it should have avoided.",
|
|
72
|
-
};
|
|
73
|
-
const MISSING_READ_PATH = {
|
|
74
|
-
label: "Missing read path",
|
|
75
|
-
hint: "Expected a Read tool call whose args mention this path fragment (registry-first / grounding).",
|
|
76
|
-
};
|
|
77
|
-
const FORBIDDEN_READ_PATH = {
|
|
78
|
-
label: "Forbidden read path",
|
|
79
|
-
hint: "A successful Read returned content from a path the scenario forbids (miss attempts do not fail).",
|
|
80
|
-
};
|
|
81
|
-
const ROUTING = {
|
|
82
|
-
label: "Routing",
|
|
83
|
-
hint: "The routing announcement did not match expectations.",
|
|
84
|
-
};
|
|
85
|
-
/** Human-readable label + one-line explanation for the matcher codes assertRubric/judge emit. */
|
|
86
|
-
const MATCHER_LABELS = {
|
|
87
|
-
runAgent: { label: "Agent run", hint: "The agent session itself failed or was cut short." },
|
|
88
|
-
liveScenario: {
|
|
89
|
-
label: "Isolated scenario",
|
|
90
|
-
hint: "The scenario subprocess exited before it wrote a result.",
|
|
91
|
-
},
|
|
92
|
-
must: MISSING_REQUIREMENT,
|
|
93
|
-
mustInclude: MISSING_REQUIREMENT,
|
|
94
|
-
mustNot: FORBIDDEN_BEHAVIOR,
|
|
95
|
-
mustNotInclude: FORBIDDEN_BEHAVIOR,
|
|
96
|
-
mustRun: MISSING_COMMAND,
|
|
97
|
-
toHaveRunCommand: MISSING_COMMAND,
|
|
98
|
-
mustRunSuccessfully: MISSING_COMMAND,
|
|
99
|
-
toHaveRunCommandSuccessfully: {
|
|
100
|
-
label: "Command did not pass",
|
|
101
|
-
hint: "The required command was missing, failed, or did not report a structured execution status.",
|
|
102
|
-
},
|
|
103
|
-
allowedCommands: FORBIDDEN_COMMAND,
|
|
104
|
-
toHaveAllowedCommands: FORBIDDEN_COMMAND,
|
|
105
|
-
mustCallTool: MISSING_TOOL,
|
|
106
|
-
toHaveCalledTool: MISSING_TOOL,
|
|
107
|
-
mustCallToolsInOrder: MISSING_TOOL,
|
|
108
|
-
toHaveCalledToolsInOrder: MISSING_TOOL,
|
|
109
|
-
mustNotCallTool: FORBIDDEN_TOOL,
|
|
110
|
-
toHaveNotCalledTool: FORBIDDEN_TOOL,
|
|
111
|
-
mustInvokeSkill: MISSING_SKILL,
|
|
112
|
-
toHaveInvokedSkill: MISSING_SKILL,
|
|
113
|
-
mustNotInvokeSkill: UNEXPECTED_SKILL,
|
|
114
|
-
toHaveNotInvokedSkill: UNEXPECTED_SKILL,
|
|
115
|
-
mustReadPath: MISSING_READ_PATH,
|
|
116
|
-
toHaveReadPath: MISSING_READ_PATH,
|
|
117
|
-
mustNotReadPath: FORBIDDEN_READ_PATH,
|
|
118
|
-
toHaveNotReadPath: FORBIDDEN_READ_PATH,
|
|
119
|
-
routingBlock: ROUTING,
|
|
120
|
-
toIncludeRoutingBlock: ROUTING,
|
|
121
|
-
toHaveRoutingBlockBeforeTools: ROUTING,
|
|
122
|
-
toHaveTier: ROUTING,
|
|
123
|
-
toHaveHandsOnTier: ROUTING,
|
|
124
|
-
toHaveHandsOnTierBeforeTools: ROUTING,
|
|
125
|
-
toHaveReviewDepth: {
|
|
126
|
-
label: "Review depth",
|
|
127
|
-
hint: "The review depth announcement did not match the rubric.",
|
|
128
|
-
},
|
|
129
|
-
workingTreeLeak: {
|
|
130
|
-
label: "Working tree leak",
|
|
131
|
-
hint: "The agent used a path outside the sealed workspace, or edited the caller checkout.",
|
|
132
|
-
},
|
|
133
|
-
recordTrace: { label: "Recording failed", hint: "Saving the trace to disk failed." },
|
|
134
|
-
judge: { label: "Judge", hint: "The LLM judge flagged this scenario." },
|
|
135
|
-
compareGate: { label: "Comparison gate", hint: "A declared comparison condition did not pass." },
|
|
136
|
-
};
|
|
137
|
-
function humanizeMatcher(matcher) {
|
|
138
|
-
const [base] = matcher.split(":");
|
|
139
|
-
const known = MATCHER_LABELS[base ?? matcher];
|
|
140
|
-
if (known) {
|
|
141
|
-
if (matcher.includes(":")) {
|
|
142
|
-
const suffix = matcher.slice(matcher.indexOf(":") + 1);
|
|
143
|
-
return { label: `${known.label} — ${suffix}`, hint: known.hint };
|
|
144
|
-
}
|
|
145
|
-
return known;
|
|
146
|
-
}
|
|
147
|
-
return { label: matcher };
|
|
148
|
-
}
|
|
149
|
-
const ROLE_META = {
|
|
150
|
-
user: { label: "User", side: "right" },
|
|
151
|
-
assistant: { label: "Agent", side: "left" },
|
|
152
|
-
system: { label: "System", side: "center" },
|
|
153
|
-
tool: { label: "Tool", side: "center" },
|
|
154
|
-
};
|
|
155
|
-
function renderMessageBubble(message) {
|
|
156
|
-
const meta = ROLE_META[message.role] ?? { label: message.role, side: "center" };
|
|
157
|
-
return `
|
|
158
|
-
<div class="chat-row side-${meta.side}">
|
|
159
|
-
<div class="bubble role-${escapeHtml(message.role)}">
|
|
160
|
-
<div class="bubble-label">${escapeHtml(meta.label)}</div>
|
|
161
|
-
<div class="bubble-text">${escapeHtml(message.content)}</div>
|
|
162
|
-
</div>
|
|
163
|
-
</div>`;
|
|
164
|
-
}
|
|
165
|
-
const PATH_ARG_KEYS = new Set(["path", "file_path", "filePath", "target_file", "uri", "cwd"]);
|
|
166
|
-
function isPathArg(key, value) {
|
|
167
|
-
return PATH_ARG_KEYS.has(key) || value.startsWith("/") || value.startsWith("file://");
|
|
168
|
-
}
|
|
169
|
-
/** Truncate long single-line arg values so a stray file body doesn't blow up the card. */
|
|
170
|
-
function formatArgValue(key, value) {
|
|
171
|
-
if (typeof value === "string" && isPathArg(key, value)) {
|
|
172
|
-
const text = displayToolPath(value);
|
|
173
|
-
return text === value ? { text } : { text, title: value };
|
|
174
|
-
}
|
|
175
|
-
const raw = typeof value === "string" ? value : (JSON.stringify(value) ?? String(value));
|
|
176
|
-
const oneLine = raw.replaceAll("\n", " ↵ ");
|
|
177
|
-
return { text: oneLine.length > 140 ? `${oneLine.slice(0, 140)}…` : oneLine };
|
|
178
|
-
}
|
|
179
|
-
function renderToolArgs(args) {
|
|
180
|
-
const entries = args ? Object.entries(args) : [];
|
|
181
|
-
if (entries.length === 0) {
|
|
182
|
-
return "";
|
|
183
|
-
}
|
|
184
|
-
const rows = entries
|
|
185
|
-
.map(([key, value]) => {
|
|
186
|
-
const formatted = formatArgValue(key, value);
|
|
187
|
-
const title = formatted.title ? ` title="${escapeHtml(formatted.title)}"` : "";
|
|
188
|
-
return `<div class="tool-arg"><span class="tool-arg-key">${escapeHtml(key)}</span><code${title}>${escapeHtml(formatted.text)}</code></div>`;
|
|
189
|
-
})
|
|
190
|
-
.join("");
|
|
191
|
-
return `<div class="tool-args">${rows}</div>`;
|
|
192
|
-
}
|
|
193
|
-
function renderToolCallCard(call) {
|
|
194
|
-
return `
|
|
195
|
-
<div class="chat-row side-left">
|
|
196
|
-
<div class="tool-card">
|
|
197
|
-
<div class="tool-card-head"><span class="tool-icon">⚙</span><span class="tool-name">${escapeHtml(call.name)}</span></div>
|
|
198
|
-
${renderToolArgs(call.args)}
|
|
199
|
-
</div>
|
|
200
|
-
</div>`;
|
|
201
|
-
}
|
|
202
|
-
/** Interleave messages and tool calls chronologically when the trace recorded emission order. */
|
|
203
|
-
function buildOrderedTimeline(trace) {
|
|
204
|
-
const items = [
|
|
205
|
-
...trace.messages.map((message) => message.seq === undefined
|
|
206
|
-
? undefined
|
|
207
|
-
: { kind: "message", seq: message.seq, message }),
|
|
208
|
-
...trace.toolCalls.map((call) => call.seq === undefined ? undefined : { kind: "tool", seq: call.seq, call }),
|
|
209
|
-
].filter((item) => item !== undefined);
|
|
210
|
-
const totalItems = trace.messages.length + trace.toolCalls.length;
|
|
211
|
-
if (items.length !== totalItems || totalItems === 0) {
|
|
212
|
-
return undefined;
|
|
213
|
-
}
|
|
214
|
-
return items.sort((a, b) => a.seq - b.seq);
|
|
215
|
-
}
|
|
216
|
-
function promptAlreadyInTrace(trace, prompt) {
|
|
217
|
-
return (trace?.messages.some((message) => message.role === "user" && message.content.trim() === prompt) ?? false);
|
|
218
|
-
}
|
|
219
|
-
function renderPromptBubble(prompt, trace) {
|
|
220
|
-
const text = prompt?.trim();
|
|
221
|
-
if (!text || promptAlreadyInTrace(trace, text)) {
|
|
222
|
-
return "";
|
|
223
|
-
}
|
|
224
|
-
return renderMessageBubble({ role: "user", content: text });
|
|
225
|
-
}
|
|
226
|
-
function renderChat(trace, prompt) {
|
|
227
|
-
const promptBubble = renderPromptBubble(prompt, trace);
|
|
228
|
-
if (!trace) {
|
|
229
|
-
if (promptBubble) {
|
|
230
|
-
return `<div class="chat">${promptBubble}</div>`;
|
|
231
|
-
}
|
|
232
|
-
return `<p class="empty">No transcript recorded for this scenario.</p>`;
|
|
233
|
-
}
|
|
234
|
-
const parts = [];
|
|
235
|
-
const timeline = buildOrderedTimeline(trace);
|
|
236
|
-
if (timeline) {
|
|
237
|
-
parts.push(`<div class="chat">`);
|
|
238
|
-
if (promptBubble) {
|
|
239
|
-
parts.push(promptBubble);
|
|
240
|
-
}
|
|
241
|
-
for (const item of timeline) {
|
|
242
|
-
parts.push(item.kind === "message" ? renderMessageBubble(item.message) : renderToolCallCard(item.call));
|
|
243
|
-
}
|
|
244
|
-
parts.push(`</div>`);
|
|
245
|
-
}
|
|
246
|
-
else if (trace.messages.length > 0 || trace.toolCalls.length > 0 || promptBubble) {
|
|
247
|
-
if (!timeline && (trace.messages.length > 0 || trace.toolCalls.length > 0)) {
|
|
248
|
-
parts.push(`<p class="empty note">Emission order wasn't recorded for this trace — messages and tool calls are shown in separate groups below.</p>`);
|
|
249
|
-
}
|
|
250
|
-
if (promptBubble || trace.messages.length > 0) {
|
|
251
|
-
parts.push(`<div class="chat">`);
|
|
252
|
-
if (promptBubble) {
|
|
253
|
-
parts.push(promptBubble);
|
|
254
|
-
}
|
|
255
|
-
for (const message of trace.messages) {
|
|
256
|
-
parts.push(renderMessageBubble(message));
|
|
257
|
-
}
|
|
258
|
-
parts.push(`</div>`);
|
|
259
|
-
}
|
|
260
|
-
if (trace.toolCalls.length > 0) {
|
|
261
|
-
parts.push(`<h4>Tool calls</h4><div class="chat">`);
|
|
262
|
-
for (const call of trace.toolCalls) {
|
|
263
|
-
parts.push(renderToolCallCard(call));
|
|
264
|
-
}
|
|
265
|
-
parts.push(`</div>`);
|
|
266
|
-
}
|
|
267
|
-
}
|
|
268
|
-
else {
|
|
269
|
-
parts.push(`<p class="empty">No messages in transcript.</p>`);
|
|
270
|
-
}
|
|
271
|
-
if (trace.shellCommands.length > 0) {
|
|
272
|
-
parts.push(`<h4>Shell commands</h4><ul class="shell-commands">`);
|
|
273
|
-
for (const command of trace.shellCommands) {
|
|
274
|
-
parts.push(`<li><code>${escapeHtml(command)}</code></li>`);
|
|
275
|
-
}
|
|
276
|
-
parts.push(`</ul>`);
|
|
277
|
-
}
|
|
278
|
-
return parts.join("\n");
|
|
279
|
-
}
|
|
280
|
-
function renderJudgeVerdicts(result) {
|
|
281
|
-
const verdicts = result.judgeVerdicts;
|
|
282
|
-
if (!verdicts || verdicts.length === 0) {
|
|
283
|
-
return "";
|
|
284
|
-
}
|
|
285
|
-
const items = verdicts
|
|
286
|
-
.map((verdict) => {
|
|
287
|
-
const badge = verdict.pass ? "pass" : "fail";
|
|
288
|
-
const icon = verdict.pass ? "✓" : "✗";
|
|
289
|
-
const evidence = verdict.evidence?.length
|
|
290
|
-
? `<ul class="judge-evidence">${verdict.evidence.map((line) => `<li>${escapeHtml(line)}</li>`).join("")}</ul>`
|
|
291
|
-
: "";
|
|
292
|
-
const workspaceEvidence = verdict.workspaceEvidence?.length
|
|
293
|
-
? `<p class="judge-workspaces">Workspace evidence: ${verdict.workspaceEvidence.map(escapeHtml).join(", ")}</p>`
|
|
294
|
-
: "";
|
|
295
|
-
const exchange = verdict.prompt || verdict.response
|
|
296
|
-
? `<details class="judge-exchange"><summary>Judge conversation</summary><div class="judge-chat">
|
|
297
|
-
${verdict.prompt ? `<div class="judge-turn judge-input"><span>Input</span><pre>${escapeHtml(verdict.prompt)}</pre></div>` : ""}
|
|
298
|
-
${verdict.response ? `<div class="judge-turn judge-response"><span>Response</span><pre>${escapeHtml(verdict.response)}</pre></div>` : ""}
|
|
299
|
-
</div></details>`
|
|
300
|
-
: "";
|
|
301
|
-
return `
|
|
302
|
-
<article class="verdict verdict-${badge}">
|
|
303
|
-
<p class="question"><span class="verdict-icon">${icon}</span>${escapeHtml(verdict.question)}</p>
|
|
304
|
-
<p class="rationale">${escapeHtml(verdict.rationale)}</p>
|
|
305
|
-
${workspaceEvidence}
|
|
306
|
-
${evidence}
|
|
307
|
-
${exchange}
|
|
308
|
-
</article>`;
|
|
309
|
-
})
|
|
310
|
-
.join("\n");
|
|
311
|
-
return `<div class="verdicts">${items}</div>`;
|
|
312
|
-
}
|
|
313
|
-
function renderFailures(result) {
|
|
314
|
-
if (result.failures.length === 0) {
|
|
315
|
-
return "";
|
|
316
|
-
}
|
|
317
|
-
const items = result.failures
|
|
318
|
-
.map((failure) => {
|
|
319
|
-
const { label, hint } = humanizeMatcher(failure.matcher);
|
|
320
|
-
const evidence = failure.evidence
|
|
321
|
-
? `<pre class="failure-evidence">${escapeHtml(failure.evidence)}</pre>`
|
|
322
|
-
: "";
|
|
323
|
-
return `
|
|
324
|
-
<li>
|
|
325
|
-
<p class="failure-label">${escapeHtml(label)}</p>
|
|
326
|
-
${hint ? `<p class="failure-hint">${escapeHtml(hint)}</p>` : ""}
|
|
327
|
-
<p class="failure-message">${escapeHtml(failure.message)}</p>
|
|
328
|
-
${evidence}
|
|
329
|
-
</li>`;
|
|
330
|
-
})
|
|
331
|
-
.join("\n");
|
|
332
|
-
return `<ul class="failures">${items}</ul>`;
|
|
333
|
-
}
|
|
334
|
-
function usageOf(result) {
|
|
335
|
-
return result.usage?.total ?? result.trace?.usage;
|
|
336
|
-
}
|
|
337
|
-
function formatTokensBadge(result) {
|
|
338
|
-
const usage = usageOf(result);
|
|
339
|
-
if (!usage) {
|
|
340
|
-
return undefined;
|
|
341
|
-
}
|
|
342
|
-
if (typeof usage.totalTokens === "number") {
|
|
343
|
-
return `${formatInteger(usage.totalTokens)} tokens`;
|
|
344
|
-
}
|
|
345
|
-
const parts = [];
|
|
346
|
-
if (typeof usage.inputTokens === "number") {
|
|
347
|
-
parts.push(`in ${formatInteger(usage.inputTokens)}`);
|
|
348
|
-
}
|
|
349
|
-
if (typeof usage.outputTokens === "number") {
|
|
350
|
-
parts.push(`out ${formatInteger(usage.outputTokens)}`);
|
|
351
|
-
}
|
|
352
|
-
return parts.length > 0 ? parts.join(" · ") : undefined;
|
|
353
|
-
}
|
|
354
|
-
function renderUsageDetail(usage) {
|
|
355
|
-
if (!usage) {
|
|
356
|
-
return "";
|
|
357
|
-
}
|
|
358
|
-
const rows = [
|
|
359
|
-
["Total tokens", usage.totalTokens],
|
|
360
|
-
["Input", usage.inputTokens],
|
|
361
|
-
["Output", usage.outputTokens],
|
|
362
|
-
["Cache read", usage.cacheReadTokens],
|
|
363
|
-
["Cache write", usage.cacheWriteTokens],
|
|
364
|
-
["Reasoning", usage.reasoningTokens],
|
|
365
|
-
];
|
|
366
|
-
const present = rows.filter(([, value]) => typeof value === "number");
|
|
367
|
-
if (present.length === 0) {
|
|
368
|
-
return "";
|
|
369
|
-
}
|
|
370
|
-
const items = present
|
|
371
|
-
.map(([label, value]) => `<div class="meta-item"><span class="meta-key">${escapeHtml(label)}</span><span class="meta-val">${formatInteger(value ?? 0)}</span></div>`)
|
|
372
|
-
.join("");
|
|
373
|
-
return `<section class="scenario-meta"><h3>Token usage</h3><div class="meta-grid">${items}</div></section>`;
|
|
374
|
-
}
|
|
375
|
-
function renderTraceMeta(result) {
|
|
376
|
-
const trace = result.trace;
|
|
377
|
-
if (!trace) {
|
|
378
|
-
return "";
|
|
379
|
-
}
|
|
380
|
-
const skills = trace.skillsInvoked?.length ? trace.skillsInvoked.join(", ") : "(none)";
|
|
381
|
-
const items = [
|
|
382
|
-
["Messages", formatInteger(trace.messages.length)],
|
|
383
|
-
["Tool calls", formatInteger(trace.toolCalls.length)],
|
|
384
|
-
["Skills invoked", skills],
|
|
385
|
-
["Routing tier", trace.routing?.tier ?? "(none)"],
|
|
386
|
-
];
|
|
387
|
-
if (result.attempts !== undefined && result.attempts > 1) {
|
|
388
|
-
items.push(["Attempts", formatInteger(result.attempts)]);
|
|
389
|
-
}
|
|
390
|
-
const html = items
|
|
391
|
-
.map(([label, value]) => `<div class="meta-item"><span class="meta-key">${escapeHtml(label)}</span><span class="meta-val">${escapeHtml(value)}</span></div>`)
|
|
392
|
-
.join("");
|
|
393
|
-
return `<section class="scenario-meta"><h3>Trace stats</h3><div class="meta-grid">${html}</div></section>`;
|
|
394
|
-
}
|
|
395
|
-
function formatSigned(value) {
|
|
396
|
-
if (value === undefined) {
|
|
397
|
-
return "n/a";
|
|
398
|
-
}
|
|
399
|
-
if (value > 0) {
|
|
400
|
-
return `+${formatInteger(value)}`;
|
|
401
|
-
}
|
|
402
|
-
if (value < 0) {
|
|
403
|
-
return `-${formatInteger(Math.abs(value))}`;
|
|
404
|
-
}
|
|
405
|
-
return formatInteger(value);
|
|
406
|
-
}
|
|
407
|
-
function deltaClass(value) {
|
|
408
|
-
if (value === undefined || value === 0) {
|
|
409
|
-
return "delta-flat";
|
|
410
|
-
}
|
|
411
|
-
return value > 0 ? "delta-up" : "delta-down";
|
|
412
|
-
}
|
|
413
|
-
function armToolCount(arm) {
|
|
414
|
-
return arm.trace ? arm.trace.toolCalls.length : undefined;
|
|
415
|
-
}
|
|
416
|
-
function formatArmTurns(arm) {
|
|
417
|
-
const turns = compareArmTurns(arm);
|
|
418
|
-
return turns === undefined ? "n/a" : formatCompareTurns(turns);
|
|
419
|
-
}
|
|
420
|
-
function formatArmTokens(arm, key) {
|
|
421
|
-
const usage = arm.trace?.usage;
|
|
422
|
-
const value = key === "total"
|
|
423
|
-
? usage?.totalTokens
|
|
424
|
-
: key === "input"
|
|
425
|
-
? usage?.inputTokens
|
|
426
|
-
: usage?.outputTokens;
|
|
427
|
-
return typeof value === "number" ? formatInteger(value) : "n/a";
|
|
428
|
-
}
|
|
429
|
-
function tokenDelta(a, b, key) {
|
|
430
|
-
const pick = (arm) => {
|
|
431
|
-
const usage = arm.trace?.usage;
|
|
432
|
-
return key === "total"
|
|
433
|
-
? usage?.totalTokens
|
|
434
|
-
: key === "input"
|
|
435
|
-
? usage?.inputTokens
|
|
436
|
-
: usage?.outputTokens;
|
|
437
|
-
};
|
|
438
|
-
const aValue = pick(a);
|
|
439
|
-
const bValue = pick(b);
|
|
440
|
-
if (typeof aValue !== "number" || typeof bValue !== "number") {
|
|
441
|
-
return undefined;
|
|
442
|
-
}
|
|
443
|
-
return bValue - aValue;
|
|
444
|
-
}
|
|
445
|
-
function winnerCellClass(delta, side) {
|
|
446
|
-
if (delta === undefined || delta === 0) {
|
|
447
|
-
return "";
|
|
448
|
-
}
|
|
449
|
-
if (side === "a") {
|
|
450
|
-
return delta > 0 ? "is-better" : "is-worse";
|
|
451
|
-
}
|
|
452
|
-
return delta < 0 ? "is-better" : "is-worse";
|
|
453
|
-
}
|
|
454
|
-
function renderCompareMetricRow(label, aText, bText, delta, deltaText) {
|
|
455
|
-
return `<tr>
|
|
456
|
-
<th scope="row">${escapeHtml(label)}</th>
|
|
457
|
-
<td class="${winnerCellClass(delta, "a")}">${escapeHtml(aText)}</td>
|
|
458
|
-
<td class="${winnerCellClass(delta, "b")}">${escapeHtml(bText)}</td>
|
|
459
|
-
<td class="${deltaClass(delta)}">${escapeHtml(deltaText)}</td>
|
|
460
|
-
</tr>`;
|
|
461
|
-
}
|
|
462
|
-
function renderNamedCompareMetricRow(label, cells, values) {
|
|
463
|
-
const numeric = (values ?? []).filter((value) => value !== undefined);
|
|
464
|
-
const low = numeric.length > 1 ? Math.min(...numeric) : undefined;
|
|
465
|
-
const high = numeric.length > 1 ? Math.max(...numeric) : undefined;
|
|
466
|
-
return `<tr>
|
|
467
|
-
<th scope="row">${escapeHtml(label)}</th>
|
|
468
|
-
${cells
|
|
469
|
-
.map((cell, index) => {
|
|
470
|
-
const value = values?.[index];
|
|
471
|
-
const state = value !== undefined && low !== high
|
|
472
|
-
? value === low
|
|
473
|
-
? "is-better"
|
|
474
|
-
: value === high
|
|
475
|
-
? "is-worse"
|
|
476
|
-
: ""
|
|
477
|
-
: "";
|
|
478
|
-
return `<td class="${state}">${escapeHtml(cell)}</td>`;
|
|
479
|
-
})
|
|
480
|
-
.join("")}
|
|
481
|
-
</tr>`;
|
|
482
|
-
}
|
|
483
|
-
function outcomeCell(arm, gates) {
|
|
484
|
-
if (arm.passed === undefined)
|
|
485
|
-
return "<td>n/a</td>";
|
|
486
|
-
if (arm.passed)
|
|
487
|
-
return '<td class="is-outcome-pass">pass</td>';
|
|
488
|
-
const expected = gates?.some((gate) => !("winner" in gate) &&
|
|
489
|
-
gate.arm === arm.id &&
|
|
490
|
-
gate.metric === "outcome" &&
|
|
491
|
-
gate.operator === "equal" &&
|
|
492
|
-
gate.value === "fail");
|
|
493
|
-
return expected
|
|
494
|
-
? '<td class="is-expected-failure">expected fail</td>'
|
|
495
|
-
: '<td class="is-outcome-fail">fail</td>';
|
|
496
|
-
}
|
|
497
|
-
function renderOutcomeRow(arms, gates, includeDelta = false) {
|
|
498
|
-
return `<tr><th scope="row">Outcome</th>${arms.map((arm) => outcomeCell(arm, gates)).join("")}${includeDelta ? '<td class="delta-flat">n/a</td>' : ""}</tr>`;
|
|
499
|
-
}
|
|
500
|
-
function renderTwoArmCompareMetrics(arms, gates) {
|
|
501
|
-
const [left, right] = arms;
|
|
502
|
-
const aTurns = compareArmTurns(left);
|
|
503
|
-
const bTurns = compareArmTurns(right);
|
|
504
|
-
const turnDelta = aTurns !== undefined && bTurns !== undefined ? bTurns - aTurns : undefined;
|
|
505
|
-
const aTools = armToolCount(left);
|
|
506
|
-
const bTools = armToolCount(right);
|
|
507
|
-
const toolDelta = aTools !== undefined && bTools !== undefined ? bTools - aTools : undefined;
|
|
508
|
-
const totalDelta = tokenDelta(left, right, "total");
|
|
509
|
-
return `
|
|
510
|
-
<div class="compare-table-wrap">
|
|
511
|
-
<table class="compare-table">
|
|
512
|
-
<thead>
|
|
513
|
-
<tr>
|
|
514
|
-
<th>Metric</th>
|
|
515
|
-
<th>${escapeHtml(left.label)}</th>
|
|
516
|
-
<th>${escapeHtml(right.label)}</th>
|
|
517
|
-
<th>Δ</th>
|
|
518
|
-
</tr>
|
|
519
|
-
</thead>
|
|
520
|
-
<tbody>
|
|
521
|
-
${renderOutcomeRow(arms, gates, true)}
|
|
522
|
-
${renderCompareMetricRow("Turns", formatArmTurns(left), formatArmTurns(right), turnDelta, formatSigned(turnDelta))}
|
|
523
|
-
${renderCompareMetricRow("Tokens", formatArmTokens(left, "total"), formatArmTokens(right, "total"), totalDelta, formatSigned(totalDelta))}
|
|
524
|
-
${renderCompareMetricRow("In", formatArmTokens(left, "input"), formatArmTokens(right, "input"), tokenDelta(left, right, "input"), formatSigned(tokenDelta(left, right, "input")))}
|
|
525
|
-
${renderCompareMetricRow("Out", formatArmTokens(left, "output"), formatArmTokens(right, "output"), tokenDelta(left, right, "output"), formatSigned(tokenDelta(left, right, "output")))}
|
|
526
|
-
${renderCompareMetricRow("Tools", aTools === undefined ? "n/a" : formatInteger(aTools), bTools === undefined ? "n/a" : formatInteger(bTools), toolDelta, formatSigned(toolDelta))}
|
|
527
|
-
</tbody>
|
|
528
|
-
</table>
|
|
529
|
-
</div>`;
|
|
530
|
-
}
|
|
531
|
-
function renderNamedCompareMetrics(arms, gates) {
|
|
532
|
-
return `
|
|
533
|
-
<div class="compare-table-wrap">
|
|
534
|
-
<table class="compare-table">
|
|
535
|
-
<thead>
|
|
536
|
-
<tr>
|
|
537
|
-
<th>Metric</th>
|
|
538
|
-
${arms.map((arm) => `<th>${escapeHtml(arm.label)}</th>`).join("")}
|
|
539
|
-
</tr>
|
|
540
|
-
</thead>
|
|
541
|
-
<tbody>
|
|
542
|
-
${renderOutcomeRow(arms, gates)}
|
|
543
|
-
${renderNamedCompareMetricRow("Turns", arms.map((arm) => formatArmTurns(arm)), arms.map(compareArmTurns))}
|
|
544
|
-
${renderNamedCompareMetricRow("Tokens", arms.map((arm) => formatArmTokens(arm, "total")), arms.map(compareArmTokens))}
|
|
545
|
-
${renderNamedCompareMetricRow("In", arms.map((arm) => formatArmTokens(arm, "input")))}
|
|
546
|
-
${renderNamedCompareMetricRow("Out", arms.map((arm) => formatArmTokens(arm, "output")))}
|
|
547
|
-
${renderNamedCompareMetricRow("Tools", arms.map((arm) => {
|
|
548
|
-
const tools = armToolCount(arm);
|
|
549
|
-
return tools === undefined ? "n/a" : formatInteger(tools);
|
|
550
|
-
}), arms.map(armToolCount))}
|
|
551
|
-
</tbody>
|
|
552
|
-
</table>
|
|
553
|
-
</div>`;
|
|
554
|
-
}
|
|
555
|
-
function renderCompareMetrics(result) {
|
|
556
|
-
const compare = result.compare;
|
|
557
|
-
if (!compare) {
|
|
558
|
-
return "";
|
|
559
|
-
}
|
|
560
|
-
const arms = compareResultArms(compare);
|
|
561
|
-
if (arms.length === 0) {
|
|
562
|
-
return "";
|
|
563
|
-
}
|
|
564
|
-
const twoArm = arms.length === 2 && arms[0] && arms[1];
|
|
565
|
-
const callouts = describeCompareOutcome(compare);
|
|
566
|
-
const calloutList = callouts.length
|
|
567
|
-
? `<details class="compare-details"><summary>Arm details</summary><ul class="compare-callouts">${callouts.map((line) => `<li>${escapeHtml(line)}</li>`).join("")}</ul></details>`
|
|
568
|
-
: "";
|
|
569
|
-
const lede = twoArm
|
|
570
|
-
? `${arms[0].label} vs ${arms[1].label}. Lower is better: green is lowest and red is highest.`
|
|
571
|
-
: `${arms.map((arm) => arm.label).join(", ")}. Lower is better: green is lowest and red is highest. Ties are neutral.`;
|
|
572
|
-
return `
|
|
573
|
-
<section class="compare comparison-metrics">
|
|
574
|
-
<header class="compare-header">
|
|
575
|
-
<h4>Comparison metrics</h4>
|
|
576
|
-
<p class="muted">${escapeHtml(lede)}</p>
|
|
577
|
-
</header>
|
|
578
|
-
${twoArm ? renderTwoArmCompareMetrics([arms[0], arms[1]], compare.gates) : renderNamedCompareMetrics(arms, compare.gates)}
|
|
579
|
-
${calloutList}
|
|
580
|
-
</section>`;
|
|
581
|
-
}
|
|
582
|
-
function renderArmMetrics(arm) {
|
|
583
|
-
const parts = [];
|
|
584
|
-
const turns = compareArmTurns(arm);
|
|
585
|
-
if (turns !== undefined) {
|
|
586
|
-
parts.push(formatCompareTurns(turns));
|
|
587
|
-
}
|
|
588
|
-
const tokens = formatArmTokens(arm, "total");
|
|
589
|
-
if (tokens !== "n/a") {
|
|
590
|
-
parts.push(`${tokens} tokens`);
|
|
591
|
-
}
|
|
592
|
-
const tools = armToolCount(arm);
|
|
593
|
-
if (tools !== undefined) {
|
|
594
|
-
parts.push(tools === 1 ? "1 tool" : `${formatInteger(tools)} tools`);
|
|
595
|
-
}
|
|
596
|
-
if (parts.length === 0) {
|
|
597
|
-
return "";
|
|
598
|
-
}
|
|
599
|
-
return `<p class="compare-arm-metrics">${escapeHtml(parts.join(" · "))}</p>`;
|
|
600
|
-
}
|
|
601
|
-
function renderArmFailures(arm) {
|
|
602
|
-
if (!arm.failures?.length)
|
|
603
|
-
return "";
|
|
604
|
-
return `<ul class="failures compare-arm-failures">${arm.failures
|
|
605
|
-
.map((failure) => `<li>${escapeHtml(failure.message)}</li>`)
|
|
606
|
-
.join("")}</ul>`;
|
|
607
|
-
}
|
|
608
|
-
function compareTabGroupId(result, host) {
|
|
609
|
-
return `ct-${result.suite}-${result.scenario}-${host ?? "host"}`
|
|
610
|
-
.toLowerCase()
|
|
611
|
-
.replace(/[^a-z0-9]+/g, "-")
|
|
612
|
-
.replace(/^-+|-+$/g, "");
|
|
613
|
-
}
|
|
614
|
-
function renderArmColumn(arm, index) {
|
|
615
|
-
const description = arm.description
|
|
616
|
-
? `<p class="compare-arm-description">${escapeHtml(arm.description)}</p>`
|
|
617
|
-
: "";
|
|
618
|
-
const sideClass = arm.id === "a" || arm.id === "b" ? ` compare-arm-${arm.id}` : "";
|
|
619
|
-
const kicker = arm.id === "a" ? "Arm A" : arm.id === "b" ? "Arm B" : `Arm ${arm.id}`;
|
|
620
|
-
return `
|
|
621
|
-
<article class="compare-arm${sideClass}" data-arm-id="${escapeHtml(arm.id)}" style="--arm-accent: var(--arm-${index % 4});">
|
|
622
|
-
<header class="compare-arm-header">
|
|
623
|
-
<p class="compare-arm-kicker">${escapeHtml(kicker)}</p>
|
|
624
|
-
<h3>${escapeHtml(arm.label)}</h3>
|
|
625
|
-
${description}
|
|
626
|
-
${renderArmMetrics(arm)}
|
|
627
|
-
${renderArmFailures(arm)}
|
|
628
|
-
</header>
|
|
629
|
-
${renderTraceDetails(arm.trace, arm.prompt)}
|
|
630
|
-
</article>`;
|
|
631
|
-
}
|
|
632
|
-
function renderTraceDetails(trace, prompt) {
|
|
633
|
-
const tools = trace?.toolCalls.length ?? 0;
|
|
634
|
-
const shell = trace?.shellCommands.length ?? 0;
|
|
635
|
-
const messages = trace?.messages.length ?? 0;
|
|
636
|
-
const label = `${messages} messages · ${tools} tools${shell ? ` · ${shell} shell command${shell === 1 ? "" : "s"}` : ""}`;
|
|
637
|
-
return `<details class="trace-details" open><summary>Conversation & evidence <span>${escapeHtml(label)}</span></summary>${renderChat(trace, prompt)}</details>`;
|
|
638
|
-
}
|
|
639
|
-
function renderCompareConversations(result, host) {
|
|
640
|
-
const compare = result.compare;
|
|
641
|
-
if (!compare) {
|
|
642
|
-
return `<section class="conversation">
|
|
643
|
-
<h3>Conversation</h3>
|
|
644
|
-
${renderTraceDetails(result.trace, result.prompt)}
|
|
645
|
-
</section>`;
|
|
646
|
-
}
|
|
647
|
-
const arms = compareResultArms(compare);
|
|
648
|
-
const group = compareTabGroupId(result, host);
|
|
649
|
-
const tablist = arms
|
|
650
|
-
.map((arm, index) => {
|
|
651
|
-
const id = `${group}-${arm.id}`;
|
|
652
|
-
return `<input type="radio" class="compare-tab-input" name="${escapeHtml(group)}" id="${escapeHtml(id)}"${index === 0 ? " checked" : ""}>
|
|
653
|
-
<label class="compare-tab" for="${escapeHtml(id)}">${escapeHtml(arm.label)}</label>`;
|
|
654
|
-
})
|
|
655
|
-
.join("");
|
|
656
|
-
const rules = arms
|
|
657
|
-
.map((arm) => `.${group}:has(#${group}-${arm.id}:checked) [data-arm-id="${arm.id}"]{display:grid}`)
|
|
658
|
-
.join("");
|
|
659
|
-
return `
|
|
660
|
-
<style>${rules}</style>
|
|
661
|
-
<div class="compare-layout compare-tabs ${escapeHtml(group)}">
|
|
662
|
-
<div class="compare-tablist" role="tablist">${tablist}</div>
|
|
663
|
-
<div class="compare-arms" data-arm-count="${arms.length}">
|
|
664
|
-
${arms.map((arm, index) => renderArmColumn(arm, index)).join("")}
|
|
665
|
-
</div>
|
|
666
|
-
</div>`;
|
|
667
|
-
}
|
|
668
|
-
function renderStoryCheck(check) {
|
|
669
|
-
const icon = check.status === "pass" ? "✓" : check.status === "fail" ? "✗" : "·";
|
|
670
|
-
return `<li class="story-check story-check-${check.status}"><span class="story-check-icon">${icon}</span>${escapeHtml(check.text)}</li>`;
|
|
671
|
-
}
|
|
672
|
-
function renderStorySection(section) {
|
|
673
|
-
const titled = Boolean(section.title);
|
|
674
|
-
const title = section.title ? `<h4>${escapeHtml(section.title)}</h4>` : "";
|
|
675
|
-
const description = section.description
|
|
676
|
-
? `<p class="story-section-description">${escapeHtml(section.description)}</p>`
|
|
677
|
-
: "";
|
|
678
|
-
const checks = section.checks.length > 0
|
|
679
|
-
? `<ul class="story-checks">${section.checks.map(renderStoryCheck).join("")}</ul>`
|
|
680
|
-
: "";
|
|
681
|
-
const titledClass = titled ? " story-section-titled" : "";
|
|
682
|
-
return `<div class="story-section${titledClass}">${title}${description}${checks}</div>`;
|
|
683
|
-
}
|
|
684
|
-
function renderStory(result, comparison = "") {
|
|
685
|
-
const story = result.story;
|
|
686
|
-
if (!story && !comparison) {
|
|
687
|
-
return "";
|
|
688
|
-
}
|
|
689
|
-
const sections = story?.sections?.filter((section) => section.checks.length > 0) ?? [];
|
|
690
|
-
const storyCriteria = sections.length
|
|
691
|
-
? sections.map(renderStorySection).join("")
|
|
692
|
-
: story?.criteria.length
|
|
693
|
-
? `<ul class="story-checks">${story.criteria
|
|
694
|
-
.map((text) => renderStoryCheck({ text, status: result.passed ? "pass" : "fail" }))
|
|
695
|
-
.join("")}</ul>`
|
|
696
|
-
: "";
|
|
697
|
-
const fallbackCompareCriteria = result.compare?.gateResults?.length
|
|
698
|
-
? `<div class="story-section story-section-titled"><h4>Compare</h4><ul class="story-checks">${result.compare.gateResults
|
|
699
|
-
.map((gate) => renderStoryCheck({
|
|
700
|
-
text: gate.message,
|
|
701
|
-
status: gate.passed ? "pass" : "fail",
|
|
702
|
-
}))
|
|
703
|
-
.join("")}</ul></div>`
|
|
704
|
-
: "";
|
|
705
|
-
const criteria = storyCriteria || fallbackCompareCriteria;
|
|
706
|
-
if (!criteria && !comparison)
|
|
707
|
-
return "";
|
|
708
|
-
return `<div class="story"><section class="story-criteria"><h3>Pass criteria</h3>${criteria}${comparison}</section></div>`;
|
|
709
|
-
}
|
|
710
|
-
export function renderScenarioResult(result, host) {
|
|
711
|
-
const open = result.skipped ? "" : " open";
|
|
712
|
-
const failures = renderFailures(result);
|
|
713
|
-
const judgeVerdicts = renderJudgeVerdicts(result);
|
|
714
|
-
const tokens = formatTokensBadge(result);
|
|
715
|
-
const usageDetail = result.compare ? "" : renderUsageDetail(usageOf(result));
|
|
716
|
-
const traceMeta = result.compare ? "" : renderTraceMeta(result);
|
|
717
|
-
const story = renderStory(result, renderCompareMetrics(result));
|
|
718
|
-
const diagnostics = story || failures || judgeVerdicts
|
|
719
|
-
? `<div class="diagnostics">
|
|
720
|
-
${story}
|
|
721
|
-
${failures ? `<section><h3>What went wrong</h3>${failures}</section>` : ""}
|
|
722
|
-
${judgeVerdicts ? `<section><h3>Judge verdict</h3>${judgeVerdicts}</section>` : ""}
|
|
723
|
-
</div>`
|
|
724
|
-
: "";
|
|
725
|
-
const metaRow = usageDetail || traceMeta
|
|
726
|
-
? `<div class="diagnostics meta-row">${usageDetail}${traceMeta}</div>`
|
|
727
|
-
: "";
|
|
728
|
-
const compareClass = result.compare ? " compare-scenario" : "";
|
|
729
|
-
const description = result.description
|
|
730
|
-
? `<span class="scenario-lede">${escapeHtml(result.description)}</span>`
|
|
731
|
-
: "";
|
|
732
|
-
return `
|
|
733
|
-
<details class="scenario ${statusClass(result)}${compareClass}"${open}>
|
|
734
|
-
<summary>
|
|
735
|
-
<span class="badge ${statusClass(result)}">${statusLabel(result)}</span>
|
|
736
|
-
<span class="scenario-heading">
|
|
737
|
-
<span class="scenario-name">${escapeHtml(result.scenario)}</span>
|
|
738
|
-
${description}
|
|
739
|
-
</span>
|
|
740
|
-
${tokens ? `<span class="tokens">${escapeHtml(tokens)}</span>` : ""}
|
|
741
|
-
<span class="duration">${escapeHtml(formatDuration(result.durationMs))}</span>
|
|
742
|
-
</summary>
|
|
743
|
-
<div class="scenario-body">
|
|
744
|
-
${diagnostics}
|
|
745
|
-
${metaRow}
|
|
746
|
-
${renderCompareConversations(result, host)}
|
|
747
|
-
</div>
|
|
748
|
-
</details>`;
|
|
749
|
-
}
|
|
750
|
-
export function reportCss() {
|
|
751
|
-
return sharedReportCss();
|
|
752
|
-
}
|
|
753
|
-
function sharedReportCss() {
|
|
754
|
-
return `
|
|
755
|
-
@layer reset, base, layout, components;
|
|
756
|
-
:root {
|
|
757
|
-
color-scheme: dark;
|
|
758
|
-
--bg: oklch(0.18 0.02 250);
|
|
759
|
-
--panel: oklch(0.23 0.025 250);
|
|
760
|
-
--panel-2: oklch(0.2 0.022 250);
|
|
761
|
-
--text: oklch(0.93 0.015 250);
|
|
762
|
-
--muted: oklch(0.72 0.03 250);
|
|
763
|
-
--border: oklch(0.32 0.03 250);
|
|
764
|
-
--pass: oklch(0.8 0.16 155);
|
|
765
|
-
--fail: oklch(0.72 0.16 20);
|
|
766
|
-
--skip: oklch(0.84 0.14 85);
|
|
767
|
-
--user-bubble: oklch(0.3 0.06 250);
|
|
768
|
-
--assistant-bubble: oklch(0.26 0.04 155);
|
|
769
|
-
--system-bubble: oklch(0.26 0.04 300);
|
|
770
|
-
--tool: oklch(0.82 0.12 80);
|
|
771
|
-
--tool-bubble: oklch(0.26 0.04 80);
|
|
772
|
-
--arm-a: oklch(0.72 0.12 250);
|
|
773
|
-
--arm-b: oklch(0.78 0.14 75);
|
|
774
|
-
--arm-0: var(--arm-a);
|
|
775
|
-
--arm-1: var(--arm-b);
|
|
776
|
-
--arm-2: oklch(0.76 0.13 145);
|
|
777
|
-
--arm-3: oklch(0.74 0.12 20);
|
|
778
|
-
}
|
|
779
|
-
@layer reset {
|
|
780
|
-
* { box-sizing: border-box; }
|
|
781
|
-
body { margin: 0; }
|
|
782
|
-
h1, h2, h3, h4, p, ol { margin: 0; }
|
|
783
|
-
dl, dd { margin: 0; }
|
|
784
|
-
}
|
|
785
|
-
@layer base {
|
|
786
|
-
body {
|
|
787
|
-
font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
|
|
788
|
-
background: var(--bg);
|
|
789
|
-
color: var(--text);
|
|
790
|
-
line-height: 1.5;
|
|
791
|
-
word-spacing: normal;
|
|
792
|
-
}
|
|
793
|
-
main { max-width: 68rem; margin-inline: auto; padding: 1.5rem 1.25rem 3rem; }
|
|
794
|
-
main:has(.compare-layout) { max-width: 96rem; }
|
|
795
|
-
h1 { font-size: 1.7rem; font-weight: 650; letter-spacing: -0.02em; }
|
|
796
|
-
h2 { font-size: 1.05rem; font-weight: 650; }
|
|
797
|
-
h3 { font-size: 0.8rem; color: var(--muted); font-weight: 650; }
|
|
798
|
-
h4 { font-size: 0.75rem; margin-block: 0.9rem 0.35rem; color: var(--muted); font-weight: 650; }
|
|
799
|
-
.muted { color: var(--muted); }
|
|
800
|
-
.empty { color: var(--muted); font-size: 0.85rem; font-style: italic; }
|
|
801
|
-
.empty.note { margin-bottom: 0.5rem; }
|
|
802
|
-
}
|
|
803
|
-
@layer layout {
|
|
804
|
-
.report-header { display: grid; gap: 0.25rem; margin-bottom: 1rem; }
|
|
805
|
-
.brand { color: var(--muted); font-size: 0.75rem; font-weight: 700; letter-spacing: 0.06em; text-transform: uppercase; }
|
|
806
|
-
.lede, .when { color: var(--muted); font-size: 0.85rem; }
|
|
807
|
-
.summary, .cost, .guide, .compare {
|
|
808
|
-
background: var(--panel);
|
|
809
|
-
border: 1px solid var(--border);
|
|
810
|
-
border-radius: 12px;
|
|
811
|
-
padding: 0.9rem 1rem;
|
|
812
|
-
}
|
|
813
|
-
.summary { display: grid; gap: 0.65rem; }
|
|
814
|
-
.stats { display: flex; flex-wrap: wrap; gap: 0.5rem; }
|
|
815
|
-
.suite { margin-top: 1.75rem; display: grid; gap: 0.55rem; }
|
|
816
|
-
.suite-header { display: flex; flex-wrap: wrap; align-items: center; justify-content: space-between; gap: 0.5rem 1rem; }
|
|
817
|
-
.suite-title { display: flex; align-items: center; gap: 0.5rem; }
|
|
818
|
-
.cost { margin-top: 0.85rem; display: grid; gap: 0.65rem; }
|
|
819
|
-
.cost-grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(9.5rem, 1fr)); gap: 0.75rem; }
|
|
820
|
-
.guide { margin-top: 0.85rem; }
|
|
821
|
-
.diagnostics { display: grid; grid-template-columns: minmax(0, 1fr); gap: 0.7rem; margin-bottom: 0.9rem; }
|
|
822
|
-
.meta-grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(9rem, 1fr)); gap: 0.45rem 0.75rem; }
|
|
823
|
-
}
|
|
824
|
-
@layer components {
|
|
825
|
-
.stat {
|
|
826
|
-
display: inline-flex;
|
|
827
|
-
align-items: baseline;
|
|
828
|
-
gap: 0.35rem;
|
|
829
|
-
padding: 0.4rem 0.65rem;
|
|
830
|
-
border-radius: 8px;
|
|
831
|
-
background: var(--panel-2);
|
|
832
|
-
font-size: 0.85rem;
|
|
833
|
-
}
|
|
834
|
-
.stat strong { font-size: 1.15rem; font-variant-numeric: tabular-nums; }
|
|
835
|
-
.stat-pass strong { color: var(--pass); }
|
|
836
|
-
.stat-fail strong { color: var(--fail); }
|
|
837
|
-
.stat-skip strong { color: var(--skip); }
|
|
838
|
-
.guide h2 { font-size: 0.95rem; }
|
|
839
|
-
.guide ol { margin: 0.65rem 0 0; padding-inline-start: 1.2rem; display: grid; gap: 0.35rem; color: var(--muted); font-size: 0.88rem; }
|
|
840
|
-
.cost-lede { color: var(--muted); font-size: 0.85rem; }
|
|
841
|
-
.cost-item { display: grid; gap: 0.15rem; }
|
|
842
|
-
.cost-value { font-size: 1.2rem; font-weight: 650; font-variant-numeric: tabular-nums; }
|
|
843
|
-
.cost-label { font-size: 0.8rem; font-weight: 650; }
|
|
844
|
-
.cost-detail { color: var(--muted); font-size: 0.75rem; }
|
|
845
|
-
.host {
|
|
846
|
-
color: var(--muted);
|
|
847
|
-
font-size: 0.72rem;
|
|
848
|
-
background: var(--panel-2);
|
|
849
|
-
border: 1px solid var(--border);
|
|
850
|
-
border-radius: 999px;
|
|
851
|
-
padding: 0.1rem 0.5rem;
|
|
852
|
-
}
|
|
853
|
-
.suite-counts { display: flex; flex-wrap: wrap; gap: 0.75rem; color: var(--muted); font-size: 0.85rem; }
|
|
854
|
-
.tokens { color: var(--muted); font-size: 0.78rem; font-variant-numeric: tabular-nums; }
|
|
855
|
-
.usage-summary, .scenario-meta {
|
|
856
|
-
background: var(--panel-2);
|
|
857
|
-
border: 1px solid var(--border);
|
|
858
|
-
border-radius: 8px;
|
|
859
|
-
padding: 0.65rem 0.7rem;
|
|
860
|
-
}
|
|
861
|
-
.meta-item { display: grid; gap: 0.1rem; }
|
|
862
|
-
.meta-key { color: var(--muted); font-size: 0.75rem; }
|
|
863
|
-
.meta-val { font-size: 0.88rem; font-variant-numeric: tabular-nums; word-break: break-word; }
|
|
864
|
-
.meta-row { margin-bottom: 0.7rem; }
|
|
865
|
-
details.scenario {
|
|
866
|
-
background: var(--panel);
|
|
867
|
-
border: 1px solid var(--border);
|
|
868
|
-
border-radius: 10px;
|
|
869
|
-
margin: 0.45rem 0;
|
|
870
|
-
overflow: hidden;
|
|
871
|
-
}
|
|
872
|
-
details.scenario summary {
|
|
873
|
-
cursor: pointer;
|
|
874
|
-
display: flex;
|
|
875
|
-
flex-wrap: wrap;
|
|
876
|
-
gap: 0.45rem 0.65rem;
|
|
877
|
-
align-items: center;
|
|
878
|
-
list-style: none;
|
|
879
|
-
padding: 0.55rem 0.7rem;
|
|
880
|
-
}
|
|
881
|
-
details.scenario[open] summary { border-bottom: 1px solid var(--border); background: #17202d; }
|
|
882
|
-
details.scenario summary::-webkit-details-marker { display: none; }
|
|
883
|
-
.scenario-heading { display: grid; gap: 0.1rem; min-width: 12rem; flex: 1 1 16rem; }
|
|
884
|
-
.scenario-name { font-weight: 600; font-size: 0.9rem; }
|
|
885
|
-
.scenario-lede { color: var(--muted); font-size: 0.78rem; font-weight: 400; }
|
|
886
|
-
.duration { color: var(--muted); margin-left: auto; font-variant-numeric: tabular-nums; }
|
|
887
|
-
.badge {
|
|
888
|
-
display: inline-block;
|
|
889
|
-
font-size: 0.66rem;
|
|
890
|
-
font-weight: 700;
|
|
891
|
-
text-transform: uppercase;
|
|
892
|
-
letter-spacing: 0.04em;
|
|
893
|
-
padding: 0.1rem 0.4rem;
|
|
894
|
-
border-radius: 999px;
|
|
895
|
-
border: 1px solid transparent;
|
|
896
|
-
}
|
|
897
|
-
.status-passed { color: var(--pass); border-color: color-mix(in srgb, var(--pass) 40%, transparent); }
|
|
898
|
-
.status-failed { color: var(--fail); border-color: color-mix(in srgb, var(--fail) 40%, transparent); }
|
|
899
|
-
.status-skipped { color: var(--skip); border-color: color-mix(in srgb, var(--skip) 40%, transparent); }
|
|
900
|
-
.scenario-body { padding: 0.7rem; }
|
|
901
|
-
.diagnostics { display: grid; grid-template-columns: minmax(0, 1fr); gap: 0.7rem; margin-bottom: 0.9rem; }
|
|
902
|
-
.diagnostics section { background: var(--panel-2); border: 1px solid var(--border); border-radius: 8px; padding: 0.65rem 0.7rem; min-width: 0; }
|
|
903
|
-
.conversation { min-width: 0; }
|
|
904
|
-
|
|
905
|
-
.story { margin-bottom: 0.85rem; }
|
|
906
|
-
.story-criteria { display: grid; gap: 0.55rem; }
|
|
907
|
-
.story-criteria > h3 { margin: 0; font-size: 0.78rem; text-transform: uppercase; letter-spacing: 0.04em; color: var(--muted); }
|
|
908
|
-
.story-section { display: grid; gap: 0.3rem; }
|
|
909
|
-
.story-section + .story-section { padding-block-start: 0.55rem; border-block-start: 1px solid var(--border); }
|
|
910
|
-
.story-section h4 { margin: 0; font-size: 0.84rem; color: var(--text); }
|
|
911
|
-
.story-section-description { margin: 0 0 0.4rem; color: var(--muted); font-size: 0.82rem; }
|
|
912
|
-
.story-list, .story-checks, .story-notes { margin: 0; padding-left: 1.1rem; display: flex; flex-direction: column; gap: 0.25rem; color: var(--text); }
|
|
913
|
-
.story-checks { list-style: none; padding-left: 0; }
|
|
914
|
-
.story-check { display: flex; gap: 0.4rem; }
|
|
915
|
-
.story-check-icon { width: 1rem; flex: none; font-weight: 700; }
|
|
916
|
-
.story-check-pass .story-check-icon { color: var(--pass); }
|
|
917
|
-
.story-check-fail .story-check-icon { color: var(--fail); }
|
|
918
|
-
.story-notes { color: var(--muted); }
|
|
919
|
-
.failures { list-style: none; margin: 0; padding: 0; display: flex; flex-direction: column; gap: 0.55rem; }
|
|
920
|
-
.failures li { border-left: 3px solid var(--fail); padding-left: 0.6rem; }
|
|
921
|
-
.failure-label { margin: 0; font-weight: 600; font-size: 0.85rem; color: var(--fail); }
|
|
922
|
-
.failure-hint { margin: 0.1rem 0 0; font-size: 0.78rem; color: var(--muted); }
|
|
923
|
-
.failure-message { margin: 0.3rem 0 0; font-size: 0.82rem; font-family: ui-monospace, SFMono-Regular, Menlo, monospace; white-space: pre-wrap; word-break: break-word; background: #0b1017; border: 1px solid var(--border); border-radius: 6px; padding: 0.45rem 0.55rem; }
|
|
924
|
-
.failure-evidence { margin: 0.35rem 0 0; font-size: 0.75rem; font-family: ui-monospace, SFMono-Regular, Menlo, monospace; white-space: pre-wrap; word-break: break-word; color: var(--muted); background: #0b1017; border: 1px dashed var(--border); border-radius: 6px; padding: 0.4rem 0.5rem; }
|
|
925
|
-
|
|
926
|
-
.verdicts { display: flex; flex-direction: column; gap: 0.55rem; }
|
|
927
|
-
.verdict { border-left: 3px solid var(--border); padding-left: 0.6rem; }
|
|
928
|
-
.verdict-pass { border-left-color: var(--pass); }
|
|
929
|
-
.verdict-fail { border-left-color: var(--fail); }
|
|
930
|
-
.verdict-icon { display: inline-block; width: 1.1rem; font-weight: 700; }
|
|
931
|
-
.verdict-pass .verdict-icon { color: var(--pass); }
|
|
932
|
-
.verdict-fail .verdict-icon { color: var(--fail); }
|
|
933
|
-
.question { margin: 0; font-size: 0.85rem; font-weight: 600; }
|
|
934
|
-
.rationale { margin: 0.3rem 0 0; font-size: 0.82rem; color: var(--muted); }
|
|
935
|
-
.judge-evidence { margin: 0.35rem 0 0; padding-inline-start: 1.1rem; color: var(--muted); font-size: 0.78rem; }
|
|
936
|
-
.judge-exchange { margin-block-start: 0.5rem; border: 1px solid var(--border); border-radius: 7px; overflow: hidden; }
|
|
937
|
-
.judge-exchange > summary { cursor: pointer; padding: 0.4rem 0.55rem; color: var(--muted); font-size: 0.75rem; font-weight: 650; }
|
|
938
|
-
.judge-exchange[open] > summary { border-block-end: 1px solid var(--border); background: var(--panel); }
|
|
939
|
-
.judge-chat { display: grid; gap: 0.55rem; padding: 0.55rem; }
|
|
940
|
-
.judge-turn { display: grid; gap: 0.25rem; min-width: 0; }
|
|
941
|
-
.judge-turn > span { color: var(--muted); font-size: 0.66rem; font-weight: 700; letter-spacing: 0.04em; text-transform: uppercase; }
|
|
942
|
-
.judge-turn pre { max-height: 22rem; margin: 0; padding: 0.5rem; overflow: auto; border: 1px solid var(--border); border-radius: 6px; background: #0b1017; color: var(--text); font: 0.72rem/1.45 ui-monospace, SFMono-Regular, Menlo, monospace; white-space: pre-wrap; overflow-wrap: anywhere; }
|
|
943
|
-
.judge-response pre { border-inline-start: 3px solid var(--pass); }
|
|
944
|
-
|
|
945
|
-
.chat {
|
|
946
|
-
display: flex;
|
|
947
|
-
flex-direction: column;
|
|
948
|
-
gap: 0.75rem;
|
|
949
|
-
max-width: 46rem;
|
|
950
|
-
word-spacing: normal;
|
|
951
|
-
letter-spacing: normal;
|
|
952
|
-
}
|
|
953
|
-
.chat-row { display: flex; }
|
|
954
|
-
.chat-row.side-left { justify-content: flex-start; }
|
|
955
|
-
.chat-row.side-right { justify-content: flex-end; }
|
|
956
|
-
.chat-row.side-center { justify-content: center; }
|
|
957
|
-
.bubble {
|
|
958
|
-
max-width: min(40rem, 92%);
|
|
959
|
-
border-radius: 10px;
|
|
960
|
-
padding: 0.7rem 0.9rem;
|
|
961
|
-
border: 1px solid var(--border);
|
|
962
|
-
word-spacing: normal;
|
|
963
|
-
}
|
|
964
|
-
.bubble-label {
|
|
965
|
-
font-size: 0.68rem;
|
|
966
|
-
font-weight: 700;
|
|
967
|
-
text-transform: uppercase;
|
|
968
|
-
letter-spacing: 0.04em;
|
|
969
|
-
color: var(--muted);
|
|
970
|
-
margin-bottom: 0.35rem;
|
|
971
|
-
}
|
|
972
|
-
.bubble-text {
|
|
973
|
-
white-space: pre-wrap;
|
|
974
|
-
overflow-wrap: break-word;
|
|
975
|
-
word-break: normal;
|
|
976
|
-
font-size: 0.95rem;
|
|
977
|
-
line-height: 1.5;
|
|
978
|
-
}
|
|
979
|
-
.bubble.role-user { background: var(--user-bubble); border-top-right-radius: 3px; }
|
|
980
|
-
.bubble.role-assistant { background: var(--assistant-bubble); border-top-left-radius: 3px; }
|
|
981
|
-
.bubble.role-context {
|
|
982
|
-
background: color-mix(in srgb, var(--skip) 14%, var(--panel-2));
|
|
983
|
-
max-width: min(44rem, 94%);
|
|
984
|
-
}
|
|
985
|
-
.bubble.role-system, .bubble.role-tool { background: var(--system-bubble); font-size: 0.88rem; max-width: min(42rem, 94%); }
|
|
986
|
-
|
|
987
|
-
.tool-card {
|
|
988
|
-
display: flex;
|
|
989
|
-
flex-wrap: wrap;
|
|
990
|
-
align-items: baseline;
|
|
991
|
-
gap: 0.3rem 0.55rem;
|
|
992
|
-
width: fit-content;
|
|
993
|
-
max-width: min(40rem, 100%);
|
|
994
|
-
border-radius: 7px;
|
|
995
|
-
padding: 0.38rem 0.5rem;
|
|
996
|
-
border: 1px solid color-mix(in srgb, var(--tool) 35%, var(--border));
|
|
997
|
-
background: var(--tool-bubble);
|
|
998
|
-
word-spacing: normal;
|
|
999
|
-
}
|
|
1000
|
-
.tool-card-head {
|
|
1001
|
-
display: flex;
|
|
1002
|
-
align-items: center;
|
|
1003
|
-
gap: 0.4rem;
|
|
1004
|
-
font-size: 0.74rem;
|
|
1005
|
-
font-weight: 700;
|
|
1006
|
-
color: var(--tool);
|
|
1007
|
-
}
|
|
1008
|
-
.tool-icon { display: none; }
|
|
1009
|
-
.tool-name { font-weight: 700; }
|
|
1010
|
-
.tool-args {
|
|
1011
|
-
margin-top: 0;
|
|
1012
|
-
display: grid;
|
|
1013
|
-
gap: 0.22rem;
|
|
1014
|
-
padding-top: 0;
|
|
1015
|
-
border-top: 0;
|
|
1016
|
-
}
|
|
1017
|
-
.tool-arg {
|
|
1018
|
-
display: grid;
|
|
1019
|
-
grid-template-columns: max-content minmax(0, 1fr);
|
|
1020
|
-
gap: 0.3rem 0.45rem;
|
|
1021
|
-
align-items: start;
|
|
1022
|
-
font-size: 0.72rem;
|
|
1023
|
-
}
|
|
1024
|
-
.tool-arg-key { color: var(--muted); padding-top: 0.2rem; }
|
|
1025
|
-
.tool-arg code {
|
|
1026
|
-
display: block;
|
|
1027
|
-
color: var(--text);
|
|
1028
|
-
background: #0b1017;
|
|
1029
|
-
border-radius: 6px;
|
|
1030
|
-
padding: 0.18rem 0.35rem;
|
|
1031
|
-
overflow-wrap: anywhere;
|
|
1032
|
-
word-break: break-word;
|
|
1033
|
-
white-space: pre-wrap;
|
|
1034
|
-
line-height: 1.3;
|
|
1035
|
-
}
|
|
1036
|
-
|
|
1037
|
-
.shell-commands { margin: 0; padding-left: 0; list-style: none; display: flex; flex-direction: column; gap: 0.3rem; }
|
|
1038
|
-
.shell-commands li { font-size: 0.8rem; background: #0b1017; border: 1px solid var(--border); border-radius: 6px; padding: 0.3rem 0.5rem; }
|
|
1039
|
-
code { font-family: ui-monospace, SFMono-Regular, Menlo, monospace; font-size: 0.85em; }
|
|
1040
|
-
.trace-details { margin-top: 0.75rem; border: 1px solid var(--border); border-radius: 8px; overflow: hidden; }
|
|
1041
|
-
.trace-details summary { cursor: pointer; display: flex; justify-content: space-between; gap: 0.75rem; padding: 0.55rem 0.7rem; font-size: 0.82rem; font-weight: 650; }
|
|
1042
|
-
.trace-details summary span { color: var(--muted); font-weight: 400; font-variant-numeric: tabular-nums; }
|
|
1043
|
-
.trace-details[open] summary { border-bottom: 1px solid var(--border); background: var(--panel-2); }
|
|
1044
|
-
.trace-details > .chat, .trace-details > h4, .trace-details > .shell-commands, .trace-details > .empty { margin-inline: 0.7rem; }
|
|
1045
|
-
.trace-details > .chat { margin-block: 0.7rem; }
|
|
1046
|
-
.trace-details > .shell-commands { margin-bottom: 0.7rem; }
|
|
1047
|
-
|
|
1048
|
-
.compare-layout {
|
|
1049
|
-
container-type: inline-size;
|
|
1050
|
-
display: grid;
|
|
1051
|
-
gap: 0.9rem;
|
|
1052
|
-
}
|
|
1053
|
-
.compare-tablist {
|
|
1054
|
-
display: flex;
|
|
1055
|
-
flex-wrap: wrap;
|
|
1056
|
-
gap: 0.35rem;
|
|
1057
|
-
}
|
|
1058
|
-
.compare-tab-input {
|
|
1059
|
-
position: absolute;
|
|
1060
|
-
width: 1px;
|
|
1061
|
-
height: 1px;
|
|
1062
|
-
padding: 0;
|
|
1063
|
-
margin: -1px;
|
|
1064
|
-
overflow: hidden;
|
|
1065
|
-
clip: rect(0, 0, 0, 0);
|
|
1066
|
-
white-space: nowrap;
|
|
1067
|
-
border: 0;
|
|
1068
|
-
}
|
|
1069
|
-
.compare-tab {
|
|
1070
|
-
font: inherit;
|
|
1071
|
-
border-radius: 8px;
|
|
1072
|
-
border: 1px solid var(--border);
|
|
1073
|
-
background: var(--panel-2);
|
|
1074
|
-
color: var(--text);
|
|
1075
|
-
padding: 0.35rem 0.65rem;
|
|
1076
|
-
cursor: pointer;
|
|
1077
|
-
}
|
|
1078
|
-
.compare-tab-input:checked + .compare-tab,
|
|
1079
|
-
.compare-tab[aria-selected="true"] {
|
|
1080
|
-
background: color-mix(in srgb, var(--pass) 16%, var(--panel-2));
|
|
1081
|
-
border-color: color-mix(in srgb, var(--pass) 40%, var(--border));
|
|
1082
|
-
}
|
|
1083
|
-
.compare-tabs .compare-arms,
|
|
1084
|
-
.compare-tabs .compare-arms[data-arm-count] {
|
|
1085
|
-
grid-template-columns: minmax(0, 1fr);
|
|
1086
|
-
}
|
|
1087
|
-
.compare-tabs .compare-arm { display: none; }
|
|
1088
|
-
.compare-arms {
|
|
1089
|
-
display: grid;
|
|
1090
|
-
grid-template-columns: minmax(0, 1fr) minmax(0, 1fr);
|
|
1091
|
-
gap: 0.85rem;
|
|
1092
|
-
align-items: stretch;
|
|
1093
|
-
}
|
|
1094
|
-
.compare-arms[data-arm-count="3"],
|
|
1095
|
-
.compare-arms[data-arm-count="4"] {
|
|
1096
|
-
grid-template-columns: repeat(auto-fit, minmax(16rem, 1fr));
|
|
1097
|
-
}
|
|
1098
|
-
.compare-arm {
|
|
1099
|
-
min-width: 0;
|
|
1100
|
-
display: grid;
|
|
1101
|
-
align-content: start;
|
|
1102
|
-
gap: 0.65rem;
|
|
1103
|
-
background: var(--panel-2);
|
|
1104
|
-
border: 1px solid var(--border);
|
|
1105
|
-
border-radius: 10px;
|
|
1106
|
-
padding: 0.75rem 0.8rem 0.85rem;
|
|
1107
|
-
border-block-start-width: 3px;
|
|
1108
|
-
border-block-start-color: var(--arm-accent, var(--arm-a));
|
|
1109
|
-
}
|
|
1110
|
-
.compare-arm-a { border-block-start-color: var(--arm-a); }
|
|
1111
|
-
.compare-arm-b { border-block-start-color: var(--arm-b); }
|
|
1112
|
-
.compare-arm-header { display: grid; gap: 0.15rem; padding-bottom: 0.55rem; border-bottom: 1px solid var(--border); }
|
|
1113
|
-
.compare-arm-kicker {
|
|
1114
|
-
color: var(--muted);
|
|
1115
|
-
font-size: 0.66rem;
|
|
1116
|
-
font-weight: 700;
|
|
1117
|
-
letter-spacing: 0.06em;
|
|
1118
|
-
text-transform: uppercase;
|
|
1119
|
-
}
|
|
1120
|
-
.compare-arm .compare-arm-kicker { color: var(--arm-accent, var(--muted)); }
|
|
1121
|
-
.compare-arm-a .compare-arm-kicker { color: var(--arm-a); }
|
|
1122
|
-
.compare-arm-b .compare-arm-kicker { color: var(--arm-b); }
|
|
1123
|
-
.compare-arm-header h3 { color: var(--text); font-size: 0.95rem; }
|
|
1124
|
-
.compare-arm-description { color: var(--muted); font-size: 0.8rem; }
|
|
1125
|
-
.compare-arm-metrics { color: var(--muted); font-size: 0.78rem; font-variant-numeric: tabular-nums; }
|
|
1126
|
-
.compare-arm .bubble, .compare-arm .tool-card { max-width: 100%; }
|
|
1127
|
-
|
|
1128
|
-
.compare {
|
|
1129
|
-
margin-top: 0;
|
|
1130
|
-
background: var(--panel);
|
|
1131
|
-
border: 1px solid var(--border);
|
|
1132
|
-
border-radius: 10px;
|
|
1133
|
-
padding: 0.85rem 0.9rem 1rem;
|
|
1134
|
-
}
|
|
1135
|
-
.compare-header { margin-bottom: 0.65rem; }
|
|
1136
|
-
.compare-header h3, .compare-header h4 { margin-bottom: 0.2rem; }
|
|
1137
|
-
.story-criteria > .comparison-metrics { margin-top: 0.85rem; padding: 0.8rem 0 0; background: transparent; border: 0; border-top: 1px solid var(--border); border-radius: 0; }
|
|
1138
|
-
.compare-winners { margin: 0 0 0.75rem; }
|
|
1139
|
-
.compare-winners h3 { margin-bottom: 0.35rem; }
|
|
1140
|
-
.compare-winners ul {
|
|
1141
|
-
margin: 0;
|
|
1142
|
-
padding-left: 1.1rem;
|
|
1143
|
-
display: grid;
|
|
1144
|
-
gap: 0.25rem;
|
|
1145
|
-
font-size: 0.85rem;
|
|
1146
|
-
}
|
|
1147
|
-
.compare-winner-pass { color: var(--pass); }
|
|
1148
|
-
.compare-winner-fail { color: var(--fail); }
|
|
1149
|
-
.compare-callouts {
|
|
1150
|
-
margin: 0.5rem 0 0;
|
|
1151
|
-
padding-left: 1.1rem;
|
|
1152
|
-
display: grid;
|
|
1153
|
-
gap: 0.25rem;
|
|
1154
|
-
font-size: 0.85rem;
|
|
1155
|
-
}
|
|
1156
|
-
.compare-details { margin-top: 0.65rem; color: var(--muted); font-size: 0.78rem; }
|
|
1157
|
-
.compare-table-wrap { overflow-x: auto; }
|
|
1158
|
-
.compare-table {
|
|
1159
|
-
width: 100%;
|
|
1160
|
-
border-collapse: collapse;
|
|
1161
|
-
font-size: 0.8rem;
|
|
1162
|
-
font-variant-numeric: tabular-nums;
|
|
1163
|
-
}
|
|
1164
|
-
.compare-table th, .compare-table td {
|
|
1165
|
-
border-bottom: 1px solid var(--border);
|
|
1166
|
-
padding: 0.4rem 0.45rem;
|
|
1167
|
-
text-align: left;
|
|
1168
|
-
vertical-align: middle;
|
|
1169
|
-
}
|
|
1170
|
-
.compare-table th { color: var(--muted); font-weight: 600; font-size: 0.7rem; text-transform: uppercase; letter-spacing: 0.04em; }
|
|
1171
|
-
.delta-up { color: var(--fail); }
|
|
1172
|
-
.delta-down { color: var(--pass); }
|
|
1173
|
-
.delta-flat { color: var(--muted); }
|
|
1174
|
-
.compare-table td.is-better { color: var(--pass); font-weight: 650; background: color-mix(in srgb, var(--pass) 12%, transparent); }
|
|
1175
|
-
.compare-table td.is-worse { color: var(--fail); font-weight: 650; background: color-mix(in srgb, var(--fail) 12%, transparent); }
|
|
1176
|
-
.compare-table td.is-outcome-pass { color: var(--pass); font-weight: 650; }
|
|
1177
|
-
.compare-table td.is-outcome-fail { color: var(--fail); font-weight: 650; background: color-mix(in srgb, var(--fail) 12%, transparent); }
|
|
1178
|
-
.compare-table td.is-expected-failure { color: var(--muted); font-weight: 650; background: color-mix(in srgb, var(--muted) 9%, transparent); }
|
|
1179
|
-
.is-better { color: var(--pass); font-weight: 650; }
|
|
1180
|
-
.is-worse { color: var(--fail); font-weight: 650; }
|
|
1181
|
-
|
|
1182
|
-
@container (max-width: 44rem) {
|
|
1183
|
-
.compare-arms { grid-template-columns: minmax(0, 1fr); }
|
|
1184
|
-
}
|
|
1185
|
-
|
|
1186
|
-
@media (max-width: 720px) {
|
|
1187
|
-
main { padding: 0.75rem; }
|
|
1188
|
-
.suite-header { align-items: flex-start; flex-direction: column; gap: 0.2rem; }
|
|
1189
|
-
.diagnostics { grid-template-columns: 1fr; }
|
|
1190
|
-
.bubble, .tool-card { max-width: 92%; }
|
|
1191
|
-
.compare-arms { grid-template-columns: minmax(0, 1fr); }
|
|
1192
|
-
}
|
|
1193
|
-
}
|
|
1194
|
-
`;
|
|
1195
|
-
}
|
|
1196
|
-
/** Build a self-contained HTML report for a completed suite run. */
|
|
1197
|
-
export function renderHtmlReport(reports, meta = {}) {
|
|
1198
|
-
const generatedAt = meta.generatedAt ?? new Date();
|
|
1199
|
-
const hostNames = [...new Set(reports.map((report) => report.host).filter(Boolean))];
|
|
1200
|
-
const host = meta.host ?? (hostNames.length > 0 ? hostNames.join(", ") : "unknown");
|
|
1201
|
-
const catalog = meta.catalog ?? viewerCatalogFromReports(reports, meta.suitesDir);
|
|
1202
|
-
const runId = "report";
|
|
1203
|
-
const bootstrap = {
|
|
1204
|
-
catalog,
|
|
1205
|
-
runs: [
|
|
1206
|
-
{
|
|
1207
|
-
id: runId,
|
|
1208
|
-
request: {},
|
|
1209
|
-
status: "completed",
|
|
1210
|
-
startedAt: generatedAt.toISOString(),
|
|
1211
|
-
finishedAt: generatedAt.toISOString(),
|
|
1212
|
-
reports,
|
|
1213
|
-
},
|
|
1214
|
-
],
|
|
1215
|
-
selectedRunId: runId,
|
|
1216
|
-
capabilities: { canRun: false },
|
|
1217
|
-
reportMeta: {
|
|
1218
|
-
host: String(host),
|
|
1219
|
-
suitesDir: meta.suitesDir ?? catalog.suitesDir,
|
|
1220
|
-
generatedAt: generatedAt.toISOString(),
|
|
1221
|
-
},
|
|
1222
|
-
};
|
|
1223
|
-
return renderViewerPage(bootstrap, {
|
|
1224
|
-
baseCss: sharedReportCss(),
|
|
1225
|
-
});
|
|
1226
|
-
}
|
|
1227
|
-
function viewerCatalogFromReports(reports, suitesDir = "agent-suites") {
|
|
1228
|
-
const suites = new Map();
|
|
1229
|
-
for (const report of reports) {
|
|
1230
|
-
let suite = suites.get(report.suite);
|
|
1231
|
-
if (!suite) {
|
|
1232
|
-
suite = { name: report.suite, hosts: [], scenarios: [] };
|
|
1233
|
-
suites.set(report.suite, suite);
|
|
1234
|
-
}
|
|
1235
|
-
if (!suite.hosts.includes(report.host))
|
|
1236
|
-
suite.hosts.push(report.host);
|
|
1237
|
-
for (const result of report.results) {
|
|
1238
|
-
if (suite.scenarios.some((scenario) => scenario.name === result.scenario))
|
|
1239
|
-
continue;
|
|
1240
|
-
suite.scenarios.push({
|
|
1241
|
-
name: result.scenario,
|
|
1242
|
-
description: result.description,
|
|
1243
|
-
prompt: "",
|
|
1244
|
-
rubric: {},
|
|
1245
|
-
...(result.skipped ? { skip: true } : {}),
|
|
1246
|
-
...(result.compare
|
|
1247
|
-
? {
|
|
1248
|
-
compare: compareResultArms(result.compare).map((arm) => ({
|
|
1249
|
-
id: arm.id,
|
|
1250
|
-
label: arm.label,
|
|
1251
|
-
description: arm.description,
|
|
1252
|
-
prompt: arm.prompt ?? result.prompt ?? "",
|
|
1253
|
-
rubric: {},
|
|
1254
|
-
})),
|
|
1255
|
-
gates: result.compare.gates,
|
|
1256
|
-
}
|
|
1257
|
-
: {}),
|
|
1258
|
-
});
|
|
1259
|
-
}
|
|
1260
|
-
}
|
|
1261
|
-
return { suitesDir, defaultSelectedHosts: [], suites: [...suites.values()] };
|
|
1262
|
-
}
|
|
1263
|
-
/**
|
|
1264
|
-
* Write an HTML report. Writes to `outPath` when given (parent dirs created),
|
|
1265
|
-
* otherwise a fresh temp directory. Returns the report file path.
|
|
1266
|
-
*/
|
|
1267
|
-
export async function writeHtmlReport(reports, meta = {}, outPath) {
|
|
1268
|
-
const path = outPath ?? join(await mkdtemp(join(tmpdir(), "agent-test-report-")), "report.html");
|
|
1269
|
-
if (outPath) {
|
|
1270
|
-
await mkdir(dirname(outPath), { recursive: true });
|
|
1271
|
-
}
|
|
1272
|
-
const html = renderHtmlReport(reports, meta);
|
|
1273
|
-
await writeFile(path, html, "utf8");
|
|
1274
|
-
return path;
|
|
1275
|
-
}
|
|
1276
|
-
//# sourceMappingURL=html-report.js.map
|