@post-print/agent-test 1.1.8 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (238) hide show
  1. package/README.md +18 -66
  2. package/dist/cli.d.ts +2 -64
  3. package/dist/cli.d.ts.map +1 -1
  4. package/dist/cli.js +26 -691
  5. package/dist/cli.js.map +1 -1
  6. package/dist/compare-scenario.d.ts +1 -1
  7. package/dist/compare-scenario.d.ts.map +1 -1
  8. package/dist/compare-scenario.js +72 -75
  9. package/dist/compare-scenario.js.map +1 -1
  10. package/dist/failures.d.ts +5 -1
  11. package/dist/failures.d.ts.map +1 -1
  12. package/dist/failures.js +1 -1
  13. package/dist/failures.js.map +1 -1
  14. package/dist/host-login.d.ts +1 -26
  15. package/dist/host-login.d.ts.map +1 -1
  16. package/dist/host-login.js +1 -92
  17. package/dist/host-login.js.map +1 -1
  18. package/dist/index.d.ts +6 -28
  19. package/dist/index.d.ts.map +1 -1
  20. package/dist/index.js +4 -26
  21. package/dist/index.js.map +1 -1
  22. package/dist/scenario-story.d.ts.map +1 -1
  23. package/dist/scenario-story.js +57 -39
  24. package/dist/scenario-story.js.map +1 -1
  25. package/dist/sdk/cli.d.ts +3 -0
  26. package/dist/sdk/cli.d.ts.map +1 -0
  27. package/dist/sdk/cli.js +20 -0
  28. package/dist/sdk/cli.js.map +1 -0
  29. package/dist/sdk/definitions.d.ts +22 -0
  30. package/dist/sdk/definitions.d.ts.map +1 -0
  31. package/dist/sdk/definitions.js +58 -0
  32. package/dist/sdk/definitions.js.map +1 -0
  33. package/dist/sdk/expect.d.ts +31 -0
  34. package/dist/sdk/expect.d.ts.map +1 -0
  35. package/dist/sdk/expect.js +68 -0
  36. package/dist/sdk/expect.js.map +1 -0
  37. package/dist/sdk/judge.d.ts +16 -0
  38. package/dist/sdk/judge.d.ts.map +1 -0
  39. package/dist/sdk/judge.js +82 -0
  40. package/dist/sdk/judge.js.map +1 -0
  41. package/dist/sdk/metrics.d.ts +3 -0
  42. package/dist/sdk/metrics.d.ts.map +1 -0
  43. package/dist/sdk/metrics.js +24 -0
  44. package/dist/sdk/metrics.js.map +1 -0
  45. package/dist/sdk/reporter.d.ts +12 -0
  46. package/dist/sdk/reporter.d.ts.map +1 -0
  47. package/dist/sdk/reporter.js +58 -0
  48. package/dist/sdk/reporter.js.map +1 -0
  49. package/dist/sdk/runtime.d.ts +33 -0
  50. package/dist/sdk/runtime.d.ts.map +1 -0
  51. package/dist/sdk/runtime.js +188 -0
  52. package/dist/sdk/runtime.js.map +1 -0
  53. package/dist/sdk/test.d.ts +27 -0
  54. package/dist/sdk/test.d.ts.map +1 -0
  55. package/dist/sdk/test.js +92 -0
  56. package/dist/sdk/test.js.map +1 -0
  57. package/dist/sdk/types.d.ts +75 -0
  58. package/dist/sdk/types.d.ts.map +1 -0
  59. package/dist/sdk/types.js +2 -0
  60. package/dist/sdk/types.js.map +1 -0
  61. package/dist/sdk/usage.d.ts +10 -0
  62. package/dist/sdk/usage.d.ts.map +1 -0
  63. package/dist/sdk/usage.js +13 -0
  64. package/dist/sdk/usage.js.map +1 -0
  65. package/dist/sdk/viewer-results.d.ts +43 -0
  66. package/dist/sdk/viewer-results.d.ts.map +1 -0
  67. package/dist/sdk/viewer-results.js +126 -0
  68. package/dist/sdk/viewer-results.js.map +1 -0
  69. package/dist/sdk/viewer.d.ts +11 -0
  70. package/dist/sdk/viewer.d.ts.map +1 -0
  71. package/dist/sdk/viewer.js +160 -0
  72. package/dist/sdk/viewer.js.map +1 -0
  73. package/dist/sdk/workspace.d.ts +33 -0
  74. package/dist/sdk/workspace.d.ts.map +1 -0
  75. package/dist/sdk/workspace.js +100 -0
  76. package/dist/sdk/workspace.js.map +1 -0
  77. package/dist/types.d.ts +4 -40
  78. package/dist/types.d.ts.map +1 -1
  79. package/dist/viewer/catalog.d.ts +2 -8
  80. package/dist/viewer/catalog.d.ts.map +1 -1
  81. package/dist/viewer/catalog.js +23 -128
  82. package/dist/viewer/catalog.js.map +1 -1
  83. package/dist/viewer/client.d.ts.map +1 -1
  84. package/dist/viewer/client.js +12 -12
  85. package/dist/viewer/client.js.map +1 -1
  86. package/dist/viewer/run-controller.d.ts +4 -2
  87. package/dist/viewer/run-controller.d.ts.map +1 -1
  88. package/dist/viewer/run-controller.js +297 -248
  89. package/dist/viewer/run-controller.js.map +1 -1
  90. package/dist/viewer/server.d.ts +4 -3
  91. package/dist/viewer/server.d.ts.map +1 -1
  92. package/dist/viewer/server.js +149 -112
  93. package/dist/viewer/server.js.map +1 -1
  94. package/dist/viewer/styles.d.ts +2 -0
  95. package/dist/viewer/styles.d.ts.map +1 -0
  96. package/dist/viewer/styles.js +444 -0
  97. package/dist/viewer/styles.js.map +1 -0
  98. package/dist/worker-pool.d.ts +5 -1
  99. package/dist/worker-pool.d.ts.map +1 -1
  100. package/dist/worker-pool.js +1 -1
  101. package/dist/worker-pool.js.map +1 -1
  102. package/package.json +7 -7
  103. package/dist/check.d.ts +0 -38
  104. package/dist/check.d.ts.map +0 -1
  105. package/dist/check.js +0 -169
  106. package/dist/check.js.map +0 -1
  107. package/dist/command-allowlist.d.ts +0 -9
  108. package/dist/command-allowlist.d.ts.map +0 -1
  109. package/dist/command-allowlist.js +0 -29
  110. package/dist/command-allowlist.js.map +0 -1
  111. package/dist/debug-bundle.d.ts +0 -81
  112. package/dist/debug-bundle.d.ts.map +0 -1
  113. package/dist/debug-bundle.js +0 -392
  114. package/dist/debug-bundle.js.map +0 -1
  115. package/dist/discover-suites.d.ts +0 -2
  116. package/dist/discover-suites.d.ts.map +0 -1
  117. package/dist/discover-suites.js +0 -19
  118. package/dist/discover-suites.js.map +0 -1
  119. package/dist/doctor.d.ts +0 -24
  120. package/dist/doctor.d.ts.map +0 -1
  121. package/dist/doctor.js +0 -272
  122. package/dist/doctor.js.map +0 -1
  123. package/dist/expect.d.ts +0 -59
  124. package/dist/expect.d.ts.map +0 -1
  125. package/dist/expect.js +0 -518
  126. package/dist/expect.js.map +0 -1
  127. package/dist/host-log.d.ts +0 -17
  128. package/dist/host-log.d.ts.map +0 -1
  129. package/dist/host-log.js +0 -154
  130. package/dist/host-log.js.map +0 -1
  131. package/dist/hosts.d.ts +0 -22
  132. package/dist/hosts.d.ts.map +0 -1
  133. package/dist/hosts.js +0 -84
  134. package/dist/hosts.js.map +0 -1
  135. package/dist/html-report.d.ts +0 -18
  136. package/dist/html-report.d.ts.map +0 -1
  137. package/dist/html-report.js +0 -1276
  138. package/dist/html-report.js.map +0 -1
  139. package/dist/live-isolation.d.ts +0 -82
  140. package/dist/live-isolation.d.ts.map +0 -1
  141. package/dist/live-isolation.js +0 -399
  142. package/dist/live-isolation.js.map +0 -1
  143. package/dist/live-timeout.d.ts +0 -11
  144. package/dist/live-timeout.d.ts.map +0 -1
  145. package/dist/live-timeout.js +0 -29
  146. package/dist/live-timeout.js.map +0 -1
  147. package/dist/load-adapters.d.ts +0 -13
  148. package/dist/load-adapters.d.ts.map +0 -1
  149. package/dist/load-adapters.js +0 -76
  150. package/dist/load-adapters.js.map +0 -1
  151. package/dist/load-suite.d.ts +0 -22
  152. package/dist/load-suite.d.ts.map +0 -1
  153. package/dist/load-suite.js +0 -154
  154. package/dist/load-suite.js.map +0 -1
  155. package/dist/mcp-config.d.ts +0 -13
  156. package/dist/mcp-config.d.ts.map +0 -1
  157. package/dist/mcp-config.js +0 -121
  158. package/dist/mcp-config.js.map +0 -1
  159. package/dist/preflight.d.ts +0 -3
  160. package/dist/preflight.d.ts.map +0 -1
  161. package/dist/preflight.js +0 -17
  162. package/dist/preflight.js.map +0 -1
  163. package/dist/progress.d.ts +0 -24
  164. package/dist/progress.d.ts.map +0 -1
  165. package/dist/progress.js +0 -111
  166. package/dist/progress.js.map +0 -1
  167. package/dist/record-trace.d.ts +0 -64
  168. package/dist/record-trace.d.ts.map +0 -1
  169. package/dist/record-trace.js +0 -124
  170. package/dist/record-trace.js.map +0 -1
  171. package/dist/report-preview-server.d.ts +0 -2
  172. package/dist/report-preview-server.d.ts.map +0 -1
  173. package/dist/report-preview-server.js +0 -24
  174. package/dist/report-preview-server.js.map +0 -1
  175. package/dist/report-preview.d.ts +0 -35
  176. package/dist/report-preview.d.ts.map +0 -1
  177. package/dist/report-preview.js +0 -205
  178. package/dist/report-preview.js.map +0 -1
  179. package/dist/run-suite.d.ts +0 -139
  180. package/dist/run-suite.d.ts.map +0 -1
  181. package/dist/run-suite.js +0 -1669
  182. package/dist/run-suite.js.map +0 -1
  183. package/dist/scenario-finalizer.d.ts +0 -27
  184. package/dist/scenario-finalizer.d.ts.map +0 -1
  185. package/dist/scenario-finalizer.js +0 -39
  186. package/dist/scenario-finalizer.js.map +0 -1
  187. package/dist/scenario-retry.d.ts +0 -15
  188. package/dist/scenario-retry.d.ts.map +0 -1
  189. package/dist/scenario-retry.js +0 -52
  190. package/dist/scenario-retry.js.map +0 -1
  191. package/dist/scenario-seed.d.ts +0 -13
  192. package/dist/scenario-seed.d.ts.map +0 -1
  193. package/dist/scenario-seed.js +0 -72
  194. package/dist/scenario-seed.js.map +0 -1
  195. package/dist/suite-summary.d.ts +0 -16
  196. package/dist/suite-summary.d.ts.map +0 -1
  197. package/dist/suite-summary.js +0 -218
  198. package/dist/suite-summary.js.map +0 -1
  199. package/dist/theme.d.ts +0 -96
  200. package/dist/theme.d.ts.map +0 -1
  201. package/dist/theme.js +0 -446
  202. package/dist/theme.js.map +0 -1
  203. package/dist/validate-seeds.d.ts +0 -20
  204. package/dist/validate-seeds.d.ts.map +0 -1
  205. package/dist/validate-seeds.js +0 -72
  206. package/dist/validate-seeds.js.map +0 -1
  207. package/dist/validate-suite.d.ts +0 -24
  208. package/dist/validate-suite.d.ts.map +0 -1
  209. package/dist/validate-suite.js +0 -542
  210. package/dist/validate-suite.js.map +0 -1
  211. package/dist/viewer/cli.d.ts +0 -17
  212. package/dist/viewer/cli.d.ts.map +0 -1
  213. package/dist/viewer/cli.js +0 -38
  214. package/dist/viewer/cli.js.map +0 -1
  215. package/dist/viewer/compare-metrics.d.ts +0 -30
  216. package/dist/viewer/compare-metrics.d.ts.map +0 -1
  217. package/dist/viewer/compare-metrics.js +0 -101
  218. package/dist/viewer/compare-metrics.js.map +0 -1
  219. package/dist/viewer/context-files.d.ts +0 -8
  220. package/dist/viewer/context-files.d.ts.map +0 -1
  221. package/dist/viewer/context-files.js +0 -57
  222. package/dist/viewer/context-files.js.map +0 -1
  223. package/dist/viewer/emit.d.ts +0 -7
  224. package/dist/viewer/emit.d.ts.map +0 -1
  225. package/dist/viewer/emit.js +0 -25
  226. package/dist/viewer/emit.js.map +0 -1
  227. package/dist/viewer/live-runner.d.ts +0 -27
  228. package/dist/viewer/live-runner.d.ts.map +0 -1
  229. package/dist/viewer/live-runner.js +0 -166
  230. package/dist/viewer/live-runner.js.map +0 -1
  231. package/dist/viewer/session-server.d.ts +0 -2
  232. package/dist/viewer/session-server.d.ts.map +0 -1
  233. package/dist/viewer/session-server.js +0 -71
  234. package/dist/viewer/session-server.js.map +0 -1
  235. package/dist/warnings.d.ts +0 -3
  236. package/dist/warnings.d.ts.map +0 -1
  237. package/dist/warnings.js +0 -13
  238. package/dist/warnings.js.map +0 -1
@@ -1,1276 +0,0 @@
1
- import { mkdir, mkdtemp, writeFile } from "node:fs/promises";
2
- import { tmpdir } from "node:os";
3
- import { dirname, join } from "node:path";
4
- import { compareArmTokens, compareArmTurns, compareResultArms, describeCompareOutcome, formatCompareTurns, } from "./compare-scenario.js";
5
- import { displayToolPath } from "./scenario-story.js";
6
- import { renderViewerPage } from "./viewer/page.js";
7
- function escapeHtml(value) {
8
- return value
9
- .replaceAll("&", "&")
10
- .replaceAll("<", "&lt;")
11
- .replaceAll(">", "&gt;")
12
- .replaceAll('"', "&quot;")
13
- .replaceAll("'", "&#39;");
14
- }
15
- const INTEGER_FORMAT = new Intl.NumberFormat("en-US", { maximumFractionDigits: 0 });
16
- const DECIMAL_FORMAT = new Intl.NumberFormat("en-US", {
17
- minimumFractionDigits: 1,
18
- maximumFractionDigits: 1,
19
- });
20
- function formatInteger(value) {
21
- return INTEGER_FORMAT.format(Math.round(value));
22
- }
23
- function formatDuration(ms) {
24
- if (ms < 1000) {
25
- return `${formatInteger(ms)}ms`;
26
- }
27
- return `${DECIMAL_FORMAT.format(ms / 1000)}s`;
28
- }
29
- function statusLabel(result) {
30
- if (result.skipped) {
31
- return "skipped";
32
- }
33
- return result.passed ? "passed" : "failed";
34
- }
35
- function statusClass(result) {
36
- if (result.skipped) {
37
- return "status-skipped";
38
- }
39
- return result.passed ? "status-passed" : "status-failed";
40
- }
41
- const MISSING_REQUIREMENT = {
42
- label: "Missing requirement",
43
- hint: "A required string did not appear in assistant text, commands, or tool results.",
44
- };
45
- const FORBIDDEN_BEHAVIOR = {
46
- label: "Forbidden behavior",
47
- hint: "The agent did something it was told not to do.",
48
- };
49
- const MISSING_COMMAND = {
50
- label: "Missing command",
51
- hint: "An expected command never ran.",
52
- };
53
- const FORBIDDEN_COMMAND = {
54
- label: "Forbidden command",
55
- hint: "A shell command ran that is not on the allowedCommands list.",
56
- };
57
- const MISSING_TOOL = {
58
- label: "Missing tool call",
59
- hint: "An expected tool was never invoked.",
60
- };
61
- const FORBIDDEN_TOOL = {
62
- label: "Forbidden tool call",
63
- hint: "A tool the scenario forbids was invoked.",
64
- };
65
- const MISSING_SKILL = {
66
- label: "Missing skill",
67
- hint: "The agent did not read a skill it was expected to use.",
68
- };
69
- const UNEXPECTED_SKILL = {
70
- label: "Unexpected skill",
71
- hint: "The agent read a skill it should have avoided.",
72
- };
73
- const MISSING_READ_PATH = {
74
- label: "Missing read path",
75
- hint: "Expected a Read tool call whose args mention this path fragment (registry-first / grounding).",
76
- };
77
- const FORBIDDEN_READ_PATH = {
78
- label: "Forbidden read path",
79
- hint: "A successful Read returned content from a path the scenario forbids (miss attempts do not fail).",
80
- };
81
- const ROUTING = {
82
- label: "Routing",
83
- hint: "The routing announcement did not match expectations.",
84
- };
85
- /** Human-readable label + one-line explanation for the matcher codes assertRubric/judge emit. */
86
- const MATCHER_LABELS = {
87
- runAgent: { label: "Agent run", hint: "The agent session itself failed or was cut short." },
88
- liveScenario: {
89
- label: "Isolated scenario",
90
- hint: "The scenario subprocess exited before it wrote a result.",
91
- },
92
- must: MISSING_REQUIREMENT,
93
- mustInclude: MISSING_REQUIREMENT,
94
- mustNot: FORBIDDEN_BEHAVIOR,
95
- mustNotInclude: FORBIDDEN_BEHAVIOR,
96
- mustRun: MISSING_COMMAND,
97
- toHaveRunCommand: MISSING_COMMAND,
98
- mustRunSuccessfully: MISSING_COMMAND,
99
- toHaveRunCommandSuccessfully: {
100
- label: "Command did not pass",
101
- hint: "The required command was missing, failed, or did not report a structured execution status.",
102
- },
103
- allowedCommands: FORBIDDEN_COMMAND,
104
- toHaveAllowedCommands: FORBIDDEN_COMMAND,
105
- mustCallTool: MISSING_TOOL,
106
- toHaveCalledTool: MISSING_TOOL,
107
- mustCallToolsInOrder: MISSING_TOOL,
108
- toHaveCalledToolsInOrder: MISSING_TOOL,
109
- mustNotCallTool: FORBIDDEN_TOOL,
110
- toHaveNotCalledTool: FORBIDDEN_TOOL,
111
- mustInvokeSkill: MISSING_SKILL,
112
- toHaveInvokedSkill: MISSING_SKILL,
113
- mustNotInvokeSkill: UNEXPECTED_SKILL,
114
- toHaveNotInvokedSkill: UNEXPECTED_SKILL,
115
- mustReadPath: MISSING_READ_PATH,
116
- toHaveReadPath: MISSING_READ_PATH,
117
- mustNotReadPath: FORBIDDEN_READ_PATH,
118
- toHaveNotReadPath: FORBIDDEN_READ_PATH,
119
- routingBlock: ROUTING,
120
- toIncludeRoutingBlock: ROUTING,
121
- toHaveRoutingBlockBeforeTools: ROUTING,
122
- toHaveTier: ROUTING,
123
- toHaveHandsOnTier: ROUTING,
124
- toHaveHandsOnTierBeforeTools: ROUTING,
125
- toHaveReviewDepth: {
126
- label: "Review depth",
127
- hint: "The review depth announcement did not match the rubric.",
128
- },
129
- workingTreeLeak: {
130
- label: "Working tree leak",
131
- hint: "The agent used a path outside the sealed workspace, or edited the caller checkout.",
132
- },
133
- recordTrace: { label: "Recording failed", hint: "Saving the trace to disk failed." },
134
- judge: { label: "Judge", hint: "The LLM judge flagged this scenario." },
135
- compareGate: { label: "Comparison gate", hint: "A declared comparison condition did not pass." },
136
- };
137
- function humanizeMatcher(matcher) {
138
- const [base] = matcher.split(":");
139
- const known = MATCHER_LABELS[base ?? matcher];
140
- if (known) {
141
- if (matcher.includes(":")) {
142
- const suffix = matcher.slice(matcher.indexOf(":") + 1);
143
- return { label: `${known.label} — ${suffix}`, hint: known.hint };
144
- }
145
- return known;
146
- }
147
- return { label: matcher };
148
- }
149
- const ROLE_META = {
150
- user: { label: "User", side: "right" },
151
- assistant: { label: "Agent", side: "left" },
152
- system: { label: "System", side: "center" },
153
- tool: { label: "Tool", side: "center" },
154
- };
155
- function renderMessageBubble(message) {
156
- const meta = ROLE_META[message.role] ?? { label: message.role, side: "center" };
157
- return `
158
- <div class="chat-row side-${meta.side}">
159
- <div class="bubble role-${escapeHtml(message.role)}">
160
- <div class="bubble-label">${escapeHtml(meta.label)}</div>
161
- <div class="bubble-text">${escapeHtml(message.content)}</div>
162
- </div>
163
- </div>`;
164
- }
165
- const PATH_ARG_KEYS = new Set(["path", "file_path", "filePath", "target_file", "uri", "cwd"]);
166
- function isPathArg(key, value) {
167
- return PATH_ARG_KEYS.has(key) || value.startsWith("/") || value.startsWith("file://");
168
- }
169
- /** Truncate long single-line arg values so a stray file body doesn't blow up the card. */
170
- function formatArgValue(key, value) {
171
- if (typeof value === "string" && isPathArg(key, value)) {
172
- const text = displayToolPath(value);
173
- return text === value ? { text } : { text, title: value };
174
- }
175
- const raw = typeof value === "string" ? value : (JSON.stringify(value) ?? String(value));
176
- const oneLine = raw.replaceAll("\n", " ↵ ");
177
- return { text: oneLine.length > 140 ? `${oneLine.slice(0, 140)}…` : oneLine };
178
- }
179
- function renderToolArgs(args) {
180
- const entries = args ? Object.entries(args) : [];
181
- if (entries.length === 0) {
182
- return "";
183
- }
184
- const rows = entries
185
- .map(([key, value]) => {
186
- const formatted = formatArgValue(key, value);
187
- const title = formatted.title ? ` title="${escapeHtml(formatted.title)}"` : "";
188
- return `<div class="tool-arg"><span class="tool-arg-key">${escapeHtml(key)}</span><code${title}>${escapeHtml(formatted.text)}</code></div>`;
189
- })
190
- .join("");
191
- return `<div class="tool-args">${rows}</div>`;
192
- }
193
- function renderToolCallCard(call) {
194
- return `
195
- <div class="chat-row side-left">
196
- <div class="tool-card">
197
- <div class="tool-card-head"><span class="tool-icon">&#9881;</span><span class="tool-name">${escapeHtml(call.name)}</span></div>
198
- ${renderToolArgs(call.args)}
199
- </div>
200
- </div>`;
201
- }
202
- /** Interleave messages and tool calls chronologically when the trace recorded emission order. */
203
- function buildOrderedTimeline(trace) {
204
- const items = [
205
- ...trace.messages.map((message) => message.seq === undefined
206
- ? undefined
207
- : { kind: "message", seq: message.seq, message }),
208
- ...trace.toolCalls.map((call) => call.seq === undefined ? undefined : { kind: "tool", seq: call.seq, call }),
209
- ].filter((item) => item !== undefined);
210
- const totalItems = trace.messages.length + trace.toolCalls.length;
211
- if (items.length !== totalItems || totalItems === 0) {
212
- return undefined;
213
- }
214
- return items.sort((a, b) => a.seq - b.seq);
215
- }
216
- function promptAlreadyInTrace(trace, prompt) {
217
- return (trace?.messages.some((message) => message.role === "user" && message.content.trim() === prompt) ?? false);
218
- }
219
- function renderPromptBubble(prompt, trace) {
220
- const text = prompt?.trim();
221
- if (!text || promptAlreadyInTrace(trace, text)) {
222
- return "";
223
- }
224
- return renderMessageBubble({ role: "user", content: text });
225
- }
226
- function renderChat(trace, prompt) {
227
- const promptBubble = renderPromptBubble(prompt, trace);
228
- if (!trace) {
229
- if (promptBubble) {
230
- return `<div class="chat">${promptBubble}</div>`;
231
- }
232
- return `<p class="empty">No transcript recorded for this scenario.</p>`;
233
- }
234
- const parts = [];
235
- const timeline = buildOrderedTimeline(trace);
236
- if (timeline) {
237
- parts.push(`<div class="chat">`);
238
- if (promptBubble) {
239
- parts.push(promptBubble);
240
- }
241
- for (const item of timeline) {
242
- parts.push(item.kind === "message" ? renderMessageBubble(item.message) : renderToolCallCard(item.call));
243
- }
244
- parts.push(`</div>`);
245
- }
246
- else if (trace.messages.length > 0 || trace.toolCalls.length > 0 || promptBubble) {
247
- if (!timeline && (trace.messages.length > 0 || trace.toolCalls.length > 0)) {
248
- parts.push(`<p class="empty note">Emission order wasn't recorded for this trace — messages and tool calls are shown in separate groups below.</p>`);
249
- }
250
- if (promptBubble || trace.messages.length > 0) {
251
- parts.push(`<div class="chat">`);
252
- if (promptBubble) {
253
- parts.push(promptBubble);
254
- }
255
- for (const message of trace.messages) {
256
- parts.push(renderMessageBubble(message));
257
- }
258
- parts.push(`</div>`);
259
- }
260
- if (trace.toolCalls.length > 0) {
261
- parts.push(`<h4>Tool calls</h4><div class="chat">`);
262
- for (const call of trace.toolCalls) {
263
- parts.push(renderToolCallCard(call));
264
- }
265
- parts.push(`</div>`);
266
- }
267
- }
268
- else {
269
- parts.push(`<p class="empty">No messages in transcript.</p>`);
270
- }
271
- if (trace.shellCommands.length > 0) {
272
- parts.push(`<h4>Shell commands</h4><ul class="shell-commands">`);
273
- for (const command of trace.shellCommands) {
274
- parts.push(`<li><code>${escapeHtml(command)}</code></li>`);
275
- }
276
- parts.push(`</ul>`);
277
- }
278
- return parts.join("\n");
279
- }
280
- function renderJudgeVerdicts(result) {
281
- const verdicts = result.judgeVerdicts;
282
- if (!verdicts || verdicts.length === 0) {
283
- return "";
284
- }
285
- const items = verdicts
286
- .map((verdict) => {
287
- const badge = verdict.pass ? "pass" : "fail";
288
- const icon = verdict.pass ? "✓" : "✗";
289
- const evidence = verdict.evidence?.length
290
- ? `<ul class="judge-evidence">${verdict.evidence.map((line) => `<li>${escapeHtml(line)}</li>`).join("")}</ul>`
291
- : "";
292
- const workspaceEvidence = verdict.workspaceEvidence?.length
293
- ? `<p class="judge-workspaces">Workspace evidence: ${verdict.workspaceEvidence.map(escapeHtml).join(", ")}</p>`
294
- : "";
295
- const exchange = verdict.prompt || verdict.response
296
- ? `<details class="judge-exchange"><summary>Judge conversation</summary><div class="judge-chat">
297
- ${verdict.prompt ? `<div class="judge-turn judge-input"><span>Input</span><pre>${escapeHtml(verdict.prompt)}</pre></div>` : ""}
298
- ${verdict.response ? `<div class="judge-turn judge-response"><span>Response</span><pre>${escapeHtml(verdict.response)}</pre></div>` : ""}
299
- </div></details>`
300
- : "";
301
- return `
302
- <article class="verdict verdict-${badge}">
303
- <p class="question"><span class="verdict-icon">${icon}</span>${escapeHtml(verdict.question)}</p>
304
- <p class="rationale">${escapeHtml(verdict.rationale)}</p>
305
- ${workspaceEvidence}
306
- ${evidence}
307
- ${exchange}
308
- </article>`;
309
- })
310
- .join("\n");
311
- return `<div class="verdicts">${items}</div>`;
312
- }
313
- function renderFailures(result) {
314
- if (result.failures.length === 0) {
315
- return "";
316
- }
317
- const items = result.failures
318
- .map((failure) => {
319
- const { label, hint } = humanizeMatcher(failure.matcher);
320
- const evidence = failure.evidence
321
- ? `<pre class="failure-evidence">${escapeHtml(failure.evidence)}</pre>`
322
- : "";
323
- return `
324
- <li>
325
- <p class="failure-label">${escapeHtml(label)}</p>
326
- ${hint ? `<p class="failure-hint">${escapeHtml(hint)}</p>` : ""}
327
- <p class="failure-message">${escapeHtml(failure.message)}</p>
328
- ${evidence}
329
- </li>`;
330
- })
331
- .join("\n");
332
- return `<ul class="failures">${items}</ul>`;
333
- }
334
- function usageOf(result) {
335
- return result.usage?.total ?? result.trace?.usage;
336
- }
337
- function formatTokensBadge(result) {
338
- const usage = usageOf(result);
339
- if (!usage) {
340
- return undefined;
341
- }
342
- if (typeof usage.totalTokens === "number") {
343
- return `${formatInteger(usage.totalTokens)} tokens`;
344
- }
345
- const parts = [];
346
- if (typeof usage.inputTokens === "number") {
347
- parts.push(`in ${formatInteger(usage.inputTokens)}`);
348
- }
349
- if (typeof usage.outputTokens === "number") {
350
- parts.push(`out ${formatInteger(usage.outputTokens)}`);
351
- }
352
- return parts.length > 0 ? parts.join(" · ") : undefined;
353
- }
354
- function renderUsageDetail(usage) {
355
- if (!usage) {
356
- return "";
357
- }
358
- const rows = [
359
- ["Total tokens", usage.totalTokens],
360
- ["Input", usage.inputTokens],
361
- ["Output", usage.outputTokens],
362
- ["Cache read", usage.cacheReadTokens],
363
- ["Cache write", usage.cacheWriteTokens],
364
- ["Reasoning", usage.reasoningTokens],
365
- ];
366
- const present = rows.filter(([, value]) => typeof value === "number");
367
- if (present.length === 0) {
368
- return "";
369
- }
370
- const items = present
371
- .map(([label, value]) => `<div class="meta-item"><span class="meta-key">${escapeHtml(label)}</span><span class="meta-val">${formatInteger(value ?? 0)}</span></div>`)
372
- .join("");
373
- return `<section class="scenario-meta"><h3>Token usage</h3><div class="meta-grid">${items}</div></section>`;
374
- }
375
- function renderTraceMeta(result) {
376
- const trace = result.trace;
377
- if (!trace) {
378
- return "";
379
- }
380
- const skills = trace.skillsInvoked?.length ? trace.skillsInvoked.join(", ") : "(none)";
381
- const items = [
382
- ["Messages", formatInteger(trace.messages.length)],
383
- ["Tool calls", formatInteger(trace.toolCalls.length)],
384
- ["Skills invoked", skills],
385
- ["Routing tier", trace.routing?.tier ?? "(none)"],
386
- ];
387
- if (result.attempts !== undefined && result.attempts > 1) {
388
- items.push(["Attempts", formatInteger(result.attempts)]);
389
- }
390
- const html = items
391
- .map(([label, value]) => `<div class="meta-item"><span class="meta-key">${escapeHtml(label)}</span><span class="meta-val">${escapeHtml(value)}</span></div>`)
392
- .join("");
393
- return `<section class="scenario-meta"><h3>Trace stats</h3><div class="meta-grid">${html}</div></section>`;
394
- }
395
- function formatSigned(value) {
396
- if (value === undefined) {
397
- return "n/a";
398
- }
399
- if (value > 0) {
400
- return `+${formatInteger(value)}`;
401
- }
402
- if (value < 0) {
403
- return `-${formatInteger(Math.abs(value))}`;
404
- }
405
- return formatInteger(value);
406
- }
407
- function deltaClass(value) {
408
- if (value === undefined || value === 0) {
409
- return "delta-flat";
410
- }
411
- return value > 0 ? "delta-up" : "delta-down";
412
- }
413
- function armToolCount(arm) {
414
- return arm.trace ? arm.trace.toolCalls.length : undefined;
415
- }
416
- function formatArmTurns(arm) {
417
- const turns = compareArmTurns(arm);
418
- return turns === undefined ? "n/a" : formatCompareTurns(turns);
419
- }
420
- function formatArmTokens(arm, key) {
421
- const usage = arm.trace?.usage;
422
- const value = key === "total"
423
- ? usage?.totalTokens
424
- : key === "input"
425
- ? usage?.inputTokens
426
- : usage?.outputTokens;
427
- return typeof value === "number" ? formatInteger(value) : "n/a";
428
- }
429
- function tokenDelta(a, b, key) {
430
- const pick = (arm) => {
431
- const usage = arm.trace?.usage;
432
- return key === "total"
433
- ? usage?.totalTokens
434
- : key === "input"
435
- ? usage?.inputTokens
436
- : usage?.outputTokens;
437
- };
438
- const aValue = pick(a);
439
- const bValue = pick(b);
440
- if (typeof aValue !== "number" || typeof bValue !== "number") {
441
- return undefined;
442
- }
443
- return bValue - aValue;
444
- }
445
- function winnerCellClass(delta, side) {
446
- if (delta === undefined || delta === 0) {
447
- return "";
448
- }
449
- if (side === "a") {
450
- return delta > 0 ? "is-better" : "is-worse";
451
- }
452
- return delta < 0 ? "is-better" : "is-worse";
453
- }
454
- function renderCompareMetricRow(label, aText, bText, delta, deltaText) {
455
- return `<tr>
456
- <th scope="row">${escapeHtml(label)}</th>
457
- <td class="${winnerCellClass(delta, "a")}">${escapeHtml(aText)}</td>
458
- <td class="${winnerCellClass(delta, "b")}">${escapeHtml(bText)}</td>
459
- <td class="${deltaClass(delta)}">${escapeHtml(deltaText)}</td>
460
- </tr>`;
461
- }
462
- function renderNamedCompareMetricRow(label, cells, values) {
463
- const numeric = (values ?? []).filter((value) => value !== undefined);
464
- const low = numeric.length > 1 ? Math.min(...numeric) : undefined;
465
- const high = numeric.length > 1 ? Math.max(...numeric) : undefined;
466
- return `<tr>
467
- <th scope="row">${escapeHtml(label)}</th>
468
- ${cells
469
- .map((cell, index) => {
470
- const value = values?.[index];
471
- const state = value !== undefined && low !== high
472
- ? value === low
473
- ? "is-better"
474
- : value === high
475
- ? "is-worse"
476
- : ""
477
- : "";
478
- return `<td class="${state}">${escapeHtml(cell)}</td>`;
479
- })
480
- .join("")}
481
- </tr>`;
482
- }
483
- function outcomeCell(arm, gates) {
484
- if (arm.passed === undefined)
485
- return "<td>n/a</td>";
486
- if (arm.passed)
487
- return '<td class="is-outcome-pass">pass</td>';
488
- const expected = gates?.some((gate) => !("winner" in gate) &&
489
- gate.arm === arm.id &&
490
- gate.metric === "outcome" &&
491
- gate.operator === "equal" &&
492
- gate.value === "fail");
493
- return expected
494
- ? '<td class="is-expected-failure">expected fail</td>'
495
- : '<td class="is-outcome-fail">fail</td>';
496
- }
497
- function renderOutcomeRow(arms, gates, includeDelta = false) {
498
- return `<tr><th scope="row">Outcome</th>${arms.map((arm) => outcomeCell(arm, gates)).join("")}${includeDelta ? '<td class="delta-flat">n/a</td>' : ""}</tr>`;
499
- }
500
- function renderTwoArmCompareMetrics(arms, gates) {
501
- const [left, right] = arms;
502
- const aTurns = compareArmTurns(left);
503
- const bTurns = compareArmTurns(right);
504
- const turnDelta = aTurns !== undefined && bTurns !== undefined ? bTurns - aTurns : undefined;
505
- const aTools = armToolCount(left);
506
- const bTools = armToolCount(right);
507
- const toolDelta = aTools !== undefined && bTools !== undefined ? bTools - aTools : undefined;
508
- const totalDelta = tokenDelta(left, right, "total");
509
- return `
510
- <div class="compare-table-wrap">
511
- <table class="compare-table">
512
- <thead>
513
- <tr>
514
- <th>Metric</th>
515
- <th>${escapeHtml(left.label)}</th>
516
- <th>${escapeHtml(right.label)}</th>
517
- <th>Δ</th>
518
- </tr>
519
- </thead>
520
- <tbody>
521
- ${renderOutcomeRow(arms, gates, true)}
522
- ${renderCompareMetricRow("Turns", formatArmTurns(left), formatArmTurns(right), turnDelta, formatSigned(turnDelta))}
523
- ${renderCompareMetricRow("Tokens", formatArmTokens(left, "total"), formatArmTokens(right, "total"), totalDelta, formatSigned(totalDelta))}
524
- ${renderCompareMetricRow("In", formatArmTokens(left, "input"), formatArmTokens(right, "input"), tokenDelta(left, right, "input"), formatSigned(tokenDelta(left, right, "input")))}
525
- ${renderCompareMetricRow("Out", formatArmTokens(left, "output"), formatArmTokens(right, "output"), tokenDelta(left, right, "output"), formatSigned(tokenDelta(left, right, "output")))}
526
- ${renderCompareMetricRow("Tools", aTools === undefined ? "n/a" : formatInteger(aTools), bTools === undefined ? "n/a" : formatInteger(bTools), toolDelta, formatSigned(toolDelta))}
527
- </tbody>
528
- </table>
529
- </div>`;
530
- }
531
- function renderNamedCompareMetrics(arms, gates) {
532
- return `
533
- <div class="compare-table-wrap">
534
- <table class="compare-table">
535
- <thead>
536
- <tr>
537
- <th>Metric</th>
538
- ${arms.map((arm) => `<th>${escapeHtml(arm.label)}</th>`).join("")}
539
- </tr>
540
- </thead>
541
- <tbody>
542
- ${renderOutcomeRow(arms, gates)}
543
- ${renderNamedCompareMetricRow("Turns", arms.map((arm) => formatArmTurns(arm)), arms.map(compareArmTurns))}
544
- ${renderNamedCompareMetricRow("Tokens", arms.map((arm) => formatArmTokens(arm, "total")), arms.map(compareArmTokens))}
545
- ${renderNamedCompareMetricRow("In", arms.map((arm) => formatArmTokens(arm, "input")))}
546
- ${renderNamedCompareMetricRow("Out", arms.map((arm) => formatArmTokens(arm, "output")))}
547
- ${renderNamedCompareMetricRow("Tools", arms.map((arm) => {
548
- const tools = armToolCount(arm);
549
- return tools === undefined ? "n/a" : formatInteger(tools);
550
- }), arms.map(armToolCount))}
551
- </tbody>
552
- </table>
553
- </div>`;
554
- }
555
- function renderCompareMetrics(result) {
556
- const compare = result.compare;
557
- if (!compare) {
558
- return "";
559
- }
560
- const arms = compareResultArms(compare);
561
- if (arms.length === 0) {
562
- return "";
563
- }
564
- const twoArm = arms.length === 2 && arms[0] && arms[1];
565
- const callouts = describeCompareOutcome(compare);
566
- const calloutList = callouts.length
567
- ? `<details class="compare-details"><summary>Arm details</summary><ul class="compare-callouts">${callouts.map((line) => `<li>${escapeHtml(line)}</li>`).join("")}</ul></details>`
568
- : "";
569
- const lede = twoArm
570
- ? `${arms[0].label} vs ${arms[1].label}. Lower is better: green is lowest and red is highest.`
571
- : `${arms.map((arm) => arm.label).join(", ")}. Lower is better: green is lowest and red is highest. Ties are neutral.`;
572
- return `
573
- <section class="compare comparison-metrics">
574
- <header class="compare-header">
575
- <h4>Comparison metrics</h4>
576
- <p class="muted">${escapeHtml(lede)}</p>
577
- </header>
578
- ${twoArm ? renderTwoArmCompareMetrics([arms[0], arms[1]], compare.gates) : renderNamedCompareMetrics(arms, compare.gates)}
579
- ${calloutList}
580
- </section>`;
581
- }
582
- function renderArmMetrics(arm) {
583
- const parts = [];
584
- const turns = compareArmTurns(arm);
585
- if (turns !== undefined) {
586
- parts.push(formatCompareTurns(turns));
587
- }
588
- const tokens = formatArmTokens(arm, "total");
589
- if (tokens !== "n/a") {
590
- parts.push(`${tokens} tokens`);
591
- }
592
- const tools = armToolCount(arm);
593
- if (tools !== undefined) {
594
- parts.push(tools === 1 ? "1 tool" : `${formatInteger(tools)} tools`);
595
- }
596
- if (parts.length === 0) {
597
- return "";
598
- }
599
- return `<p class="compare-arm-metrics">${escapeHtml(parts.join(" · "))}</p>`;
600
- }
601
- function renderArmFailures(arm) {
602
- if (!arm.failures?.length)
603
- return "";
604
- return `<ul class="failures compare-arm-failures">${arm.failures
605
- .map((failure) => `<li>${escapeHtml(failure.message)}</li>`)
606
- .join("")}</ul>`;
607
- }
608
- function compareTabGroupId(result, host) {
609
- return `ct-${result.suite}-${result.scenario}-${host ?? "host"}`
610
- .toLowerCase()
611
- .replace(/[^a-z0-9]+/g, "-")
612
- .replace(/^-+|-+$/g, "");
613
- }
614
- function renderArmColumn(arm, index) {
615
- const description = arm.description
616
- ? `<p class="compare-arm-description">${escapeHtml(arm.description)}</p>`
617
- : "";
618
- const sideClass = arm.id === "a" || arm.id === "b" ? ` compare-arm-${arm.id}` : "";
619
- const kicker = arm.id === "a" ? "Arm A" : arm.id === "b" ? "Arm B" : `Arm ${arm.id}`;
620
- return `
621
- <article class="compare-arm${sideClass}" data-arm-id="${escapeHtml(arm.id)}" style="--arm-accent: var(--arm-${index % 4});">
622
- <header class="compare-arm-header">
623
- <p class="compare-arm-kicker">${escapeHtml(kicker)}</p>
624
- <h3>${escapeHtml(arm.label)}</h3>
625
- ${description}
626
- ${renderArmMetrics(arm)}
627
- ${renderArmFailures(arm)}
628
- </header>
629
- ${renderTraceDetails(arm.trace, arm.prompt)}
630
- </article>`;
631
- }
632
- function renderTraceDetails(trace, prompt) {
633
- const tools = trace?.toolCalls.length ?? 0;
634
- const shell = trace?.shellCommands.length ?? 0;
635
- const messages = trace?.messages.length ?? 0;
636
- const label = `${messages} messages · ${tools} tools${shell ? ` · ${shell} shell command${shell === 1 ? "" : "s"}` : ""}`;
637
- return `<details class="trace-details" open><summary>Conversation & evidence <span>${escapeHtml(label)}</span></summary>${renderChat(trace, prompt)}</details>`;
638
- }
639
- function renderCompareConversations(result, host) {
640
- const compare = result.compare;
641
- if (!compare) {
642
- return `<section class="conversation">
643
- <h3>Conversation</h3>
644
- ${renderTraceDetails(result.trace, result.prompt)}
645
- </section>`;
646
- }
647
- const arms = compareResultArms(compare);
648
- const group = compareTabGroupId(result, host);
649
- const tablist = arms
650
- .map((arm, index) => {
651
- const id = `${group}-${arm.id}`;
652
- return `<input type="radio" class="compare-tab-input" name="${escapeHtml(group)}" id="${escapeHtml(id)}"${index === 0 ? " checked" : ""}>
653
- <label class="compare-tab" for="${escapeHtml(id)}">${escapeHtml(arm.label)}</label>`;
654
- })
655
- .join("");
656
- const rules = arms
657
- .map((arm) => `.${group}:has(#${group}-${arm.id}:checked) [data-arm-id="${arm.id}"]{display:grid}`)
658
- .join("");
659
- return `
660
- <style>${rules}</style>
661
- <div class="compare-layout compare-tabs ${escapeHtml(group)}">
662
- <div class="compare-tablist" role="tablist">${tablist}</div>
663
- <div class="compare-arms" data-arm-count="${arms.length}">
664
- ${arms.map((arm, index) => renderArmColumn(arm, index)).join("")}
665
- </div>
666
- </div>`;
667
- }
668
- function renderStoryCheck(check) {
669
- const icon = check.status === "pass" ? "✓" : check.status === "fail" ? "✗" : "·";
670
- return `<li class="story-check story-check-${check.status}"><span class="story-check-icon">${icon}</span>${escapeHtml(check.text)}</li>`;
671
- }
672
- function renderStorySection(section) {
673
- const titled = Boolean(section.title);
674
- const title = section.title ? `<h4>${escapeHtml(section.title)}</h4>` : "";
675
- const description = section.description
676
- ? `<p class="story-section-description">${escapeHtml(section.description)}</p>`
677
- : "";
678
- const checks = section.checks.length > 0
679
- ? `<ul class="story-checks">${section.checks.map(renderStoryCheck).join("")}</ul>`
680
- : "";
681
- const titledClass = titled ? " story-section-titled" : "";
682
- return `<div class="story-section${titledClass}">${title}${description}${checks}</div>`;
683
- }
684
- function renderStory(result, comparison = "") {
685
- const story = result.story;
686
- if (!story && !comparison) {
687
- return "";
688
- }
689
- const sections = story?.sections?.filter((section) => section.checks.length > 0) ?? [];
690
- const storyCriteria = sections.length
691
- ? sections.map(renderStorySection).join("")
692
- : story?.criteria.length
693
- ? `<ul class="story-checks">${story.criteria
694
- .map((text) => renderStoryCheck({ text, status: result.passed ? "pass" : "fail" }))
695
- .join("")}</ul>`
696
- : "";
697
- const fallbackCompareCriteria = result.compare?.gateResults?.length
698
- ? `<div class="story-section story-section-titled"><h4>Compare</h4><ul class="story-checks">${result.compare.gateResults
699
- .map((gate) => renderStoryCheck({
700
- text: gate.message,
701
- status: gate.passed ? "pass" : "fail",
702
- }))
703
- .join("")}</ul></div>`
704
- : "";
705
- const criteria = storyCriteria || fallbackCompareCriteria;
706
- if (!criteria && !comparison)
707
- return "";
708
- return `<div class="story"><section class="story-criteria"><h3>Pass criteria</h3>${criteria}${comparison}</section></div>`;
709
- }
710
- export function renderScenarioResult(result, host) {
711
- const open = result.skipped ? "" : " open";
712
- const failures = renderFailures(result);
713
- const judgeVerdicts = renderJudgeVerdicts(result);
714
- const tokens = formatTokensBadge(result);
715
- const usageDetail = result.compare ? "" : renderUsageDetail(usageOf(result));
716
- const traceMeta = result.compare ? "" : renderTraceMeta(result);
717
- const story = renderStory(result, renderCompareMetrics(result));
718
- const diagnostics = story || failures || judgeVerdicts
719
- ? `<div class="diagnostics">
720
- ${story}
721
- ${failures ? `<section><h3>What went wrong</h3>${failures}</section>` : ""}
722
- ${judgeVerdicts ? `<section><h3>Judge verdict</h3>${judgeVerdicts}</section>` : ""}
723
- </div>`
724
- : "";
725
- const metaRow = usageDetail || traceMeta
726
- ? `<div class="diagnostics meta-row">${usageDetail}${traceMeta}</div>`
727
- : "";
728
- const compareClass = result.compare ? " compare-scenario" : "";
729
- const description = result.description
730
- ? `<span class="scenario-lede">${escapeHtml(result.description)}</span>`
731
- : "";
732
- return `
733
- <details class="scenario ${statusClass(result)}${compareClass}"${open}>
734
- <summary>
735
- <span class="badge ${statusClass(result)}">${statusLabel(result)}</span>
736
- <span class="scenario-heading">
737
- <span class="scenario-name">${escapeHtml(result.scenario)}</span>
738
- ${description}
739
- </span>
740
- ${tokens ? `<span class="tokens">${escapeHtml(tokens)}</span>` : ""}
741
- <span class="duration">${escapeHtml(formatDuration(result.durationMs))}</span>
742
- </summary>
743
- <div class="scenario-body">
744
- ${diagnostics}
745
- ${metaRow}
746
- ${renderCompareConversations(result, host)}
747
- </div>
748
- </details>`;
749
- }
750
- export function reportCss() {
751
- return sharedReportCss();
752
- }
753
- function sharedReportCss() {
754
- return `
755
- @layer reset, base, layout, components;
756
- :root {
757
- color-scheme: dark;
758
- --bg: oklch(0.18 0.02 250);
759
- --panel: oklch(0.23 0.025 250);
760
- --panel-2: oklch(0.2 0.022 250);
761
- --text: oklch(0.93 0.015 250);
762
- --muted: oklch(0.72 0.03 250);
763
- --border: oklch(0.32 0.03 250);
764
- --pass: oklch(0.8 0.16 155);
765
- --fail: oklch(0.72 0.16 20);
766
- --skip: oklch(0.84 0.14 85);
767
- --user-bubble: oklch(0.3 0.06 250);
768
- --assistant-bubble: oklch(0.26 0.04 155);
769
- --system-bubble: oklch(0.26 0.04 300);
770
- --tool: oklch(0.82 0.12 80);
771
- --tool-bubble: oklch(0.26 0.04 80);
772
- --arm-a: oklch(0.72 0.12 250);
773
- --arm-b: oklch(0.78 0.14 75);
774
- --arm-0: var(--arm-a);
775
- --arm-1: var(--arm-b);
776
- --arm-2: oklch(0.76 0.13 145);
777
- --arm-3: oklch(0.74 0.12 20);
778
- }
779
- @layer reset {
780
- * { box-sizing: border-box; }
781
- body { margin: 0; }
782
- h1, h2, h3, h4, p, ol { margin: 0; }
783
- dl, dd { margin: 0; }
784
- }
785
- @layer base {
786
- body {
787
- font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
788
- background: var(--bg);
789
- color: var(--text);
790
- line-height: 1.5;
791
- word-spacing: normal;
792
- }
793
- main { max-width: 68rem; margin-inline: auto; padding: 1.5rem 1.25rem 3rem; }
794
- main:has(.compare-layout) { max-width: 96rem; }
795
- h1 { font-size: 1.7rem; font-weight: 650; letter-spacing: -0.02em; }
796
- h2 { font-size: 1.05rem; font-weight: 650; }
797
- h3 { font-size: 0.8rem; color: var(--muted); font-weight: 650; }
798
- h4 { font-size: 0.75rem; margin-block: 0.9rem 0.35rem; color: var(--muted); font-weight: 650; }
799
- .muted { color: var(--muted); }
800
- .empty { color: var(--muted); font-size: 0.85rem; font-style: italic; }
801
- .empty.note { margin-bottom: 0.5rem; }
802
- }
803
- @layer layout {
804
- .report-header { display: grid; gap: 0.25rem; margin-bottom: 1rem; }
805
- .brand { color: var(--muted); font-size: 0.75rem; font-weight: 700; letter-spacing: 0.06em; text-transform: uppercase; }
806
- .lede, .when { color: var(--muted); font-size: 0.85rem; }
807
- .summary, .cost, .guide, .compare {
808
- background: var(--panel);
809
- border: 1px solid var(--border);
810
- border-radius: 12px;
811
- padding: 0.9rem 1rem;
812
- }
813
- .summary { display: grid; gap: 0.65rem; }
814
- .stats { display: flex; flex-wrap: wrap; gap: 0.5rem; }
815
- .suite { margin-top: 1.75rem; display: grid; gap: 0.55rem; }
816
- .suite-header { display: flex; flex-wrap: wrap; align-items: center; justify-content: space-between; gap: 0.5rem 1rem; }
817
- .suite-title { display: flex; align-items: center; gap: 0.5rem; }
818
- .cost { margin-top: 0.85rem; display: grid; gap: 0.65rem; }
819
- .cost-grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(9.5rem, 1fr)); gap: 0.75rem; }
820
- .guide { margin-top: 0.85rem; }
821
- .diagnostics { display: grid; grid-template-columns: minmax(0, 1fr); gap: 0.7rem; margin-bottom: 0.9rem; }
822
- .meta-grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(9rem, 1fr)); gap: 0.45rem 0.75rem; }
823
- }
824
- @layer components {
825
- .stat {
826
- display: inline-flex;
827
- align-items: baseline;
828
- gap: 0.35rem;
829
- padding: 0.4rem 0.65rem;
830
- border-radius: 8px;
831
- background: var(--panel-2);
832
- font-size: 0.85rem;
833
- }
834
- .stat strong { font-size: 1.15rem; font-variant-numeric: tabular-nums; }
835
- .stat-pass strong { color: var(--pass); }
836
- .stat-fail strong { color: var(--fail); }
837
- .stat-skip strong { color: var(--skip); }
838
- .guide h2 { font-size: 0.95rem; }
839
- .guide ol { margin: 0.65rem 0 0; padding-inline-start: 1.2rem; display: grid; gap: 0.35rem; color: var(--muted); font-size: 0.88rem; }
840
- .cost-lede { color: var(--muted); font-size: 0.85rem; }
841
- .cost-item { display: grid; gap: 0.15rem; }
842
- .cost-value { font-size: 1.2rem; font-weight: 650; font-variant-numeric: tabular-nums; }
843
- .cost-label { font-size: 0.8rem; font-weight: 650; }
844
- .cost-detail { color: var(--muted); font-size: 0.75rem; }
845
- .host {
846
- color: var(--muted);
847
- font-size: 0.72rem;
848
- background: var(--panel-2);
849
- border: 1px solid var(--border);
850
- border-radius: 999px;
851
- padding: 0.1rem 0.5rem;
852
- }
853
- .suite-counts { display: flex; flex-wrap: wrap; gap: 0.75rem; color: var(--muted); font-size: 0.85rem; }
854
- .tokens { color: var(--muted); font-size: 0.78rem; font-variant-numeric: tabular-nums; }
855
- .usage-summary, .scenario-meta {
856
- background: var(--panel-2);
857
- border: 1px solid var(--border);
858
- border-radius: 8px;
859
- padding: 0.65rem 0.7rem;
860
- }
861
- .meta-item { display: grid; gap: 0.1rem; }
862
- .meta-key { color: var(--muted); font-size: 0.75rem; }
863
- .meta-val { font-size: 0.88rem; font-variant-numeric: tabular-nums; word-break: break-word; }
864
- .meta-row { margin-bottom: 0.7rem; }
865
- details.scenario {
866
- background: var(--panel);
867
- border: 1px solid var(--border);
868
- border-radius: 10px;
869
- margin: 0.45rem 0;
870
- overflow: hidden;
871
- }
872
- details.scenario summary {
873
- cursor: pointer;
874
- display: flex;
875
- flex-wrap: wrap;
876
- gap: 0.45rem 0.65rem;
877
- align-items: center;
878
- list-style: none;
879
- padding: 0.55rem 0.7rem;
880
- }
881
- details.scenario[open] summary { border-bottom: 1px solid var(--border); background: #17202d; }
882
- details.scenario summary::-webkit-details-marker { display: none; }
883
- .scenario-heading { display: grid; gap: 0.1rem; min-width: 12rem; flex: 1 1 16rem; }
884
- .scenario-name { font-weight: 600; font-size: 0.9rem; }
885
- .scenario-lede { color: var(--muted); font-size: 0.78rem; font-weight: 400; }
886
- .duration { color: var(--muted); margin-left: auto; font-variant-numeric: tabular-nums; }
887
- .badge {
888
- display: inline-block;
889
- font-size: 0.66rem;
890
- font-weight: 700;
891
- text-transform: uppercase;
892
- letter-spacing: 0.04em;
893
- padding: 0.1rem 0.4rem;
894
- border-radius: 999px;
895
- border: 1px solid transparent;
896
- }
897
- .status-passed { color: var(--pass); border-color: color-mix(in srgb, var(--pass) 40%, transparent); }
898
- .status-failed { color: var(--fail); border-color: color-mix(in srgb, var(--fail) 40%, transparent); }
899
- .status-skipped { color: var(--skip); border-color: color-mix(in srgb, var(--skip) 40%, transparent); }
900
- .scenario-body { padding: 0.7rem; }
901
- .diagnostics { display: grid; grid-template-columns: minmax(0, 1fr); gap: 0.7rem; margin-bottom: 0.9rem; }
902
- .diagnostics section { background: var(--panel-2); border: 1px solid var(--border); border-radius: 8px; padding: 0.65rem 0.7rem; min-width: 0; }
903
- .conversation { min-width: 0; }
904
-
905
- .story { margin-bottom: 0.85rem; }
906
- .story-criteria { display: grid; gap: 0.55rem; }
907
- .story-criteria > h3 { margin: 0; font-size: 0.78rem; text-transform: uppercase; letter-spacing: 0.04em; color: var(--muted); }
908
- .story-section { display: grid; gap: 0.3rem; }
909
- .story-section + .story-section { padding-block-start: 0.55rem; border-block-start: 1px solid var(--border); }
910
- .story-section h4 { margin: 0; font-size: 0.84rem; color: var(--text); }
911
- .story-section-description { margin: 0 0 0.4rem; color: var(--muted); font-size: 0.82rem; }
912
- .story-list, .story-checks, .story-notes { margin: 0; padding-left: 1.1rem; display: flex; flex-direction: column; gap: 0.25rem; color: var(--text); }
913
- .story-checks { list-style: none; padding-left: 0; }
914
- .story-check { display: flex; gap: 0.4rem; }
915
- .story-check-icon { width: 1rem; flex: none; font-weight: 700; }
916
- .story-check-pass .story-check-icon { color: var(--pass); }
917
- .story-check-fail .story-check-icon { color: var(--fail); }
918
- .story-notes { color: var(--muted); }
919
- .failures { list-style: none; margin: 0; padding: 0; display: flex; flex-direction: column; gap: 0.55rem; }
920
- .failures li { border-left: 3px solid var(--fail); padding-left: 0.6rem; }
921
- .failure-label { margin: 0; font-weight: 600; font-size: 0.85rem; color: var(--fail); }
922
- .failure-hint { margin: 0.1rem 0 0; font-size: 0.78rem; color: var(--muted); }
923
- .failure-message { margin: 0.3rem 0 0; font-size: 0.82rem; font-family: ui-monospace, SFMono-Regular, Menlo, monospace; white-space: pre-wrap; word-break: break-word; background: #0b1017; border: 1px solid var(--border); border-radius: 6px; padding: 0.45rem 0.55rem; }
924
- .failure-evidence { margin: 0.35rem 0 0; font-size: 0.75rem; font-family: ui-monospace, SFMono-Regular, Menlo, monospace; white-space: pre-wrap; word-break: break-word; color: var(--muted); background: #0b1017; border: 1px dashed var(--border); border-radius: 6px; padding: 0.4rem 0.5rem; }
925
-
926
- .verdicts { display: flex; flex-direction: column; gap: 0.55rem; }
927
- .verdict { border-left: 3px solid var(--border); padding-left: 0.6rem; }
928
- .verdict-pass { border-left-color: var(--pass); }
929
- .verdict-fail { border-left-color: var(--fail); }
930
- .verdict-icon { display: inline-block; width: 1.1rem; font-weight: 700; }
931
- .verdict-pass .verdict-icon { color: var(--pass); }
932
- .verdict-fail .verdict-icon { color: var(--fail); }
933
- .question { margin: 0; font-size: 0.85rem; font-weight: 600; }
934
- .rationale { margin: 0.3rem 0 0; font-size: 0.82rem; color: var(--muted); }
935
- .judge-evidence { margin: 0.35rem 0 0; padding-inline-start: 1.1rem; color: var(--muted); font-size: 0.78rem; }
936
- .judge-exchange { margin-block-start: 0.5rem; border: 1px solid var(--border); border-radius: 7px; overflow: hidden; }
937
- .judge-exchange > summary { cursor: pointer; padding: 0.4rem 0.55rem; color: var(--muted); font-size: 0.75rem; font-weight: 650; }
938
- .judge-exchange[open] > summary { border-block-end: 1px solid var(--border); background: var(--panel); }
939
- .judge-chat { display: grid; gap: 0.55rem; padding: 0.55rem; }
940
- .judge-turn { display: grid; gap: 0.25rem; min-width: 0; }
941
- .judge-turn > span { color: var(--muted); font-size: 0.66rem; font-weight: 700; letter-spacing: 0.04em; text-transform: uppercase; }
942
- .judge-turn pre { max-height: 22rem; margin: 0; padding: 0.5rem; overflow: auto; border: 1px solid var(--border); border-radius: 6px; background: #0b1017; color: var(--text); font: 0.72rem/1.45 ui-monospace, SFMono-Regular, Menlo, monospace; white-space: pre-wrap; overflow-wrap: anywhere; }
943
- .judge-response pre { border-inline-start: 3px solid var(--pass); }
944
-
945
- .chat {
946
- display: flex;
947
- flex-direction: column;
948
- gap: 0.75rem;
949
- max-width: 46rem;
950
- word-spacing: normal;
951
- letter-spacing: normal;
952
- }
953
- .chat-row { display: flex; }
954
- .chat-row.side-left { justify-content: flex-start; }
955
- .chat-row.side-right { justify-content: flex-end; }
956
- .chat-row.side-center { justify-content: center; }
957
- .bubble {
958
- max-width: min(40rem, 92%);
959
- border-radius: 10px;
960
- padding: 0.7rem 0.9rem;
961
- border: 1px solid var(--border);
962
- word-spacing: normal;
963
- }
964
- .bubble-label {
965
- font-size: 0.68rem;
966
- font-weight: 700;
967
- text-transform: uppercase;
968
- letter-spacing: 0.04em;
969
- color: var(--muted);
970
- margin-bottom: 0.35rem;
971
- }
972
- .bubble-text {
973
- white-space: pre-wrap;
974
- overflow-wrap: break-word;
975
- word-break: normal;
976
- font-size: 0.95rem;
977
- line-height: 1.5;
978
- }
979
- .bubble.role-user { background: var(--user-bubble); border-top-right-radius: 3px; }
980
- .bubble.role-assistant { background: var(--assistant-bubble); border-top-left-radius: 3px; }
981
- .bubble.role-context {
982
- background: color-mix(in srgb, var(--skip) 14%, var(--panel-2));
983
- max-width: min(44rem, 94%);
984
- }
985
- .bubble.role-system, .bubble.role-tool { background: var(--system-bubble); font-size: 0.88rem; max-width: min(42rem, 94%); }
986
-
987
- .tool-card {
988
- display: flex;
989
- flex-wrap: wrap;
990
- align-items: baseline;
991
- gap: 0.3rem 0.55rem;
992
- width: fit-content;
993
- max-width: min(40rem, 100%);
994
- border-radius: 7px;
995
- padding: 0.38rem 0.5rem;
996
- border: 1px solid color-mix(in srgb, var(--tool) 35%, var(--border));
997
- background: var(--tool-bubble);
998
- word-spacing: normal;
999
- }
1000
- .tool-card-head {
1001
- display: flex;
1002
- align-items: center;
1003
- gap: 0.4rem;
1004
- font-size: 0.74rem;
1005
- font-weight: 700;
1006
- color: var(--tool);
1007
- }
1008
- .tool-icon { display: none; }
1009
- .tool-name { font-weight: 700; }
1010
- .tool-args {
1011
- margin-top: 0;
1012
- display: grid;
1013
- gap: 0.22rem;
1014
- padding-top: 0;
1015
- border-top: 0;
1016
- }
1017
- .tool-arg {
1018
- display: grid;
1019
- grid-template-columns: max-content minmax(0, 1fr);
1020
- gap: 0.3rem 0.45rem;
1021
- align-items: start;
1022
- font-size: 0.72rem;
1023
- }
1024
- .tool-arg-key { color: var(--muted); padding-top: 0.2rem; }
1025
- .tool-arg code {
1026
- display: block;
1027
- color: var(--text);
1028
- background: #0b1017;
1029
- border-radius: 6px;
1030
- padding: 0.18rem 0.35rem;
1031
- overflow-wrap: anywhere;
1032
- word-break: break-word;
1033
- white-space: pre-wrap;
1034
- line-height: 1.3;
1035
- }
1036
-
1037
- .shell-commands { margin: 0; padding-left: 0; list-style: none; display: flex; flex-direction: column; gap: 0.3rem; }
1038
- .shell-commands li { font-size: 0.8rem; background: #0b1017; border: 1px solid var(--border); border-radius: 6px; padding: 0.3rem 0.5rem; }
1039
- code { font-family: ui-monospace, SFMono-Regular, Menlo, monospace; font-size: 0.85em; }
1040
- .trace-details { margin-top: 0.75rem; border: 1px solid var(--border); border-radius: 8px; overflow: hidden; }
1041
- .trace-details summary { cursor: pointer; display: flex; justify-content: space-between; gap: 0.75rem; padding: 0.55rem 0.7rem; font-size: 0.82rem; font-weight: 650; }
1042
- .trace-details summary span { color: var(--muted); font-weight: 400; font-variant-numeric: tabular-nums; }
1043
- .trace-details[open] summary { border-bottom: 1px solid var(--border); background: var(--panel-2); }
1044
- .trace-details > .chat, .trace-details > h4, .trace-details > .shell-commands, .trace-details > .empty { margin-inline: 0.7rem; }
1045
- .trace-details > .chat { margin-block: 0.7rem; }
1046
- .trace-details > .shell-commands { margin-bottom: 0.7rem; }
1047
-
1048
- .compare-layout {
1049
- container-type: inline-size;
1050
- display: grid;
1051
- gap: 0.9rem;
1052
- }
1053
- .compare-tablist {
1054
- display: flex;
1055
- flex-wrap: wrap;
1056
- gap: 0.35rem;
1057
- }
1058
- .compare-tab-input {
1059
- position: absolute;
1060
- width: 1px;
1061
- height: 1px;
1062
- padding: 0;
1063
- margin: -1px;
1064
- overflow: hidden;
1065
- clip: rect(0, 0, 0, 0);
1066
- white-space: nowrap;
1067
- border: 0;
1068
- }
1069
- .compare-tab {
1070
- font: inherit;
1071
- border-radius: 8px;
1072
- border: 1px solid var(--border);
1073
- background: var(--panel-2);
1074
- color: var(--text);
1075
- padding: 0.35rem 0.65rem;
1076
- cursor: pointer;
1077
- }
1078
- .compare-tab-input:checked + .compare-tab,
1079
- .compare-tab[aria-selected="true"] {
1080
- background: color-mix(in srgb, var(--pass) 16%, var(--panel-2));
1081
- border-color: color-mix(in srgb, var(--pass) 40%, var(--border));
1082
- }
1083
- .compare-tabs .compare-arms,
1084
- .compare-tabs .compare-arms[data-arm-count] {
1085
- grid-template-columns: minmax(0, 1fr);
1086
- }
1087
- .compare-tabs .compare-arm { display: none; }
1088
- .compare-arms {
1089
- display: grid;
1090
- grid-template-columns: minmax(0, 1fr) minmax(0, 1fr);
1091
- gap: 0.85rem;
1092
- align-items: stretch;
1093
- }
1094
- .compare-arms[data-arm-count="3"],
1095
- .compare-arms[data-arm-count="4"] {
1096
- grid-template-columns: repeat(auto-fit, minmax(16rem, 1fr));
1097
- }
1098
- .compare-arm {
1099
- min-width: 0;
1100
- display: grid;
1101
- align-content: start;
1102
- gap: 0.65rem;
1103
- background: var(--panel-2);
1104
- border: 1px solid var(--border);
1105
- border-radius: 10px;
1106
- padding: 0.75rem 0.8rem 0.85rem;
1107
- border-block-start-width: 3px;
1108
- border-block-start-color: var(--arm-accent, var(--arm-a));
1109
- }
1110
- .compare-arm-a { border-block-start-color: var(--arm-a); }
1111
- .compare-arm-b { border-block-start-color: var(--arm-b); }
1112
- .compare-arm-header { display: grid; gap: 0.15rem; padding-bottom: 0.55rem; border-bottom: 1px solid var(--border); }
1113
- .compare-arm-kicker {
1114
- color: var(--muted);
1115
- font-size: 0.66rem;
1116
- font-weight: 700;
1117
- letter-spacing: 0.06em;
1118
- text-transform: uppercase;
1119
- }
1120
- .compare-arm .compare-arm-kicker { color: var(--arm-accent, var(--muted)); }
1121
- .compare-arm-a .compare-arm-kicker { color: var(--arm-a); }
1122
- .compare-arm-b .compare-arm-kicker { color: var(--arm-b); }
1123
- .compare-arm-header h3 { color: var(--text); font-size: 0.95rem; }
1124
- .compare-arm-description { color: var(--muted); font-size: 0.8rem; }
1125
- .compare-arm-metrics { color: var(--muted); font-size: 0.78rem; font-variant-numeric: tabular-nums; }
1126
- .compare-arm .bubble, .compare-arm .tool-card { max-width: 100%; }
1127
-
1128
- .compare {
1129
- margin-top: 0;
1130
- background: var(--panel);
1131
- border: 1px solid var(--border);
1132
- border-radius: 10px;
1133
- padding: 0.85rem 0.9rem 1rem;
1134
- }
1135
- .compare-header { margin-bottom: 0.65rem; }
1136
- .compare-header h3, .compare-header h4 { margin-bottom: 0.2rem; }
1137
- .story-criteria > .comparison-metrics { margin-top: 0.85rem; padding: 0.8rem 0 0; background: transparent; border: 0; border-top: 1px solid var(--border); border-radius: 0; }
1138
- .compare-winners { margin: 0 0 0.75rem; }
1139
- .compare-winners h3 { margin-bottom: 0.35rem; }
1140
- .compare-winners ul {
1141
- margin: 0;
1142
- padding-left: 1.1rem;
1143
- display: grid;
1144
- gap: 0.25rem;
1145
- font-size: 0.85rem;
1146
- }
1147
- .compare-winner-pass { color: var(--pass); }
1148
- .compare-winner-fail { color: var(--fail); }
1149
- .compare-callouts {
1150
- margin: 0.5rem 0 0;
1151
- padding-left: 1.1rem;
1152
- display: grid;
1153
- gap: 0.25rem;
1154
- font-size: 0.85rem;
1155
- }
1156
- .compare-details { margin-top: 0.65rem; color: var(--muted); font-size: 0.78rem; }
1157
- .compare-table-wrap { overflow-x: auto; }
1158
- .compare-table {
1159
- width: 100%;
1160
- border-collapse: collapse;
1161
- font-size: 0.8rem;
1162
- font-variant-numeric: tabular-nums;
1163
- }
1164
- .compare-table th, .compare-table td {
1165
- border-bottom: 1px solid var(--border);
1166
- padding: 0.4rem 0.45rem;
1167
- text-align: left;
1168
- vertical-align: middle;
1169
- }
1170
- .compare-table th { color: var(--muted); font-weight: 600; font-size: 0.7rem; text-transform: uppercase; letter-spacing: 0.04em; }
1171
- .delta-up { color: var(--fail); }
1172
- .delta-down { color: var(--pass); }
1173
- .delta-flat { color: var(--muted); }
1174
- .compare-table td.is-better { color: var(--pass); font-weight: 650; background: color-mix(in srgb, var(--pass) 12%, transparent); }
1175
- .compare-table td.is-worse { color: var(--fail); font-weight: 650; background: color-mix(in srgb, var(--fail) 12%, transparent); }
1176
- .compare-table td.is-outcome-pass { color: var(--pass); font-weight: 650; }
1177
- .compare-table td.is-outcome-fail { color: var(--fail); font-weight: 650; background: color-mix(in srgb, var(--fail) 12%, transparent); }
1178
- .compare-table td.is-expected-failure { color: var(--muted); font-weight: 650; background: color-mix(in srgb, var(--muted) 9%, transparent); }
1179
- .is-better { color: var(--pass); font-weight: 650; }
1180
- .is-worse { color: var(--fail); font-weight: 650; }
1181
-
1182
- @container (max-width: 44rem) {
1183
- .compare-arms { grid-template-columns: minmax(0, 1fr); }
1184
- }
1185
-
1186
- @media (max-width: 720px) {
1187
- main { padding: 0.75rem; }
1188
- .suite-header { align-items: flex-start; flex-direction: column; gap: 0.2rem; }
1189
- .diagnostics { grid-template-columns: 1fr; }
1190
- .bubble, .tool-card { max-width: 92%; }
1191
- .compare-arms { grid-template-columns: minmax(0, 1fr); }
1192
- }
1193
- }
1194
- `;
1195
- }
1196
- /** Build a self-contained HTML report for a completed suite run. */
1197
- export function renderHtmlReport(reports, meta = {}) {
1198
- const generatedAt = meta.generatedAt ?? new Date();
1199
- const hostNames = [...new Set(reports.map((report) => report.host).filter(Boolean))];
1200
- const host = meta.host ?? (hostNames.length > 0 ? hostNames.join(", ") : "unknown");
1201
- const catalog = meta.catalog ?? viewerCatalogFromReports(reports, meta.suitesDir);
1202
- const runId = "report";
1203
- const bootstrap = {
1204
- catalog,
1205
- runs: [
1206
- {
1207
- id: runId,
1208
- request: {},
1209
- status: "completed",
1210
- startedAt: generatedAt.toISOString(),
1211
- finishedAt: generatedAt.toISOString(),
1212
- reports,
1213
- },
1214
- ],
1215
- selectedRunId: runId,
1216
- capabilities: { canRun: false },
1217
- reportMeta: {
1218
- host: String(host),
1219
- suitesDir: meta.suitesDir ?? catalog.suitesDir,
1220
- generatedAt: generatedAt.toISOString(),
1221
- },
1222
- };
1223
- return renderViewerPage(bootstrap, {
1224
- baseCss: sharedReportCss(),
1225
- });
1226
- }
1227
- function viewerCatalogFromReports(reports, suitesDir = "agent-suites") {
1228
- const suites = new Map();
1229
- for (const report of reports) {
1230
- let suite = suites.get(report.suite);
1231
- if (!suite) {
1232
- suite = { name: report.suite, hosts: [], scenarios: [] };
1233
- suites.set(report.suite, suite);
1234
- }
1235
- if (!suite.hosts.includes(report.host))
1236
- suite.hosts.push(report.host);
1237
- for (const result of report.results) {
1238
- if (suite.scenarios.some((scenario) => scenario.name === result.scenario))
1239
- continue;
1240
- suite.scenarios.push({
1241
- name: result.scenario,
1242
- description: result.description,
1243
- prompt: "",
1244
- rubric: {},
1245
- ...(result.skipped ? { skip: true } : {}),
1246
- ...(result.compare
1247
- ? {
1248
- compare: compareResultArms(result.compare).map((arm) => ({
1249
- id: arm.id,
1250
- label: arm.label,
1251
- description: arm.description,
1252
- prompt: arm.prompt ?? result.prompt ?? "",
1253
- rubric: {},
1254
- })),
1255
- gates: result.compare.gates,
1256
- }
1257
- : {}),
1258
- });
1259
- }
1260
- }
1261
- return { suitesDir, defaultSelectedHosts: [], suites: [...suites.values()] };
1262
- }
1263
- /**
1264
- * Write an HTML report. Writes to `outPath` when given (parent dirs created),
1265
- * otherwise a fresh temp directory. Returns the report file path.
1266
- */
1267
- export async function writeHtmlReport(reports, meta = {}, outPath) {
1268
- const path = outPath ?? join(await mkdtemp(join(tmpdir(), "agent-test-report-")), "report.html");
1269
- if (outPath) {
1270
- await mkdir(dirname(outPath), { recursive: true });
1271
- }
1272
- const html = renderHtmlReport(reports, meta);
1273
- await writeFile(path, html, "utf8");
1274
- return path;
1275
- }
1276
- //# sourceMappingURL=html-report.js.map