@post-print/agent-test 0.3.13 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. package/README.md +17 -4
  2. package/dist/cli.js +2 -2
  3. package/dist/cli.js.map +1 -1
  4. package/dist/compare-scenario.d.ts +13 -19
  5. package/dist/compare-scenario.d.ts.map +1 -1
  6. package/dist/compare-scenario.js +141 -261
  7. package/dist/compare-scenario.js.map +1 -1
  8. package/dist/debug-bundle.d.ts.map +1 -1
  9. package/dist/debug-bundle.js +2 -0
  10. package/dist/debug-bundle.js.map +1 -1
  11. package/dist/expect.d.ts +1 -0
  12. package/dist/expect.d.ts.map +1 -1
  13. package/dist/expect.js +19 -0
  14. package/dist/expect.js.map +1 -1
  15. package/dist/html-report.d.ts.map +1 -1
  16. package/dist/html-report.js +185 -38
  17. package/dist/html-report.js.map +1 -1
  18. package/dist/index.d.ts +2 -2
  19. package/dist/index.d.ts.map +1 -1
  20. package/dist/index.js +1 -1
  21. package/dist/index.js.map +1 -1
  22. package/dist/live-isolation.d.ts +2 -0
  23. package/dist/live-isolation.d.ts.map +1 -1
  24. package/dist/live-isolation.js +66 -6
  25. package/dist/live-isolation.js.map +1 -1
  26. package/dist/run-suite.d.ts.map +1 -1
  27. package/dist/run-suite.js +94 -23
  28. package/dist/run-suite.js.map +1 -1
  29. package/dist/scenario-story.d.ts +4 -3
  30. package/dist/scenario-story.d.ts.map +1 -1
  31. package/dist/scenario-story.js +47 -71
  32. package/dist/scenario-story.js.map +1 -1
  33. package/dist/scenario-usage.d.ts.map +1 -1
  34. package/dist/scenario-usage.js +2 -3
  35. package/dist/scenario-usage.js.map +1 -1
  36. package/dist/types.d.ts +32 -22
  37. package/dist/types.d.ts.map +1 -1
  38. package/dist/validate-suite.d.ts.map +1 -1
  39. package/dist/validate-suite.js +64 -38
  40. package/dist/validate-suite.js.map +1 -1
  41. package/dist/viewer/catalog.d.ts +3 -1
  42. package/dist/viewer/catalog.d.ts.map +1 -1
  43. package/dist/viewer/catalog.js +9 -0
  44. package/dist/viewer/catalog.js.map +1 -1
  45. package/dist/viewer/compare-metrics.d.ts +30 -0
  46. package/dist/viewer/compare-metrics.d.ts.map +1 -0
  47. package/dist/viewer/compare-metrics.js +102 -0
  48. package/dist/viewer/compare-metrics.js.map +1 -0
  49. package/dist/viewer/context-files.d.ts +8 -0
  50. package/dist/viewer/context-files.d.ts.map +1 -0
  51. package/dist/viewer/context-files.js +56 -0
  52. package/dist/viewer/context-files.js.map +1 -0
  53. package/dist/viewer/events.d.ts +20 -0
  54. package/dist/viewer/events.d.ts.map +1 -1
  55. package/dist/viewer/events.js +2 -0
  56. package/dist/viewer/events.js.map +1 -1
  57. package/dist/viewer/live-runner.d.ts +6 -0
  58. package/dist/viewer/live-runner.d.ts.map +1 -1
  59. package/dist/viewer/live-runner.js +34 -5
  60. package/dist/viewer/live-runner.js.map +1 -1
  61. package/dist/viewer/page.d.ts.map +1 -1
  62. package/dist/viewer/page.js +1325 -92
  63. package/dist/viewer/page.js.map +1 -1
  64. package/dist/viewer/run-controller.d.ts +5 -0
  65. package/dist/viewer/run-controller.d.ts.map +1 -1
  66. package/dist/viewer/run-controller.js +27 -1
  67. package/dist/viewer/run-controller.js.map +1 -1
  68. package/dist/viewer/server.d.ts.map +1 -1
  69. package/dist/viewer/server.js +18 -11
  70. package/dist/viewer/server.js.map +1 -1
  71. package/package.json +7 -5
@@ -41,19 +41,189 @@ function viewerCss() {
41
41
  button.danger { border-color: color-mix(in srgb, var(--fail) 40%, var(--border)); color: var(--fail); }
42
42
  button:disabled { opacity: 0.5; cursor: not-allowed; }
43
43
  .warn { color: var(--skip); font-size: 0.8rem; }
44
- .matrix { width: 100%; border-collapse: collapse; font-size: 0.85rem; }
45
- .matrix th, .matrix td { border: 1px solid var(--border); padding: 0.5rem 0.55rem; text-align: left; vertical-align: top; }
46
- .matrix th { color: var(--muted); font-size: 0.72rem; text-transform: uppercase; letter-spacing: 0.04em; }
44
+ .scenario-list { display: grid; gap: 0.75rem; }
45
+ .scenario-card {
46
+ background: var(--panel);
47
+ border: 1px solid var(--border);
48
+ border-radius: 12px;
49
+ padding: 0.85rem 0.95rem;
50
+ display: grid;
51
+ gap: 0.7rem;
52
+ }
47
53
  .scenario-detail { display: grid; gap: 0.35rem; }
48
54
  .prompt-preview { color: var(--muted); font-size: 0.78rem; white-space: pre-wrap; }
49
55
  .rubric-line { font-size: 0.75rem; color: var(--muted); }
50
56
  .arm-note { font-size: 0.78rem; color: var(--muted); }
51
- .cell-actions { display: flex; flex-wrap: wrap; gap: 0.35rem; }
57
+ .scenario-toolbar { display: flex; flex-wrap: wrap; gap: 0.55rem 0.85rem; align-items: center; }
58
+ .scenario-actions { display: flex; flex-wrap: wrap; gap: 0.35rem; align-items: center; }
59
+ .host-tablist { display: flex; flex-wrap: wrap; gap: 0.35rem; }
60
+ .host-tab {
61
+ display: inline-flex;
62
+ align-items: center;
63
+ gap: 0.4rem;
64
+ }
65
+ .host-tab[aria-selected="true"] {
66
+ background: color-mix(in srgb, var(--pass) 16%, var(--panel-2));
67
+ border-color: color-mix(in srgb, var(--pass) 40%, var(--border));
68
+ }
69
+ .host-tab[data-status="running"],
70
+ .host-tab[data-status="cancelling"],
71
+ .host-tab[data-status="cancelled"] { color: var(--skip); }
72
+ .host-tab[data-status="passed"] { color: var(--pass); }
73
+ .host-tab[data-status="failed"] { color: var(--fail); }
74
+ .host-tab[data-status="skipped"], .host-tab[data-status="skip"] { color: var(--muted); }
75
+ .host-status { display: inline-flex; align-items: center; }
52
76
  .cell-status { font-size: 0.72rem; font-weight: 700; text-transform: uppercase; }
53
- .live-dock { margin-top: 1.5rem; display: grid; gap: 0.85rem; }
54
- .live-cell { background: var(--panel); border: 1px solid var(--border); border-radius: 12px; padding: 0.75rem; min-width: 0; }
77
+ .scenario-live { display: grid; gap: 0.75rem; }
78
+ .scenario-live[hidden], .host-panel[hidden], .host-empty[hidden] { display: none; }
79
+ .host-panels { display: grid; min-width: 0; }
80
+ .host-panel:not([hidden]) { min-height: 4rem; }
81
+ .host-empty { color: var(--muted); font-size: 0.85rem; margin: 0; }
82
+ .host-panel > .live-cell > h3 { display: none; }
83
+ .live-cell { background: var(--panel-2); border: 1px solid var(--border); border-radius: 12px; padding: 0.75rem; min-width: 0; }
84
+ .cell-result {
85
+ margin-top: 0.65rem;
86
+ border-left: 3px solid var(--border);
87
+ padding: 0.45rem 0.7rem;
88
+ }
89
+ .cell-result.status-passed { border-left-color: var(--pass); }
90
+ .cell-result.status-failed { border-left-color: var(--fail); }
91
+ .cell-result.status-skipped { border-left-color: var(--skip); }
92
+ .cell-result-verdict { font-weight: 650; margin: 0 0 0.25rem; }
93
+ .cell-result.status-passed .cell-result-verdict { color: var(--pass); }
94
+ .cell-result.status-failed .cell-result-verdict { color: var(--fail); }
95
+ .cell-result-metrics { color: var(--muted); font-size: 0.8rem; margin: 0 0 0.35rem; }
96
+ .cell-result-failures, .cell-result-judges {
97
+ list-style: none;
98
+ margin: 0;
99
+ padding: 0;
100
+ display: grid;
101
+ gap: 0.35rem;
102
+ }
103
+ .cell-result-judges { margin-top: 0.45rem; }
104
+ .cell-result-failures li, .cell-result-judges li {
105
+ font-size: 0.85rem;
106
+ white-space: pre-wrap;
107
+ word-break: break-word;
108
+ }
109
+ .judge-pass { color: var(--pass); }
110
+ .judge-fail { color: var(--fail); }
55
111
  .live-cell h3 { margin-bottom: 0.45rem; }
112
+ .live-status {
113
+ list-style: none;
114
+ margin: 0 0 0.55rem;
115
+ padding: 0;
116
+ display: grid;
117
+ gap: 0.2rem;
118
+ max-height: 9rem;
119
+ overflow: auto;
120
+ font-size: 0.75rem;
121
+ color: var(--muted);
122
+ font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
123
+ }
124
+ .live-status li {
125
+ border-left: 2px solid var(--border);
126
+ padding: 0.12rem 0 0.12rem 0.5rem;
127
+ white-space: pre-wrap;
128
+ word-break: break-word;
129
+ }
56
130
  .run-banner { font-size: 0.85rem; color: var(--muted); }
131
+ .run-progress {
132
+ display: grid;
133
+ gap: 0.55rem;
134
+ background: var(--panel);
135
+ border: 1px solid var(--border);
136
+ border-radius: 12px;
137
+ padding: 0.8rem 1rem;
138
+ margin-bottom: 1.25rem;
139
+ }
140
+ .run-progress[hidden] { display: none; }
141
+ .run-progress-head, .run-progress-counts {
142
+ display: flex;
143
+ flex-wrap: wrap;
144
+ align-items: center;
145
+ justify-content: space-between;
146
+ gap: 0.4rem 1rem;
147
+ }
148
+ .run-progress-title { margin: 0; font-weight: 700; }
149
+ .run-progress-counts { justify-content: flex-start; color: var(--muted); font-size: 0.8rem; }
150
+ .run-progress-counts strong { color: var(--text); font-variant-numeric: tabular-nums; }
151
+ .run-progress-counts .progress-passed strong { color: var(--pass); }
152
+ .run-progress-counts .progress-failed strong { color: var(--fail); }
153
+ .run-progress-counts .progress-skipped strong { color: var(--skip); }
154
+ .run-progress-track {
155
+ height: 0.7rem;
156
+ overflow: hidden;
157
+ border-radius: 999px;
158
+ background: var(--panel-2);
159
+ border: 1px solid var(--border);
160
+ }
161
+ .run-progress-fill {
162
+ display: block;
163
+ width: 0;
164
+ height: 100%;
165
+ background: var(--pass);
166
+ transition: width 180ms ease-out;
167
+ }
168
+ .run-progress[data-has-failures="true"] .run-progress-fill {
169
+ background: linear-gradient(90deg, var(--pass), var(--fail));
170
+ }
171
+ .compare-tab[data-status="running"] { color: var(--skip); }
172
+ .compare-tab[data-status="passed"] { color: var(--pass); }
173
+ .compare-tab[data-status="failed"] { color: var(--fail); }
174
+ .compare-tab[data-status="skipped"] { color: var(--muted); }
175
+ .compare-tabs .compare-arms,
176
+ .compare-tabs .compare-arms[data-arm-count] { grid-template-columns: minmax(0, 1fr); }
177
+ .compare-tabs .live-cell[hidden] { display: none; }
178
+ .compare-tabs .live-cell > h3 { display: none; }
179
+ .chat-running-row[hidden] { display: none; }
180
+ .context-path {
181
+ font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
182
+ font-size: 0.82rem;
183
+ margin: 0 0 0.25rem;
184
+ }
185
+ .context-why {
186
+ color: var(--muted);
187
+ font-size: 0.8rem;
188
+ margin: 0 0 0.45rem;
189
+ }
190
+ .chat-running {
191
+ display: inline-flex;
192
+ align-items: center;
193
+ gap: 0.45rem;
194
+ color: var(--muted);
195
+ font-size: 0.85rem;
196
+ padding: 0.2rem 0.1rem;
197
+ }
198
+ .chat-progress {
199
+ display: inline-flex;
200
+ gap: 0.18rem;
201
+ }
202
+ .chat-progress span {
203
+ width: 0.32rem;
204
+ height: 0.32rem;
205
+ border-radius: 999px;
206
+ background: var(--skip);
207
+ animation: chat-progress 1s ease-in-out infinite;
208
+ }
209
+ .chat-progress span:nth-child(2) { animation-delay: 0.15s; }
210
+ .chat-progress span:nth-child(3) { animation-delay: 0.3s; }
211
+ @keyframes chat-progress {
212
+ 0%, 80%, 100% { opacity: 0.2; }
213
+ 40% { opacity: 1; }
214
+ }
215
+ .bubble.is-streaming .bubble-text::after {
216
+ content: "▍";
217
+ margin-left: 0.12rem;
218
+ color: var(--pass);
219
+ animation: chat-caret 1s step-end infinite;
220
+ }
221
+ @keyframes chat-caret {
222
+ 50% { opacity: 0; }
223
+ }
224
+ @media (prefers-reduced-motion: reduce) {
225
+ .chat-progress span, .bubble.is-streaming .bubble-text::after { animation: none; }
226
+ }
57
227
  `;
58
228
  }
59
229
  function rubricSummary(scenario) {
@@ -85,43 +255,79 @@ function rubricSummary(scenario) {
85
255
  if (rubric.judge) {
86
256
  parts.push("judge");
87
257
  }
258
+ if (scenario.contextSources?.length) {
259
+ parts.push(`contextSources ${scenario.contextSources.join(", ")}`);
260
+ }
88
261
  return parts.join(" · ");
89
262
  }
90
- function renderScenarioRow(suite, scenario) {
91
- const hosts = suite.hosts
263
+ function defaultScenarioHost(suite, scenario) {
264
+ if (scenario.host && suite.hosts.includes(scenario.host)) {
265
+ return scenario.host;
266
+ }
267
+ if (suite.hosts.includes("cursor")) {
268
+ return "cursor";
269
+ }
270
+ return suite.hosts[0] ?? "cursor";
271
+ }
272
+ function hostIsSkipped(scenario, host) {
273
+ return Boolean(scenario.skip || (scenario.host && scenario.host !== host));
274
+ }
275
+ function renderHostChrome(suite, scenario, selectedHost) {
276
+ if (suite.hosts.length <= 1) {
277
+ const host = suite.hosts[0] ?? selectedHost;
278
+ const skipped = hostIsSkipped(scenario, host);
279
+ return `<div class="host-status" data-cell="${escapeHtml(`${suite.name}::${scenario.name}::${host}`)}">
280
+ <span class="cell-status muted">${skipped ? "skip" : "idle"}</span>
281
+ </div>`;
282
+ }
283
+ const tabs = suite.hosts
92
284
  .map((host) => {
93
- const pinned = scenario.host && scenario.host !== host;
94
- const skipped = scenario.skip || pinned;
95
- return `<td data-cell="${escapeHtml(`${suite.name}::${scenario.name}::${host}`)}">
96
- <div class="cell-actions">
97
- <button class="primary run-cell" data-suite="${escapeHtml(suite.name)}" data-scenario="${escapeHtml(scenario.name)}" data-host="${escapeHtml(host)}"${skipped ? " disabled" : ""}>Run</button>
98
- <span class="cell-status muted">${skipped ? "skip" : "idle"}</span>
99
- </div>
100
- </td>`;
285
+ const skipped = hostIsSkipped(scenario, host);
286
+ const selected = host === selectedHost;
287
+ return `<button type="button" class="host-tab" role="tab" aria-label="${escapeHtml(host)}" data-cell="${escapeHtml(`${suite.name}::${scenario.name}::${host}`)}" data-host="${escapeHtml(host)}" aria-selected="${selected ? "true" : "false"}"${skipped ? " disabled" : ""}>${escapeHtml(host)} <span class="cell-status muted" aria-hidden="true">${skipped ? "skip" : "idle"}</span></button>`;
101
288
  })
102
289
  .join("");
290
+ return `<div class="host-tablist" role="tablist">${tabs}</div>`;
291
+ }
292
+ function renderHostPanels(suite, selectedHost) {
293
+ return suite.hosts
294
+ .map((host) => {
295
+ const selected = host === selectedHost;
296
+ return `<div class="host-panel" data-host-panel="${escapeHtml(host)}" role="tabpanel"${selected ? "" : " hidden"}><p class="host-empty">No run yet.</p></div>`;
297
+ })
298
+ .join("");
299
+ }
300
+ function renderScenarioCard(suite, scenario) {
301
+ const selectedHost = defaultScenarioHost(suite, scenario);
302
+ const runDisabled = hostIsSkipped(scenario, selectedHost);
103
303
  const compare = scenario.compare
104
304
  ? scenario.compare
105
305
  .map((arm) => `<p class="arm-note">${escapeHtml(arm.label)}: ${escapeHtml(arm.description ?? arm.prompt ?? "")}</p>`)
106
306
  .join("")
107
307
  : "";
108
- return `<tr>
109
- <th scope="row">
110
- <div class="scenario-detail">
111
- <strong>${escapeHtml(scenario.name)}</strong>
112
- ${scenario.description ? `<p class="scenario-lede">${escapeHtml(scenario.description)}</p>` : ""}
113
- <p class="prompt-preview">${escapeHtml(scenario.prompt)}</p>
114
- <p class="rubric-line">${escapeHtml(rubricSummary(scenario))}</p>
115
- ${compare}
116
- <button class="run-row" data-suite="${escapeHtml(suite.name)}" data-scenario="${escapeHtml(scenario.name)}">Run row</button>
308
+ const key = escapeHtml(`${suite.name}::${scenario.name}`);
309
+ return `<article class="scenario-card" data-scenario-card="${key}">
310
+ <div class="scenario-detail">
311
+ <strong>${escapeHtml(scenario.name)}</strong>
312
+ ${scenario.description ? `<p class="scenario-lede">${escapeHtml(scenario.description)}</p>` : ""}
313
+ <p class="prompt-preview">${escapeHtml(scenario.prompt)}</p>
314
+ <p class="rubric-line">${escapeHtml(rubricSummary(scenario))}</p>
315
+ ${compare}
316
+ </div>
317
+ <div class="scenario-toolbar">
318
+ ${renderHostChrome(suite, scenario, selectedHost)}
319
+ <div class="scenario-actions">
320
+ <button class="primary run-cell" data-suite="${escapeHtml(suite.name)}" data-scenario="${escapeHtml(scenario.name)}" data-host="${escapeHtml(selectedHost)}"${runDisabled ? " disabled" : ""}>Run</button>
321
+ <button class="run-row" data-suite="${escapeHtml(suite.name)}" data-scenario="${escapeHtml(scenario.name)}"${scenario.skip ? " disabled" : ""}>Run selected hosts</button>
117
322
  </div>
118
- </th>
119
- ${hosts}
120
- </tr>`;
323
+ </div>
324
+ <div class="scenario-live" hidden data-live-row="${key}" data-live-slot="${key}">
325
+ <div class="host-panels">${renderHostPanels(suite, selectedHost)}</div>
326
+ </div>
327
+ </article>`;
121
328
  }
122
329
  function renderSuiteSection(suite) {
123
- const columns = suite.hosts.map((host) => `<th>${escapeHtml(host)}</th>`).join("");
124
- const rows = suite.scenarios.map((scenario) => renderScenarioRow(suite, scenario)).join("");
330
+ const cards = suite.scenarios.map((scenario) => renderScenarioCard(suite, scenario)).join("");
125
331
  return `
126
332
  <section class="suite" data-suite="${escapeHtml(suite.name)}">
127
333
  <header class="suite-header">
@@ -131,10 +337,7 @@ function renderSuiteSection(suite) {
131
337
  </div>
132
338
  <button class="run-suite" data-suite="${escapeHtml(suite.name)}">Run suite</button>
133
339
  </header>
134
- <table class="matrix">
135
- <thead><tr><th>Scenario</th>${columns}</tr></thead>
136
- <tbody>${rows}</tbody>
137
- </table>
340
+ <div class="scenario-list">${cards}</div>
138
341
  </section>`;
139
342
  }
140
343
  function clientScript() {
@@ -143,6 +346,8 @@ function clientScript() {
143
346
  var catalog = JSON.parse(document.getElementById("catalog-data").textContent);
144
347
  var source = null;
145
348
  var runId = null;
349
+ var cancelRequested = false;
350
+ var progress = { total: 0, completed: 0, passed: 0, failed: 0, skipped: 0, seen: {}, experiments: {} };
146
351
 
147
352
  function selectedHosts() {
148
353
  return Array.prototype.map.call(document.querySelectorAll("[data-host-toggle]:checked"), function (box) {
@@ -158,6 +363,134 @@ function clientScript() {
158
363
  document.getElementById("run-banner").textContent = text;
159
364
  }
160
365
 
366
+ function runnableTargets(body) {
367
+ var requestedHosts = body.hosts && body.hosts.length ? body.hosts : catalog.defaultSelectedHosts;
368
+ var total = 0;
369
+ catalog.suites.forEach(function (suite) {
370
+ if (body.suite && suite.name !== body.suite) {
371
+ return;
372
+ }
373
+ suite.scenarios.forEach(function (scenario) {
374
+ if (body.scenario && scenario.name !== body.scenario) {
375
+ return;
376
+ }
377
+ if (scenario.skip) {
378
+ return;
379
+ }
380
+ requestedHosts.forEach(function (host) {
381
+ if (suite.hosts.indexOf(host) === -1 || (scenario.host && scenario.host !== host)) {
382
+ return;
383
+ }
384
+ total += 1;
385
+ });
386
+ });
387
+ });
388
+ return total;
389
+ }
390
+
391
+ function renderProgress() {
392
+ var box = document.getElementById("run-progress");
393
+ var total = Math.max(progress.total, progress.completed);
394
+ var remaining = Math.max(0, total - progress.completed);
395
+ box.hidden = total === 0;
396
+ box.setAttribute("data-has-failures", progress.failed > 0 ? "true" : "false");
397
+ var testLabel = total === 1 ? "test" : "tests";
398
+ box.querySelector(".run-progress-title").textContent = progress.completed + " of " + total + " " + testLabel + " finished";
399
+ box.querySelector(".progress-passed strong").textContent = String(progress.passed);
400
+ box.querySelector(".progress-failed strong").textContent = String(progress.failed);
401
+ box.querySelector(".progress-skipped strong").textContent = String(progress.skipped);
402
+ box.querySelector(".progress-remaining strong").textContent = String(remaining);
403
+ var percent = total > 0 ? Math.round((progress.completed / total) * 100) : 0;
404
+ var fill = box.querySelector(".run-progress-fill");
405
+ fill.style.width = percent + "%";
406
+ fill.parentNode.setAttribute("aria-valuenow", String(progress.completed));
407
+ fill.parentNode.setAttribute("aria-valuemax", String(total));
408
+ }
409
+
410
+ function resetProgress(body) {
411
+ progress = { total: runnableTargets(body), completed: 0, passed: 0, failed: 0, skipped: 0, seen: {}, experiments: {} };
412
+ renderProgress();
413
+ }
414
+
415
+ function recordProgress(event) {
416
+ var key = cellKey(event);
417
+ if (progress.seen[key]) {
418
+ return;
419
+ }
420
+ progress.seen[key] = true;
421
+ var experiment = experimentState(event);
422
+ experiment.cells[event.arm || "_"] = event;
423
+ var scenario = findScenario(event.suite, event.scenario);
424
+ var expectedArms = scenario && scenario.compare && scenario.compare.length ? scenario.compare.length : 1;
425
+ if (Object.keys(experiment.cells).length < expectedArms || experiment.counted) {
426
+ renderProgress();
427
+ return;
428
+ }
429
+ experiment.counted = true;
430
+ progress.completed += 1;
431
+ var outcome = experimentOutcome(scenario, experiment.cells);
432
+ if (outcome === "skipped") progress.skipped += 1;
433
+ else if (outcome === "passed") progress.passed += 1;
434
+ else progress.failed += 1;
435
+ renderProgress();
436
+ }
437
+
438
+ function finishProgress(event) {
439
+ // The stream reports one cell per comparison arm. Progress is based on
440
+ // the completed scenario/host experiment, so expected failed controls do
441
+ // not turn a passing comparison into a red suite result.
442
+ progress.total = Math.max(progress.total, progress.completed);
443
+ renderProgress();
444
+ }
445
+
446
+ function experimentKey(event) {
447
+ return [event.suite, event.scenario, event.host].join("::");
448
+ }
449
+
450
+ function experimentState(event) {
451
+ var key = experimentKey(event);
452
+ if (!progress.experiments[key]) {
453
+ progress.experiments[key] = { cells: {}, counted: false };
454
+ }
455
+ return progress.experiments[key];
456
+ }
457
+
458
+ function experimentOutcome(scenario, cells) {
459
+ var events = Object.keys(cells).map(function (id) { return cells[id]; });
460
+ if (events.some(function (event) { return event.skipped; })) return "skipped";
461
+ var byArm = {};
462
+ events.forEach(function (event) { byArm[event.arm || "_"] = event; });
463
+ if (!scenario || !scenario.compare || !scenario.compare.length) {
464
+ return events.every(function (event) { return event.passed; }) ? "passed" : "failed";
465
+ }
466
+ var gates = scenario.gates || [];
467
+ var behaviorOk = events.every(function (event) {
468
+ return event.passed || (event.failures || []).every(function (failure) {
469
+ return gates.some(function (gate) {
470
+ return gate.metric === "outcome" && gate.arm === event.arm && gate.operator === "equal" && gate.value === "fail";
471
+ });
472
+ });
473
+ });
474
+ var gatesOk = gates.every(function (gate) {
475
+ if (gate.metric === "outcome" && gate.arm) {
476
+ var event = byArm[gate.arm];
477
+ if (!event) return false;
478
+ var actual = event.passed ? "pass" : "fail";
479
+ return gate.operator === "equal" ? actual === gate.value : false;
480
+ }
481
+ if (["turns", "tokens", "tools", "durationMs"].indexOf(gate.metric) !== -1 && gate.winner && gate.loser) {
482
+ var winner = byArm[gate.winner];
483
+ var loser = byArm[gate.loser];
484
+ var field = gate.metric === "durationMs" ? "durationMs" : gate.metric;
485
+ var winnerValue = winner && winner.metrics && winner.metrics[field];
486
+ var loserValue = loser && loser.metrics && loser.metrics[field];
487
+ return typeof winnerValue === "number" && typeof loserValue === "number" && winnerValue < loserValue;
488
+ }
489
+ return true;
490
+ });
491
+ return behaviorOk && gatesOk ? "passed" : "failed";
492
+ }
493
+
161
494
  function cellKey(event) {
162
495
  return [event.suite, event.scenario, event.host, event.arm || "_"].join("::");
163
496
  }
@@ -166,41 +499,372 @@ function clientScript() {
166
499
  return "live-" + cellKey(event).replace(/[^a-z0-9]+/gi, "-");
167
500
  }
168
501
 
502
+ function compareWrapId(event) {
503
+ return ("live-wrap-" + event.suite + "-" + event.scenario + "-" + event.host).replace(/[^a-z0-9]+/gi, "-");
504
+ }
505
+
506
+ function selectCompareTab(wrap, armId) {
507
+ wrap.querySelectorAll(".compare-tab").forEach(function (tab) {
508
+ var on = tab.getAttribute("data-arm") === armId;
509
+ tab.setAttribute("aria-selected", on ? "true" : "false");
510
+ });
511
+ wrap.querySelectorAll(".live-cell").forEach(function (panel) {
512
+ panel.hidden = panel.getAttribute("data-arm") !== armId;
513
+ });
514
+ }
515
+
516
+ function scenarioCard(event) {
517
+ return document.querySelector('[data-scenario-card="' + event.suite + "::" + event.scenario + '"]');
518
+ }
519
+
520
+ function selectHostTab(card, host) {
521
+ if (!card || !host) {
522
+ return;
523
+ }
524
+ card.querySelectorAll(".host-tab").forEach(function (tab) {
525
+ var on = tab.getAttribute("data-host") === host;
526
+ tab.setAttribute("aria-selected", on ? "true" : "false");
527
+ });
528
+ card.querySelectorAll("[data-host-panel]").forEach(function (panel) {
529
+ panel.hidden = panel.getAttribute("data-host-panel") !== host;
530
+ });
531
+ var run = card.querySelector("button.run-cell");
532
+ if (run) {
533
+ run.setAttribute("data-host", host);
534
+ var tab = card.querySelector('.host-tab[data-host="' + host + '"]');
535
+ if (tab) {
536
+ run.disabled = tab.disabled;
537
+ }
538
+ }
539
+ }
540
+
541
+ function revealLive(event) {
542
+ var card = scenarioCard(event);
543
+ if (!card) {
544
+ return;
545
+ }
546
+ var live = card.querySelector(".scenario-live");
547
+ if (live) {
548
+ live.hidden = false;
549
+ }
550
+ if (card.dataset.userPickedHost === "1") {
551
+ return;
552
+ }
553
+ if (card.dataset.followHost && card.dataset.followHost !== event.host) {
554
+ return;
555
+ }
556
+ card.dataset.followHost = event.host;
557
+ selectHostTab(card, event.host);
558
+ }
559
+
560
+ function hostPanel(event) {
561
+ var card = scenarioCard(event);
562
+ return card ? card.querySelector('[data-host-panel="' + event.host + '"]') : null;
563
+ }
564
+
565
+ function markArmTab(wrap, armId, status) {
566
+ var tab = wrap.querySelector('.compare-tab[data-arm="' + armId + '"]');
567
+ if (tab) {
568
+ tab.setAttribute("data-status", status);
569
+ }
570
+ }
571
+
572
+ function createLiveArticle(event, title) {
573
+ var article = document.createElement("article");
574
+ article.className = "live-cell";
575
+ article.id = paneId(event);
576
+ article.dataset.chatKey = cellKey(event);
577
+ article.innerHTML = "<h3></h3><ol class=\\"live-status\\"></ol><div class=\\"chat\\"></div>";
578
+ article.querySelector("h3").textContent = title;
579
+ return article;
580
+ }
581
+
582
+ var openChats = {};
583
+
584
+ function resetArticle(article) {
585
+ if (!article) {
586
+ return;
587
+ }
588
+ var status = article.querySelector(".live-status");
589
+ if (status) {
590
+ status.replaceChildren();
591
+ }
592
+ var chat = article.querySelector(".chat");
593
+ if (chat) {
594
+ chat.replaceChildren();
595
+ }
596
+ var result = article.querySelector(".cell-result");
597
+ if (result) {
598
+ result.remove();
599
+ }
600
+ var panel = article.closest("[data-host-panel]");
601
+ var empty = panel ? panel.querySelector(".host-empty") : null;
602
+ if (empty) {
603
+ empty.hidden = false;
604
+ }
605
+ }
606
+
607
+ function chatIsOpen(event) {
608
+ return !!openChats[cellKey(event)];
609
+ }
610
+
611
+ function startChat(event) {
612
+ openChats[cellKey(event)] = true;
613
+ delete assistantBodies[cellKey(event)];
614
+ delete finishedMetrics[cellKey(event)];
615
+ ensurePane(event);
616
+ resetArticle(document.getElementById(paneId(event)));
617
+ var panel = hostPanel(event);
618
+ var empty = panel ? panel.querySelector(".host-empty") : null;
619
+ if (empty) {
620
+ empty.hidden = true;
621
+ }
622
+ }
623
+
624
+ function markRunningIdle() {
625
+ document.querySelectorAll("[data-cell]").forEach(function (node) {
626
+ var status = node.querySelector(".cell-status");
627
+ if (!status) {
628
+ return;
629
+ }
630
+ var label = status.textContent;
631
+ if (label !== "running" && label !== "starting") {
632
+ return;
633
+ }
634
+ status.textContent = "idle";
635
+ status.className = "cell-status muted";
636
+ if (node.classList.contains("host-tab")) {
637
+ node.setAttribute("data-status", "idle");
638
+ }
639
+ });
640
+ document.querySelectorAll(".compare-tab[data-status='running']").forEach(function (tab) {
641
+ tab.setAttribute("data-status", "idle");
642
+ });
643
+ }
644
+
645
+ function closeAllChats() {
646
+ Object.keys(openChats).forEach(function (key) {
647
+ resetArticle(document.getElementById("live-" + key.replace(/[^a-z0-9]+/gi, "-")));
648
+ });
649
+ openChats = {};
650
+ assistantBodies = {};
651
+ hideAllRunning();
652
+ markRunningIdle();
653
+ }
654
+
655
+ function setCancelButton(label, disabled) {
656
+ var button = document.getElementById("cancel-run");
657
+ button.textContent = label;
658
+ button.disabled = disabled;
659
+ }
660
+
661
+ function showCancelledNote(article, text) {
662
+ if (!article) {
663
+ return;
664
+ }
665
+ var existing = article.querySelector(".cell-result");
666
+ if (existing) {
667
+ existing.remove();
668
+ }
669
+ var box = document.createElement("div");
670
+ box.className = "cell-result status-skipped";
671
+ var verdict = document.createElement("p");
672
+ verdict.className = "cell-result-verdict";
673
+ verdict.textContent = text;
674
+ box.appendChild(verdict);
675
+ article.appendChild(box);
676
+ var panel = article.closest("[data-host-panel]");
677
+ var empty = panel ? panel.querySelector(".host-empty") : null;
678
+ if (empty) {
679
+ empty.hidden = true;
680
+ }
681
+ }
682
+
683
+ function cancelOpenChats() {
684
+ document.querySelectorAll(".cell-status").forEach(function (status) {
685
+ if (status.textContent !== "running" && status.textContent !== "starting") {
686
+ return;
687
+ }
688
+ status.textContent = "cancelling";
689
+ status.className = "cell-status status-skipped";
690
+ var tab = status.closest(".host-tab");
691
+ if (tab) {
692
+ tab.setAttribute("data-status", "cancelling");
693
+ }
694
+ });
695
+ document.querySelectorAll(".compare-tab[data-status='running']").forEach(function (tab) {
696
+ tab.setAttribute("data-status", "cancelling");
697
+ });
698
+ hideAllRunning();
699
+ document.querySelectorAll("[data-cell]").forEach(function (node) {
700
+ var status = node.querySelector(".cell-status");
701
+ if (!status || status.textContent !== "cancelling") {
702
+ return;
703
+ }
704
+ var card = node.closest("[data-scenario-card]");
705
+ if (!card) {
706
+ return;
707
+ }
708
+ var live = card.querySelector(".scenario-live");
709
+ if (live) {
710
+ live.hidden = false;
711
+ }
712
+ var host = node.getAttribute("data-host") || (node.getAttribute("data-cell") || "").split("::")[2];
713
+ var panel = card.querySelector('[data-host-panel="' + host + '"]');
714
+ if (!panel) {
715
+ return;
716
+ }
717
+ var articles = panel.querySelectorAll("article.live-cell");
718
+ if (articles.length === 0) {
719
+ showCancelledNote(panel, "Cancelled. Stopping the host agent.");
720
+ return;
721
+ }
722
+ articles.forEach(function (article) {
723
+ var key = article.dataset.chatKey;
724
+ resetArticle(article);
725
+ showCancelledNote(article, "Cancelled. Stopping the host agent.");
726
+ if (key) {
727
+ delete openChats[key];
728
+ delete assistantBodies[key];
729
+ }
730
+ });
731
+ });
732
+ }
733
+
734
+ function markCancelledCells() {
735
+ document.querySelectorAll(".cell-status").forEach(function (status) {
736
+ if (status.textContent === "cancelling") {
737
+ status.textContent = "cancelled";
738
+ status.className = "cell-status status-skipped";
739
+ var tab = status.closest(".host-tab");
740
+ if (tab) {
741
+ tab.setAttribute("data-status", "cancelled");
742
+ }
743
+ }
744
+ });
745
+ document.querySelectorAll(".compare-tab[data-status='cancelling']").forEach(function (tab) {
746
+ tab.setAttribute("data-status", "cancelled");
747
+ });
748
+ document.querySelectorAll(".cell-result-verdict").forEach(function (node) {
749
+ if (node.textContent === "Cancelled. Stopping the host agent.") {
750
+ node.textContent = "Cancelled.";
751
+ }
752
+ });
753
+ }
754
+
755
+ function sealAllChats() {
756
+ openChats = {};
757
+ }
758
+
759
+ function addArmTab(wrap, list, article, armId, label, selected) {
760
+ article.setAttribute("role", "tabpanel");
761
+ article.setAttribute("data-arm", armId);
762
+ article.hidden = !selected;
763
+ var tab = document.createElement("button");
764
+ tab.type = "button";
765
+ tab.className = "compare-tab";
766
+ tab.setAttribute("role", "tab");
767
+ tab.setAttribute("data-arm", armId);
768
+ tab.setAttribute("aria-controls", article.id);
769
+ tab.setAttribute("aria-selected", selected ? "true" : "false");
770
+ tab.textContent = label;
771
+ tab.addEventListener("click", function () {
772
+ wrap.dataset.userPicked = "1";
773
+ selectCompareTab(wrap, armId);
774
+ });
775
+ list.appendChild(tab);
776
+ }
777
+
778
+ function ensureCompareWrap(event, scenario, slot) {
779
+ var wrapId = compareWrapId(event);
780
+ var wrap = document.getElementById(wrapId);
781
+ if (!wrap) {
782
+ wrap = document.createElement("div");
783
+ wrap.className = "compare-layout compare-tabs";
784
+ wrap.id = wrapId;
785
+ var list = document.createElement("div");
786
+ list.className = "compare-tablist";
787
+ list.setAttribute("role", "tablist");
788
+ list.setAttribute("aria-label", "Comparison arms");
789
+ wrap.appendChild(list);
790
+ var arms = document.createElement("div");
791
+ arms.className = "compare-arms";
792
+ if (scenario && scenario.compare) {
793
+ arms.setAttribute("data-arm-count", String(scenario.compare.length));
794
+ }
795
+ wrap.appendChild(arms);
796
+ if (scenario && scenario.compare) {
797
+ scenario.compare.forEach(function (arm, index) {
798
+ var article = createLiveArticle({
799
+ suite: event.suite,
800
+ scenario: event.scenario,
801
+ host: event.host,
802
+ arm: arm.id,
803
+ }, arm.label);
804
+ addArmTab(wrap, list, article, arm.id, arm.label, index === 0);
805
+ arms.appendChild(article);
806
+ });
807
+ }
808
+ }
809
+ if (!wrap.parentNode) {
810
+ slot.appendChild(wrap);
811
+ }
812
+ return wrap;
813
+ }
814
+
169
815
  function ensurePane(event) {
170
- var id = paneId(event);
171
- var existing = document.getElementById(id);
816
+ var existing = document.getElementById(paneId(event));
172
817
  if (existing) {
818
+ revealLive(event);
173
819
  return existing.querySelector(".chat");
174
820
  }
175
- var dock = document.getElementById("live-dock");
176
- var article = document.createElement("article");
177
- article.className = "live-cell";
178
- article.id = id;
179
- var title = event.suite + " / " + event.scenario + " / " + event.host;
180
- if (event.arm) {
181
- title += " / arm " + event.arm;
821
+ var slot = hostPanel(event);
822
+ if (!slot) {
823
+ return null;
182
824
  }
183
- article.innerHTML = "<h3></h3><div class=\\"chat\\"></div>";
184
- article.querySelector("h3").textContent = title;
185
- var parent = dock;
825
+ revealLive(event);
826
+ var empty = slot.querySelector(".host-empty");
827
+ if (empty) {
828
+ empty.hidden = true;
829
+ }
830
+ var leftover = slot.querySelector(":scope > .cell-result");
831
+ if (leftover) {
832
+ leftover.remove();
833
+ }
834
+ var scenario = findScenario(event.suite, event.scenario);
186
835
  if (event.arm) {
187
- var wrapId = ("live-wrap-" + event.suite + "-" + event.scenario + "-" + event.host).replace(/[^a-z0-9]+/gi, "-");
188
- var wrap = document.getElementById(wrapId);
189
- if (!wrap) {
190
- wrap = document.createElement("div");
191
- wrap.className = "compare-layout";
192
- wrap.id = wrapId;
193
- var arms = document.createElement("div");
194
- arms.className = "compare-arms";
195
- wrap.appendChild(arms);
196
- dock.appendChild(wrap);
197
- }
198
- parent = wrap.querySelector(".compare-arms");
199
- }
200
- parent.appendChild(article);
836
+ var wrap = ensureCompareWrap(event, scenario, slot);
837
+ existing = document.getElementById(paneId(event));
838
+ if (existing) {
839
+ if (wrap.dataset.userPicked !== "1") {
840
+ selectCompareTab(wrap, event.arm);
841
+ }
842
+ return existing.querySelector(".chat");
843
+ }
844
+ return null;
845
+ }
846
+ var article = createLiveArticle(event, event.host);
847
+ slot.appendChild(article);
201
848
  return article.querySelector(".chat");
202
849
  }
203
850
 
851
+ function appendStatus(event, text) {
852
+ var article = document.getElementById(paneId(event));
853
+ var list = article ? article.querySelector(".live-status") : null;
854
+ if (!list) {
855
+ ensurePane(event);
856
+ article = document.getElementById(paneId(event));
857
+ list = article ? article.querySelector(".live-status") : null;
858
+ }
859
+ if (!list) {
860
+ return;
861
+ }
862
+ var item = document.createElement("li");
863
+ item.textContent = text;
864
+ list.appendChild(item);
865
+ list.scrollTop = list.scrollHeight;
866
+ }
867
+
204
868
  function appendBubble(chat, role, text) {
205
869
  var row = document.createElement("div");
206
870
  row.className = "chat-row " + (role === "user" ? "side-right" : "side-left");
@@ -219,6 +883,117 @@ function clientScript() {
219
883
  return body;
220
884
  }
221
885
 
886
+ function runningRow(article) {
887
+ return article.querySelector(".chat-running-row");
888
+ }
889
+
890
+ function showRunning(event) {
891
+ var article = document.getElementById(paneId(event));
892
+ if (!article) {
893
+ ensurePane(event);
894
+ article = document.getElementById(paneId(event));
895
+ }
896
+ if (!article) {
897
+ return;
898
+ }
899
+ var chat = article.querySelector(".chat");
900
+ if (!chat) {
901
+ return;
902
+ }
903
+ var row = runningRow(article);
904
+ if (!row) {
905
+ row = document.createElement("div");
906
+ row.className = "chat-row side-left chat-running-row";
907
+ row.innerHTML = '<div class="chat-running" aria-live="polite">Running<span class="chat-progress" aria-hidden="true"><span></span><span></span><span></span></span></div>';
908
+ chat.appendChild(row);
909
+ }
910
+ row.hidden = false;
911
+ chat.appendChild(row);
912
+ }
913
+
914
+ function hideRunning(event) {
915
+ var article = document.getElementById(paneId(event));
916
+ var row = article ? runningRow(article) : null;
917
+ if (row) {
918
+ row.remove();
919
+ }
920
+ }
921
+
922
+ function hideAllRunning() {
923
+ document.querySelectorAll(".chat-running-row").forEach(function (row) {
924
+ row.remove();
925
+ });
926
+ document.querySelectorAll(".bubble.is-streaming").forEach(function (bubble) {
927
+ bubble.classList.remove("is-streaming");
928
+ });
929
+ }
930
+
931
+ function setStreaming(event, on) {
932
+ var body = assistantBodies[cellKey(event)];
933
+ var bubble = body ? body.parentNode : null;
934
+ if (bubble) {
935
+ if (on) {
936
+ bubble.classList.add("is-streaming");
937
+ } else {
938
+ bubble.classList.remove("is-streaming");
939
+ }
940
+ }
941
+ }
942
+
943
+ function appendContextFiles(chat, files) {
944
+ (files || []).forEach(function (file) {
945
+ var row = document.createElement("div");
946
+ row.className = "chat-row side-left";
947
+ var bubble = document.createElement("div");
948
+ bubble.className = "bubble role-context";
949
+ var label = document.createElement("div");
950
+ label.className = "bubble-label";
951
+ label.textContent = "Context";
952
+ var path = document.createElement("p");
953
+ path.className = "context-path";
954
+ path.textContent = file.path || "context";
955
+ var why = document.createElement("p");
956
+ why.className = "context-why";
957
+ why.textContent = file.why || "The runner loaded this file into the host preamble.";
958
+ var body = document.createElement("div");
959
+ body.className = "bubble-text";
960
+ body.textContent = file.text || "";
961
+ bubble.appendChild(label);
962
+ bubble.appendChild(path);
963
+ bubble.appendChild(why);
964
+ bubble.appendChild(body);
965
+ row.appendChild(bubble);
966
+ chat.appendChild(row);
967
+ });
968
+ }
969
+
970
+ function displayToolPath(path) {
971
+ var stripped = String(path).replace(/^file:\\/\\//, "").replace(/\\/+$/, "");
972
+ var workspace = stripped.match(/\\/agent-harness-seal-[^/]+(?:\\/(.*))?$/);
973
+ if (workspace) {
974
+ return workspace[1] ? workspace[1] : ".";
975
+ }
976
+ var sealed = stripped.match(/\\/(?:\\.agents|\\.claude|\\.codex|AGENTS\\.md|CLAUDE\\.md)(?:\\/|$)/);
977
+ if (sealed && sealed.index !== undefined) {
978
+ return stripped.slice(sealed.index + 1);
979
+ }
980
+ var parts = stripped.split("/").filter(Boolean);
981
+ if (parts.length > 3) {
982
+ return ".../" + parts.slice(-3).join("/");
983
+ }
984
+ return stripped;
985
+ }
986
+
987
+ function formatArgValue(key, value) {
988
+ var pathKeys = { path: 1, file_path: 1, filePath: 1, target_file: 1, uri: 1, cwd: 1 };
989
+ if (typeof value === "string" && (pathKeys[key] || value.indexOf("/") === 0 || value.indexOf("file://") === 0)) {
990
+ return { text: displayToolPath(value), title: value };
991
+ }
992
+ var raw = typeof value === "string" ? value : JSON.stringify(value);
993
+ var oneLine = raw.replace(/\\n/g, " ↵ ");
994
+ return { text: oneLine.length > 140 ? oneLine.slice(0, 140) + "…" : oneLine };
995
+ }
996
+
222
997
  function appendTool(chat, name, args) {
223
998
  var row = document.createElement("div");
224
999
  row.className = "chat-row side-left";
@@ -241,7 +1016,11 @@ function clientScript() {
241
1016
  keyEl.className = "tool-arg-key";
242
1017
  keyEl.textContent = key;
243
1018
  var value = document.createElement("code");
244
- value.textContent = typeof args[key] === "string" ? args[key] : JSON.stringify(args[key]);
1019
+ var formatted = formatArgValue(key, args[key]);
1020
+ value.textContent = formatted.text;
1021
+ if (formatted.title && formatted.title !== formatted.text) {
1022
+ value.title = formatted.title;
1023
+ }
245
1024
  line.appendChild(keyEl);
246
1025
  line.appendChild(value);
247
1026
  box.appendChild(line);
@@ -253,101 +1032,542 @@ function clientScript() {
253
1032
  }
254
1033
 
255
1034
  var assistantBodies = {};
1035
+ var finishedMetrics = {};
1036
+
1037
+ function findScenario(suiteName, scenarioName) {
1038
+ for (var i = 0; i < catalog.suites.length; i++) {
1039
+ if (catalog.suites[i].name !== suiteName) {
1040
+ continue;
1041
+ }
1042
+ var scenarios = catalog.suites[i].scenarios;
1043
+ for (var j = 0; j < scenarios.length; j++) {
1044
+ if (scenarios[j].name === scenarioName) {
1045
+ return scenarios[j];
1046
+ }
1047
+ }
1048
+ }
1049
+ return null;
1050
+ }
1051
+
1052
+ function joinLabels(labels) {
1053
+ if (labels.length <= 1) {
1054
+ return labels[0] || "";
1055
+ }
1056
+ if (labels.length === 2) {
1057
+ return labels[0] + " and " + labels[1];
1058
+ }
1059
+ return labels.slice(0, -1).join(", ") + ", and " + labels[labels.length - 1];
1060
+ }
1061
+
1062
+ function summarizeMetric(arms, id, label) {
1063
+ var values = arms.map(function (arm) { return arm[id]; });
1064
+ var numeric = values.filter(function (value) { return typeof value === "number"; });
1065
+ if (numeric.length < 2) {
1066
+ return { line: label + ": n/a." };
1067
+ }
1068
+ var lowest = Math.min.apply(null, numeric);
1069
+ var winners = arms.filter(function (arm) { return arm[id] === lowest; });
1070
+ var shown = values.map(function (value) { return typeof value === "number" ? String(value) : "n/a"; }).join(" vs ");
1071
+ if (winners.length > 1) {
1072
+ return {
1073
+ line: arms.length === 2
1074
+ ? label + ": tie (" + shown + ")."
1075
+ : label + ": lowest is a tie between " + joinLabels(winners.map(function (arm) { return arm.label; })) + " (" + shown + ").",
1076
+ };
1077
+ }
1078
+ var winner = winners[0];
1079
+ if (!winner) {
1080
+ return { line: label + ": n/a." };
1081
+ }
1082
+ if (arms.length === 2) {
1083
+ return { line: label + ": " + winner.label + " wins (" + shown + ")." };
1084
+ }
1085
+ return { line: label + ": lowest is " + winner.label + " (" + shown + ")." };
1086
+ }
1087
+
1088
+ function summarizeGates(arms, gates) {
1089
+ var byId = {};
1090
+ arms.forEach(function (arm) { byId[arm.id] = arm; });
1091
+ return (gates || []).map(function (gate) {
1092
+ var field = gate.metric === "durationMs" ? "durationMs" : gate.metric;
1093
+ var winner = byId[gate.winner];
1094
+ var loser = byId[gate.loser];
1095
+ if (!winner || !loser || ["turns", "tokens", "tools", "durationMs"].indexOf(field) === -1) {
1096
+ return { passed: true, line: gate.metric + ": scored in the final result." };
1097
+ }
1098
+ var winnerValue = winner[field];
1099
+ var loserValue = loser[field];
1100
+ if (typeof winnerValue !== "number" || typeof loserValue !== "number") {
1101
+ return { passed: false, line: gate.metric + ": one arm did not report a value. Fail." };
1102
+ }
1103
+ var passed = winnerValue < loserValue;
1104
+ return { passed: passed, line: winner.label + " must beat " + loser.label + " on " + gate.metric + " (" + winnerValue + " vs " + loserValue + "). " + (passed ? "Pass." : "Fail.") };
1105
+ });
1106
+ }
1107
+
1108
+ function renderCompareWinners(event) {
1109
+ var scenario = findScenario(event.suite, event.scenario);
1110
+ if (!scenario || !scenario.compare || !scenario.compare.length) {
1111
+ return;
1112
+ }
1113
+ var wrap = document.getElementById(compareWrapId(event));
1114
+ if (!wrap) {
1115
+ return;
1116
+ }
1117
+ var rows = [];
1118
+ for (var i = 0; i < scenario.compare.length; i++) {
1119
+ var arm = scenario.compare[i];
1120
+ var metrics = finishedMetrics[[event.suite, event.scenario, event.host, arm.id].join("::")];
1121
+ if (!metrics) {
1122
+ return;
1123
+ }
1124
+ rows.push({
1125
+ id: arm.id,
1126
+ label: arm.label,
1127
+ turns: metrics.turns,
1128
+ tokens: metrics.tokens,
1129
+ tools: metrics.tools,
1130
+ durationMs: metrics.durationMs,
1131
+ passed: metrics.passed,
1132
+ });
1133
+ }
1134
+ var metricLines = [
1135
+ summarizeMetric(rows, "turns", "Turns"),
1136
+ summarizeMetric(rows, "tokens", "Tokens"),
1137
+ summarizeMetric(rows, "tools", "Tools"),
1138
+ summarizeMetric(rows, "durationMs", "Duration"),
1139
+ ];
1140
+ var gateLines = summarizeGates(rows, scenario.gates);
1141
+ var existing = wrap.querySelector(".compare-winners");
1142
+ if (existing) {
1143
+ existing.remove();
1144
+ }
1145
+ var section = document.createElement("section");
1146
+ section.className = "compare-winners";
1147
+ var heading = document.createElement("h3");
1148
+ heading.textContent = "Winners";
1149
+ var list = document.createElement("ul");
1150
+ metricLines.forEach(function (metric) {
1151
+ var item = document.createElement("li");
1152
+ item.textContent = metric.line;
1153
+ list.appendChild(item);
1154
+ });
1155
+ gateLines.forEach(function (gate) {
1156
+ var item = document.createElement("li");
1157
+ item.className = gate.passed ? "compare-winner-pass" : "compare-winner-fail";
1158
+ item.textContent = gate.line;
1159
+ list.appendChild(item);
1160
+ });
1161
+ section.appendChild(heading);
1162
+ section.appendChild(list);
1163
+ wrap.appendChild(section);
1164
+ }
256
1165
 
257
1166
  function markCell(event, label, statusClass) {
258
- var selector = '[data-cell="' + event.suite + "::" + event.scenario + "::" + event.host + '"] .cell-status';
1167
+ var selector = '[data-cell="' + event.suite + "::" + event.scenario + "::" + event.host + '"]';
259
1168
  document.querySelectorAll(selector).forEach(function (node) {
260
- node.textContent = label;
261
- node.className = "cell-status " + statusClass;
1169
+ var status = node.querySelector(".cell-status");
1170
+ if (status) {
1171
+ status.textContent = label;
1172
+ status.className = "cell-status " + statusClass;
1173
+ }
1174
+ if (node.classList.contains("host-tab")) {
1175
+ node.setAttribute("data-status", label);
1176
+ }
262
1177
  });
263
1178
  }
264
1179
 
1180
+ function metricPart(count, one, many) {
1181
+ if (typeof count !== "number") {
1182
+ return "";
1183
+ }
1184
+ return count + " " + (count === 1 ? one : many);
1185
+ }
1186
+
1187
+ function formatMetrics(metrics) {
1188
+ if (!metrics) {
1189
+ return "";
1190
+ }
1191
+ return [metricPart(metrics.turns, "turn", "turns"), metricPart(metrics.tokens, "token", "tokens"), metricPart(metrics.tools, "tool", "tools")]
1192
+ .filter(Boolean)
1193
+ .join(" · ");
1194
+ }
1195
+
1196
+ function clearCellResult(event) {
1197
+ var article = document.getElementById(paneId(event));
1198
+ var existing = article ? article.querySelector(".cell-result") : null;
1199
+ if (existing) {
1200
+ existing.remove();
1201
+ }
1202
+ }
1203
+
1204
+ function renderCellResult(event) {
1205
+ ensurePane(event);
1206
+ var article = document.getElementById(paneId(event));
1207
+ if (!article) {
1208
+ return;
1209
+ }
1210
+ var existing = article.querySelector(".cell-result");
1211
+ if (existing) {
1212
+ existing.remove();
1213
+ }
1214
+ var skipped = !!event.skipped;
1215
+ var passed = !!event.passed && !skipped;
1216
+ var box = document.createElement("div");
1217
+ box.className = "cell-result " + (skipped ? "status-skipped" : passed ? "status-passed" : "status-failed");
1218
+ var verdict = document.createElement("p");
1219
+ verdict.className = "cell-result-verdict";
1220
+ verdict.textContent = skipped ? "Skipped." : passed ? "Passed." : "Failed.";
1221
+ box.appendChild(verdict);
1222
+ var metrics = formatMetrics(event.metrics);
1223
+ if (metrics) {
1224
+ var line = document.createElement("p");
1225
+ line.className = "cell-result-metrics";
1226
+ line.textContent = metrics;
1227
+ box.appendChild(line);
1228
+ }
1229
+ if (!skipped && !passed) {
1230
+ var list = document.createElement("ul");
1231
+ list.className = "cell-result-failures";
1232
+ var failures = event.failures || [];
1233
+ if (failures.length === 0) {
1234
+ var item = document.createElement("li");
1235
+ item.textContent = "The run failed. The host did not report a reason.";
1236
+ list.appendChild(item);
1237
+ } else {
1238
+ failures.forEach(function (failure) {
1239
+ var item = document.createElement("li");
1240
+ item.textContent = failure.message || failure.matcher || "The run failed.";
1241
+ list.appendChild(item);
1242
+ });
1243
+ }
1244
+ box.appendChild(list);
1245
+ }
1246
+ article.appendChild(box);
1247
+ }
1248
+
1249
+ function appendJudgeVerdicts(event) {
1250
+ ensurePane(event);
1251
+ var article = document.getElementById(paneId(event));
1252
+ if (!article) {
1253
+ return;
1254
+ }
1255
+ var box = article.querySelector(".cell-result");
1256
+ if (!box) {
1257
+ box = document.createElement("div");
1258
+ box.className = "cell-result";
1259
+ article.appendChild(box);
1260
+ }
1261
+ var existing = box.querySelector(".cell-result-judges");
1262
+ if (existing) {
1263
+ existing.remove();
1264
+ }
1265
+ var list = document.createElement("ul");
1266
+ list.className = "cell-result-judges";
1267
+ (event.verdicts || []).forEach(function (verdict) {
1268
+ var item = document.createElement("li");
1269
+ item.className = verdict.pass ? "judge-pass" : "judge-fail";
1270
+ var text = (verdict.pass ? "Pass. " : "Fail. ") + (verdict.question || "");
1271
+ if (verdict.rationale) {
1272
+ text += " " + verdict.rationale;
1273
+ }
1274
+ item.textContent = text;
1275
+ list.appendChild(item);
1276
+ });
1277
+ box.appendChild(list);
1278
+ }
1279
+
265
1280
  function handleEvent(event) {
266
1281
  if (event.type === "run_started") {
1282
+ if (cancelRequested) {
1283
+ return;
1284
+ }
267
1285
  setRunBanner("Run " + event.runId + " started.");
268
- document.getElementById("cancel-run").disabled = false;
1286
+ setCancelButton("Cancel run", false);
269
1287
  return;
270
1288
  }
271
1289
  if (event.type === "run_finished") {
272
- setRunBanner(
273
- "Run finished. " + event.passed + " passed. " + event.failed + " failed. " + event.skipped + " skipped."
274
- );
275
- document.getElementById("cancel-run").disabled = true;
1290
+ finishProgress(event);
1291
+ hideAllRunning();
1292
+ sealAllChats();
1293
+ if (cancelRequested) {
1294
+ markCancelledCells();
1295
+ setRunBanner("Run cancelled.");
1296
+ } else {
1297
+ setRunBanner(
1298
+ "Run finished. " + progress.passed + " passed. " + progress.failed + " failed. " + progress.skipped + " skipped."
1299
+ );
1300
+ }
1301
+ setCancelButton("Cancel run", true);
1302
+ cancelRequested = false;
276
1303
  runId = null;
1304
+ if (source) {
1305
+ source.close();
1306
+ source = null;
1307
+ }
1308
+ return;
1309
+ }
1310
+ if (cancelRequested) {
277
1311
  return;
278
1312
  }
279
1313
  if (event.type === "error") {
280
1314
  setRunBanner(event.message);
1315
+ if (event.suite && chatIsOpen(event)) {
1316
+ appendStatus(event, event.message);
1317
+ var errorChat = ensurePane(event);
1318
+ if (errorChat) {
1319
+ appendBubble(errorChat, "system", event.message);
1320
+ }
1321
+ }
281
1322
  return;
282
1323
  }
283
1324
  if (event.type === "cell_started") {
284
1325
  markCell(event, "running", "status-skipped");
285
- ensurePane(event);
1326
+ startChat(event);
1327
+ if (event.arm) {
1328
+ var startedWrap = document.getElementById(compareWrapId(event));
1329
+ if (startedWrap) {
1330
+ markArmTab(startedWrap, event.arm, "running");
1331
+ if (startedWrap.classList.contains("compare-tabs") && startedWrap.dataset.userPicked !== "1") {
1332
+ selectCompareTab(startedWrap, event.arm);
1333
+ }
1334
+ }
1335
+ }
1336
+ return;
1337
+ }
1338
+ if (event.type === "status") {
1339
+ if (!chatIsOpen(event)) {
1340
+ return;
1341
+ }
1342
+ appendStatus(event, event.text);
1343
+ return;
1344
+ }
1345
+ if (event.type === "context") {
1346
+ if (!chatIsOpen(event)) {
1347
+ return;
1348
+ }
1349
+ var contextChat = ensurePane(event);
1350
+ if (contextChat) {
1351
+ appendContextFiles(contextChat, event.files);
1352
+ }
286
1353
  return;
287
1354
  }
288
1355
  if (event.type === "prompt") {
289
- appendBubble(ensurePane(event), "user", event.text);
1356
+ if (!chatIsOpen(event)) {
1357
+ return;
1358
+ }
1359
+ var promptChat = ensurePane(event);
1360
+ if (promptChat) {
1361
+ appendBubble(promptChat, "user", event.text);
1362
+ showRunning(event);
1363
+ }
290
1364
  return;
291
1365
  }
292
1366
  if (event.type === "text") {
1367
+ if (!chatIsOpen(event)) {
1368
+ return;
1369
+ }
293
1370
  var key = cellKey(event);
294
1371
  var chat = ensurePane(event);
1372
+ if (!chat) {
1373
+ return;
1374
+ }
1375
+ hideRunning(event);
295
1376
  if (!assistantBodies[key]) {
296
1377
  assistantBodies[key] = appendBubble(chat, "assistant", event.text);
297
1378
  } else {
298
1379
  assistantBodies[key].textContent = event.text;
299
1380
  }
1381
+ setStreaming(event, true);
300
1382
  return;
301
1383
  }
302
1384
  if (event.type === "tool") {
1385
+ if (!chatIsOpen(event)) {
1386
+ return;
1387
+ }
1388
+ var toolChat = ensurePane(event);
1389
+ if (!toolChat) {
1390
+ return;
1391
+ }
1392
+ setStreaming(event, false);
303
1393
  assistantBodies[cellKey(event)] = null;
304
- appendTool(ensurePane(event), event.name, event.args);
1394
+ hideRunning(event);
1395
+ appendTool(toolChat, event.name, event.args);
1396
+ showRunning(event);
305
1397
  return;
306
1398
  }
307
1399
  if (event.type === "cell_finished") {
1400
+ recordProgress(event);
1401
+ if (!chatIsOpen(event)) {
1402
+ return;
1403
+ }
1404
+ hideRunning(event);
1405
+ setStreaming(event, false);
308
1406
  markCell(event, event.skipped ? "skipped" : event.passed ? "passed" : "failed", event.skipped ? "status-skipped" : event.passed ? "status-passed" : "status-failed");
1407
+ finishedMetrics[cellKey(event)] = Object.assign({}, event.metrics || {}, {
1408
+ durationMs: event.durationMs,
1409
+ passed: event.passed,
1410
+ });
1411
+ renderCellResult(event);
1412
+ if (event.arm) {
1413
+ var finishedWrap = document.getElementById(compareWrapId(event));
1414
+ if (finishedWrap) {
1415
+ markArmTab(
1416
+ finishedWrap,
1417
+ event.arm,
1418
+ event.skipped ? "skipped" : event.passed ? "passed" : "failed"
1419
+ );
1420
+ }
1421
+ renderCompareWinners(event);
1422
+ }
1423
+ return;
1424
+ }
1425
+ if (event.type === "judge") {
1426
+ if (!chatIsOpen(event)) {
1427
+ return;
1428
+ }
1429
+ appendJudgeVerdicts(event);
309
1430
  }
310
1431
  }
311
1432
 
1433
+ function markRunTargets(body, label, statusClass) {
1434
+ var hosts = body.hosts && body.hosts.length ? body.hosts : catalog.defaultSelectedHosts;
1435
+ catalog.suites.forEach(function (suite) {
1436
+ if (body.suite && suite.name !== body.suite) {
1437
+ return;
1438
+ }
1439
+ suite.scenarios.forEach(function (scenario) {
1440
+ if (body.scenario && scenario.name !== body.scenario) {
1441
+ return;
1442
+ }
1443
+ if (scenario.skip) {
1444
+ return;
1445
+ }
1446
+ hosts.forEach(function (host) {
1447
+ if (scenario.host && scenario.host !== host) {
1448
+ return;
1449
+ }
1450
+ markCell({ suite: suite.name, scenario: scenario.name, host: host }, label, statusClass);
1451
+ });
1452
+ });
1453
+ });
1454
+ }
1455
+
312
1456
  async function startRun(body) {
313
1457
  if (runId) {
314
- setRunBanner("A run is already in progress.");
1458
+ setRunBanner(
1459
+ cancelRequested
1460
+ ? "Cancelling the run. Wait for it to stop."
1461
+ : "A run is already in progress. Cancel it first."
1462
+ );
315
1463
  return;
316
1464
  }
317
- var response = await fetch("/api/runs", {
318
- method: "POST",
319
- headers: { "content-type": "application/json" },
320
- body: JSON.stringify(body),
1465
+ resetProgress(body);
1466
+ markRunTargets(body, "starting", "status-skipped");
1467
+ var firstCard = document.querySelector(
1468
+ body.suite && body.scenario
1469
+ ? '[data-scenario-card="' + body.suite + "::" + body.scenario + '"]'
1470
+ : body.suite
1471
+ ? '[data-suite="' + body.suite + '"] [data-scenario-card]'
1472
+ : "[data-scenario-card]"
1473
+ );
1474
+ document.querySelectorAll("[data-scenario-card]").forEach(function (card) {
1475
+ if (body.suite && card.getAttribute("data-scenario-card").indexOf(body.suite + "::") !== 0) {
1476
+ return;
1477
+ }
1478
+ if (body.scenario && card.getAttribute("data-scenario-card") !== body.suite + "::" + body.scenario) {
1479
+ return;
1480
+ }
1481
+ delete card.dataset.followHost;
321
1482
  });
322
- var payload = await response.json();
323
- if (!response.ok) {
324
- setRunBanner(payload.error || "Run failed to start.");
325
- return;
1483
+ if (firstCard) {
1484
+ var live = firstCard.querySelector(".scenario-live");
1485
+ if (live) {
1486
+ live.hidden = false;
1487
+ }
1488
+ if (body.hosts && body.hosts.length === 1 && firstCard.dataset.userPickedHost !== "1") {
1489
+ firstCard.dataset.followHost = body.hosts[0];
1490
+ selectHostTab(firstCard, body.hosts[0]);
1491
+ }
1492
+ if (live) {
1493
+ live.scrollIntoView({ block: "nearest" });
1494
+ }
326
1495
  }
327
- runId = payload.runId;
328
- if (source) {
329
- source.close();
1496
+ try {
1497
+ var response = await fetch("/api/runs", {
1498
+ method: "POST",
1499
+ headers: { "content-type": "application/json" },
1500
+ body: JSON.stringify(body),
1501
+ });
1502
+ var payload = await response.json();
1503
+ if (!response.ok) {
1504
+ setRunBanner(payload.error || "Run failed to start.");
1505
+ progress = { total: 0, completed: 0, passed: 0, failed: 0, skipped: 0, seen: {} };
1506
+ renderProgress();
1507
+ markRunTargets(body, "idle", "muted");
1508
+ return;
1509
+ }
1510
+ runId = payload.runId;
1511
+ cancelRequested = false;
1512
+ setRunBanner("Run " + runId + " started.");
1513
+ setCancelButton("Cancel run", false);
1514
+ if (source) {
1515
+ source.close();
1516
+ }
1517
+ source = new EventSource("/api/runs/" + runId + "/events");
1518
+ source.onmessage = function (message) {
1519
+ handleEvent(JSON.parse(message.data));
1520
+ };
1521
+ source.onerror = function () {
1522
+ source.close();
1523
+ if (!runId) {
1524
+ return;
1525
+ }
1526
+ setRunBanner("Live event stream dropped.");
1527
+ };
1528
+ } catch (error) {
1529
+ setRunBanner(error instanceof Error ? error.message : String(error));
1530
+ markRunTargets(body, "idle", "muted");
330
1531
  }
331
- source = new EventSource("/api/runs/" + runId + "/events");
332
- source.onmessage = function (message) {
333
- handleEvent(JSON.parse(message.data));
334
- };
335
1532
  }
336
1533
 
337
1534
  document.getElementById("run-selection").addEventListener("click", function () {
338
1535
  startRun({ hosts: selectedHosts(), parallelHosts: parallelHosts() });
339
1536
  });
340
1537
  document.getElementById("cancel-run").addEventListener("click", function () {
341
- if (!runId) {
1538
+ if (!runId || cancelRequested) {
342
1539
  return;
343
1540
  }
344
- fetch("/api/runs/" + runId + "/cancel", { method: "POST" });
1541
+ cancelRequested = true;
1542
+ setRunBanner("Cancelling the run.");
1543
+ setCancelButton("Cancelling", true);
1544
+ cancelOpenChats();
1545
+ fetch("/api/runs/" + runId + "/cancel", { method: "POST" }).then(function (response) {
1546
+ if (!response.ok && cancelRequested) {
1547
+ setRunBanner("Cancel failed.");
1548
+ setCancelButton("Cancel run", false);
1549
+ cancelRequested = false;
1550
+ }
1551
+ }).catch(function () {
1552
+ if (cancelRequested) {
1553
+ setRunBanner("Cancel failed.");
1554
+ setCancelButton("Cancel run", false);
1555
+ cancelRequested = false;
1556
+ }
1557
+ });
345
1558
  });
346
1559
  document.body.addEventListener("click", function (event) {
347
1560
  var button = event.target.closest("button");
348
1561
  if (!button) {
349
1562
  return;
350
1563
  }
1564
+ if (button.classList.contains("host-tab")) {
1565
+ var card = button.closest("[data-scenario-card]");
1566
+ if (card) {
1567
+ card.dataset.userPickedHost = "1";
1568
+ selectHostTab(card, button.getAttribute("data-host"));
1569
+ }
1570
+ }
351
1571
  if (button.classList.contains("run-cell")) {
352
1572
  startRun({
353
1573
  suite: button.getAttribute("data-suite"),
@@ -408,7 +1628,7 @@ ${viewerCss()}
408
1628
  <div class="host-toggles">${hostToggles}</div>
409
1629
  </div>
410
1630
  <div class="toolbar-block">
411
- <p class="toolbar-label">Matrix</p>
1631
+ <p class="toolbar-label">Run order</p>
412
1632
  <label><input type="checkbox" id="parallel-hosts"> Run hosts together</label>
413
1633
  </div>
414
1634
  <div class="toolbar-actions">
@@ -417,8 +1637,21 @@ ${viewerCss()}
417
1637
  </div>
418
1638
  <p class="run-banner" id="run-banner"></p>
419
1639
  </section>
1640
+ <section class="run-progress" id="run-progress" aria-live="polite" hidden>
1641
+ <div class="run-progress-head">
1642
+ <p class="run-progress-title">0 of 0 tests finished</p>
1643
+ </div>
1644
+ <div class="run-progress-track" role="progressbar" aria-label="Suite run progress" aria-valuemin="0" aria-valuenow="0" aria-valuemax="0">
1645
+ <span class="run-progress-fill"></span>
1646
+ </div>
1647
+ <div class="run-progress-counts">
1648
+ <span class="progress-passed">Passed <strong>0</strong></span>
1649
+ <span class="progress-failed">Failed <strong>0</strong></span>
1650
+ <span class="progress-skipped">Skipped <strong>0</strong></span>
1651
+ <span class="progress-remaining">Remaining <strong>0</strong></span>
1652
+ </div>
1653
+ </section>
420
1654
  ${suites}
421
- <section class="live-dock" id="live-dock"></section>
422
1655
  </main>
423
1656
  <script type="application/json" id="catalog-data">${embedJson(catalog)}</script>
424
1657
  <script>