@post-print/agent-test 0.3.13 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -4
- package/dist/cli.js +2 -2
- package/dist/cli.js.map +1 -1
- package/dist/compare-scenario.d.ts +13 -19
- package/dist/compare-scenario.d.ts.map +1 -1
- package/dist/compare-scenario.js +141 -261
- package/dist/compare-scenario.js.map +1 -1
- package/dist/debug-bundle.d.ts.map +1 -1
- package/dist/debug-bundle.js +2 -0
- package/dist/debug-bundle.js.map +1 -1
- package/dist/expect.d.ts +1 -0
- package/dist/expect.d.ts.map +1 -1
- package/dist/expect.js +19 -0
- package/dist/expect.js.map +1 -1
- package/dist/html-report.d.ts.map +1 -1
- package/dist/html-report.js +185 -38
- package/dist/html-report.js.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/live-isolation.d.ts +2 -0
- package/dist/live-isolation.d.ts.map +1 -1
- package/dist/live-isolation.js +66 -6
- package/dist/live-isolation.js.map +1 -1
- package/dist/run-suite.d.ts.map +1 -1
- package/dist/run-suite.js +94 -23
- package/dist/run-suite.js.map +1 -1
- package/dist/scenario-story.d.ts +4 -3
- package/dist/scenario-story.d.ts.map +1 -1
- package/dist/scenario-story.js +47 -71
- package/dist/scenario-story.js.map +1 -1
- package/dist/scenario-usage.d.ts.map +1 -1
- package/dist/scenario-usage.js +2 -3
- package/dist/scenario-usage.js.map +1 -1
- package/dist/types.d.ts +32 -22
- package/dist/types.d.ts.map +1 -1
- package/dist/validate-suite.d.ts.map +1 -1
- package/dist/validate-suite.js +64 -38
- package/dist/validate-suite.js.map +1 -1
- package/dist/viewer/catalog.d.ts +3 -1
- package/dist/viewer/catalog.d.ts.map +1 -1
- package/dist/viewer/catalog.js +9 -0
- package/dist/viewer/catalog.js.map +1 -1
- package/dist/viewer/compare-metrics.d.ts +30 -0
- package/dist/viewer/compare-metrics.d.ts.map +1 -0
- package/dist/viewer/compare-metrics.js +102 -0
- package/dist/viewer/compare-metrics.js.map +1 -0
- package/dist/viewer/context-files.d.ts +8 -0
- package/dist/viewer/context-files.d.ts.map +1 -0
- package/dist/viewer/context-files.js +56 -0
- package/dist/viewer/context-files.js.map +1 -0
- package/dist/viewer/events.d.ts +20 -0
- package/dist/viewer/events.d.ts.map +1 -1
- package/dist/viewer/events.js +2 -0
- package/dist/viewer/events.js.map +1 -1
- package/dist/viewer/live-runner.d.ts +6 -0
- package/dist/viewer/live-runner.d.ts.map +1 -1
- package/dist/viewer/live-runner.js +34 -5
- package/dist/viewer/live-runner.js.map +1 -1
- package/dist/viewer/page.d.ts.map +1 -1
- package/dist/viewer/page.js +1325 -92
- package/dist/viewer/page.js.map +1 -1
- package/dist/viewer/run-controller.d.ts +5 -0
- package/dist/viewer/run-controller.d.ts.map +1 -1
- package/dist/viewer/run-controller.js +27 -1
- package/dist/viewer/run-controller.js.map +1 -1
- package/dist/viewer/server.d.ts.map +1 -1
- package/dist/viewer/server.js +18 -11
- package/dist/viewer/server.js.map +1 -1
- package/package.json +7 -5
package/dist/viewer/page.js
CHANGED
|
@@ -41,19 +41,189 @@ function viewerCss() {
|
|
|
41
41
|
button.danger { border-color: color-mix(in srgb, var(--fail) 40%, var(--border)); color: var(--fail); }
|
|
42
42
|
button:disabled { opacity: 0.5; cursor: not-allowed; }
|
|
43
43
|
.warn { color: var(--skip); font-size: 0.8rem; }
|
|
44
|
-
.
|
|
45
|
-
.
|
|
46
|
-
|
|
44
|
+
.scenario-list { display: grid; gap: 0.75rem; }
|
|
45
|
+
.scenario-card {
|
|
46
|
+
background: var(--panel);
|
|
47
|
+
border: 1px solid var(--border);
|
|
48
|
+
border-radius: 12px;
|
|
49
|
+
padding: 0.85rem 0.95rem;
|
|
50
|
+
display: grid;
|
|
51
|
+
gap: 0.7rem;
|
|
52
|
+
}
|
|
47
53
|
.scenario-detail { display: grid; gap: 0.35rem; }
|
|
48
54
|
.prompt-preview { color: var(--muted); font-size: 0.78rem; white-space: pre-wrap; }
|
|
49
55
|
.rubric-line { font-size: 0.75rem; color: var(--muted); }
|
|
50
56
|
.arm-note { font-size: 0.78rem; color: var(--muted); }
|
|
51
|
-
.
|
|
57
|
+
.scenario-toolbar { display: flex; flex-wrap: wrap; gap: 0.55rem 0.85rem; align-items: center; }
|
|
58
|
+
.scenario-actions { display: flex; flex-wrap: wrap; gap: 0.35rem; align-items: center; }
|
|
59
|
+
.host-tablist { display: flex; flex-wrap: wrap; gap: 0.35rem; }
|
|
60
|
+
.host-tab {
|
|
61
|
+
display: inline-flex;
|
|
62
|
+
align-items: center;
|
|
63
|
+
gap: 0.4rem;
|
|
64
|
+
}
|
|
65
|
+
.host-tab[aria-selected="true"] {
|
|
66
|
+
background: color-mix(in srgb, var(--pass) 16%, var(--panel-2));
|
|
67
|
+
border-color: color-mix(in srgb, var(--pass) 40%, var(--border));
|
|
68
|
+
}
|
|
69
|
+
.host-tab[data-status="running"],
|
|
70
|
+
.host-tab[data-status="cancelling"],
|
|
71
|
+
.host-tab[data-status="cancelled"] { color: var(--skip); }
|
|
72
|
+
.host-tab[data-status="passed"] { color: var(--pass); }
|
|
73
|
+
.host-tab[data-status="failed"] { color: var(--fail); }
|
|
74
|
+
.host-tab[data-status="skipped"], .host-tab[data-status="skip"] { color: var(--muted); }
|
|
75
|
+
.host-status { display: inline-flex; align-items: center; }
|
|
52
76
|
.cell-status { font-size: 0.72rem; font-weight: 700; text-transform: uppercase; }
|
|
53
|
-
.live
|
|
54
|
-
.live-
|
|
77
|
+
.scenario-live { display: grid; gap: 0.75rem; }
|
|
78
|
+
.scenario-live[hidden], .host-panel[hidden], .host-empty[hidden] { display: none; }
|
|
79
|
+
.host-panels { display: grid; min-width: 0; }
|
|
80
|
+
.host-panel:not([hidden]) { min-height: 4rem; }
|
|
81
|
+
.host-empty { color: var(--muted); font-size: 0.85rem; margin: 0; }
|
|
82
|
+
.host-panel > .live-cell > h3 { display: none; }
|
|
83
|
+
.live-cell { background: var(--panel-2); border: 1px solid var(--border); border-radius: 12px; padding: 0.75rem; min-width: 0; }
|
|
84
|
+
.cell-result {
|
|
85
|
+
margin-top: 0.65rem;
|
|
86
|
+
border-left: 3px solid var(--border);
|
|
87
|
+
padding: 0.45rem 0.7rem;
|
|
88
|
+
}
|
|
89
|
+
.cell-result.status-passed { border-left-color: var(--pass); }
|
|
90
|
+
.cell-result.status-failed { border-left-color: var(--fail); }
|
|
91
|
+
.cell-result.status-skipped { border-left-color: var(--skip); }
|
|
92
|
+
.cell-result-verdict { font-weight: 650; margin: 0 0 0.25rem; }
|
|
93
|
+
.cell-result.status-passed .cell-result-verdict { color: var(--pass); }
|
|
94
|
+
.cell-result.status-failed .cell-result-verdict { color: var(--fail); }
|
|
95
|
+
.cell-result-metrics { color: var(--muted); font-size: 0.8rem; margin: 0 0 0.35rem; }
|
|
96
|
+
.cell-result-failures, .cell-result-judges {
|
|
97
|
+
list-style: none;
|
|
98
|
+
margin: 0;
|
|
99
|
+
padding: 0;
|
|
100
|
+
display: grid;
|
|
101
|
+
gap: 0.35rem;
|
|
102
|
+
}
|
|
103
|
+
.cell-result-judges { margin-top: 0.45rem; }
|
|
104
|
+
.cell-result-failures li, .cell-result-judges li {
|
|
105
|
+
font-size: 0.85rem;
|
|
106
|
+
white-space: pre-wrap;
|
|
107
|
+
word-break: break-word;
|
|
108
|
+
}
|
|
109
|
+
.judge-pass { color: var(--pass); }
|
|
110
|
+
.judge-fail { color: var(--fail); }
|
|
55
111
|
.live-cell h3 { margin-bottom: 0.45rem; }
|
|
112
|
+
.live-status {
|
|
113
|
+
list-style: none;
|
|
114
|
+
margin: 0 0 0.55rem;
|
|
115
|
+
padding: 0;
|
|
116
|
+
display: grid;
|
|
117
|
+
gap: 0.2rem;
|
|
118
|
+
max-height: 9rem;
|
|
119
|
+
overflow: auto;
|
|
120
|
+
font-size: 0.75rem;
|
|
121
|
+
color: var(--muted);
|
|
122
|
+
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
|
|
123
|
+
}
|
|
124
|
+
.live-status li {
|
|
125
|
+
border-left: 2px solid var(--border);
|
|
126
|
+
padding: 0.12rem 0 0.12rem 0.5rem;
|
|
127
|
+
white-space: pre-wrap;
|
|
128
|
+
word-break: break-word;
|
|
129
|
+
}
|
|
56
130
|
.run-banner { font-size: 0.85rem; color: var(--muted); }
|
|
131
|
+
.run-progress {
|
|
132
|
+
display: grid;
|
|
133
|
+
gap: 0.55rem;
|
|
134
|
+
background: var(--panel);
|
|
135
|
+
border: 1px solid var(--border);
|
|
136
|
+
border-radius: 12px;
|
|
137
|
+
padding: 0.8rem 1rem;
|
|
138
|
+
margin-bottom: 1.25rem;
|
|
139
|
+
}
|
|
140
|
+
.run-progress[hidden] { display: none; }
|
|
141
|
+
.run-progress-head, .run-progress-counts {
|
|
142
|
+
display: flex;
|
|
143
|
+
flex-wrap: wrap;
|
|
144
|
+
align-items: center;
|
|
145
|
+
justify-content: space-between;
|
|
146
|
+
gap: 0.4rem 1rem;
|
|
147
|
+
}
|
|
148
|
+
.run-progress-title { margin: 0; font-weight: 700; }
|
|
149
|
+
.run-progress-counts { justify-content: flex-start; color: var(--muted); font-size: 0.8rem; }
|
|
150
|
+
.run-progress-counts strong { color: var(--text); font-variant-numeric: tabular-nums; }
|
|
151
|
+
.run-progress-counts .progress-passed strong { color: var(--pass); }
|
|
152
|
+
.run-progress-counts .progress-failed strong { color: var(--fail); }
|
|
153
|
+
.run-progress-counts .progress-skipped strong { color: var(--skip); }
|
|
154
|
+
.run-progress-track {
|
|
155
|
+
height: 0.7rem;
|
|
156
|
+
overflow: hidden;
|
|
157
|
+
border-radius: 999px;
|
|
158
|
+
background: var(--panel-2);
|
|
159
|
+
border: 1px solid var(--border);
|
|
160
|
+
}
|
|
161
|
+
.run-progress-fill {
|
|
162
|
+
display: block;
|
|
163
|
+
width: 0;
|
|
164
|
+
height: 100%;
|
|
165
|
+
background: var(--pass);
|
|
166
|
+
transition: width 180ms ease-out;
|
|
167
|
+
}
|
|
168
|
+
.run-progress[data-has-failures="true"] .run-progress-fill {
|
|
169
|
+
background: linear-gradient(90deg, var(--pass), var(--fail));
|
|
170
|
+
}
|
|
171
|
+
.compare-tab[data-status="running"] { color: var(--skip); }
|
|
172
|
+
.compare-tab[data-status="passed"] { color: var(--pass); }
|
|
173
|
+
.compare-tab[data-status="failed"] { color: var(--fail); }
|
|
174
|
+
.compare-tab[data-status="skipped"] { color: var(--muted); }
|
|
175
|
+
.compare-tabs .compare-arms,
|
|
176
|
+
.compare-tabs .compare-arms[data-arm-count] { grid-template-columns: minmax(0, 1fr); }
|
|
177
|
+
.compare-tabs .live-cell[hidden] { display: none; }
|
|
178
|
+
.compare-tabs .live-cell > h3 { display: none; }
|
|
179
|
+
.chat-running-row[hidden] { display: none; }
|
|
180
|
+
.context-path {
|
|
181
|
+
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
|
|
182
|
+
font-size: 0.82rem;
|
|
183
|
+
margin: 0 0 0.25rem;
|
|
184
|
+
}
|
|
185
|
+
.context-why {
|
|
186
|
+
color: var(--muted);
|
|
187
|
+
font-size: 0.8rem;
|
|
188
|
+
margin: 0 0 0.45rem;
|
|
189
|
+
}
|
|
190
|
+
.chat-running {
|
|
191
|
+
display: inline-flex;
|
|
192
|
+
align-items: center;
|
|
193
|
+
gap: 0.45rem;
|
|
194
|
+
color: var(--muted);
|
|
195
|
+
font-size: 0.85rem;
|
|
196
|
+
padding: 0.2rem 0.1rem;
|
|
197
|
+
}
|
|
198
|
+
.chat-progress {
|
|
199
|
+
display: inline-flex;
|
|
200
|
+
gap: 0.18rem;
|
|
201
|
+
}
|
|
202
|
+
.chat-progress span {
|
|
203
|
+
width: 0.32rem;
|
|
204
|
+
height: 0.32rem;
|
|
205
|
+
border-radius: 999px;
|
|
206
|
+
background: var(--skip);
|
|
207
|
+
animation: chat-progress 1s ease-in-out infinite;
|
|
208
|
+
}
|
|
209
|
+
.chat-progress span:nth-child(2) { animation-delay: 0.15s; }
|
|
210
|
+
.chat-progress span:nth-child(3) { animation-delay: 0.3s; }
|
|
211
|
+
@keyframes chat-progress {
|
|
212
|
+
0%, 80%, 100% { opacity: 0.2; }
|
|
213
|
+
40% { opacity: 1; }
|
|
214
|
+
}
|
|
215
|
+
.bubble.is-streaming .bubble-text::after {
|
|
216
|
+
content: "▍";
|
|
217
|
+
margin-left: 0.12rem;
|
|
218
|
+
color: var(--pass);
|
|
219
|
+
animation: chat-caret 1s step-end infinite;
|
|
220
|
+
}
|
|
221
|
+
@keyframes chat-caret {
|
|
222
|
+
50% { opacity: 0; }
|
|
223
|
+
}
|
|
224
|
+
@media (prefers-reduced-motion: reduce) {
|
|
225
|
+
.chat-progress span, .bubble.is-streaming .bubble-text::after { animation: none; }
|
|
226
|
+
}
|
|
57
227
|
`;
|
|
58
228
|
}
|
|
59
229
|
function rubricSummary(scenario) {
|
|
@@ -85,43 +255,79 @@ function rubricSummary(scenario) {
|
|
|
85
255
|
if (rubric.judge) {
|
|
86
256
|
parts.push("judge");
|
|
87
257
|
}
|
|
258
|
+
if (scenario.contextSources?.length) {
|
|
259
|
+
parts.push(`contextSources ${scenario.contextSources.join(", ")}`);
|
|
260
|
+
}
|
|
88
261
|
return parts.join(" · ");
|
|
89
262
|
}
|
|
90
|
-
function
|
|
91
|
-
|
|
263
|
+
function defaultScenarioHost(suite, scenario) {
|
|
264
|
+
if (scenario.host && suite.hosts.includes(scenario.host)) {
|
|
265
|
+
return scenario.host;
|
|
266
|
+
}
|
|
267
|
+
if (suite.hosts.includes("cursor")) {
|
|
268
|
+
return "cursor";
|
|
269
|
+
}
|
|
270
|
+
return suite.hosts[0] ?? "cursor";
|
|
271
|
+
}
|
|
272
|
+
function hostIsSkipped(scenario, host) {
|
|
273
|
+
return Boolean(scenario.skip || (scenario.host && scenario.host !== host));
|
|
274
|
+
}
|
|
275
|
+
function renderHostChrome(suite, scenario, selectedHost) {
|
|
276
|
+
if (suite.hosts.length <= 1) {
|
|
277
|
+
const host = suite.hosts[0] ?? selectedHost;
|
|
278
|
+
const skipped = hostIsSkipped(scenario, host);
|
|
279
|
+
return `<div class="host-status" data-cell="${escapeHtml(`${suite.name}::${scenario.name}::${host}`)}">
|
|
280
|
+
<span class="cell-status muted">${skipped ? "skip" : "idle"}</span>
|
|
281
|
+
</div>`;
|
|
282
|
+
}
|
|
283
|
+
const tabs = suite.hosts
|
|
92
284
|
.map((host) => {
|
|
93
|
-
const
|
|
94
|
-
const
|
|
95
|
-
return `<
|
|
96
|
-
<div class="cell-actions">
|
|
97
|
-
<button class="primary run-cell" data-suite="${escapeHtml(suite.name)}" data-scenario="${escapeHtml(scenario.name)}" data-host="${escapeHtml(host)}"${skipped ? " disabled" : ""}>Run</button>
|
|
98
|
-
<span class="cell-status muted">${skipped ? "skip" : "idle"}</span>
|
|
99
|
-
</div>
|
|
100
|
-
</td>`;
|
|
285
|
+
const skipped = hostIsSkipped(scenario, host);
|
|
286
|
+
const selected = host === selectedHost;
|
|
287
|
+
return `<button type="button" class="host-tab" role="tab" aria-label="${escapeHtml(host)}" data-cell="${escapeHtml(`${suite.name}::${scenario.name}::${host}`)}" data-host="${escapeHtml(host)}" aria-selected="${selected ? "true" : "false"}"${skipped ? " disabled" : ""}>${escapeHtml(host)} <span class="cell-status muted" aria-hidden="true">${skipped ? "skip" : "idle"}</span></button>`;
|
|
101
288
|
})
|
|
102
289
|
.join("");
|
|
290
|
+
return `<div class="host-tablist" role="tablist">${tabs}</div>`;
|
|
291
|
+
}
|
|
292
|
+
function renderHostPanels(suite, selectedHost) {
|
|
293
|
+
return suite.hosts
|
|
294
|
+
.map((host) => {
|
|
295
|
+
const selected = host === selectedHost;
|
|
296
|
+
return `<div class="host-panel" data-host-panel="${escapeHtml(host)}" role="tabpanel"${selected ? "" : " hidden"}><p class="host-empty">No run yet.</p></div>`;
|
|
297
|
+
})
|
|
298
|
+
.join("");
|
|
299
|
+
}
|
|
300
|
+
function renderScenarioCard(suite, scenario) {
|
|
301
|
+
const selectedHost = defaultScenarioHost(suite, scenario);
|
|
302
|
+
const runDisabled = hostIsSkipped(scenario, selectedHost);
|
|
103
303
|
const compare = scenario.compare
|
|
104
304
|
? scenario.compare
|
|
105
305
|
.map((arm) => `<p class="arm-note">${escapeHtml(arm.label)}: ${escapeHtml(arm.description ?? arm.prompt ?? "")}</p>`)
|
|
106
306
|
.join("")
|
|
107
307
|
: "";
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
308
|
+
const key = escapeHtml(`${suite.name}::${scenario.name}`);
|
|
309
|
+
return `<article class="scenario-card" data-scenario-card="${key}">
|
|
310
|
+
<div class="scenario-detail">
|
|
311
|
+
<strong>${escapeHtml(scenario.name)}</strong>
|
|
312
|
+
${scenario.description ? `<p class="scenario-lede">${escapeHtml(scenario.description)}</p>` : ""}
|
|
313
|
+
<p class="prompt-preview">${escapeHtml(scenario.prompt)}</p>
|
|
314
|
+
<p class="rubric-line">${escapeHtml(rubricSummary(scenario))}</p>
|
|
315
|
+
${compare}
|
|
316
|
+
</div>
|
|
317
|
+
<div class="scenario-toolbar">
|
|
318
|
+
${renderHostChrome(suite, scenario, selectedHost)}
|
|
319
|
+
<div class="scenario-actions">
|
|
320
|
+
<button class="primary run-cell" data-suite="${escapeHtml(suite.name)}" data-scenario="${escapeHtml(scenario.name)}" data-host="${escapeHtml(selectedHost)}"${runDisabled ? " disabled" : ""}>Run</button>
|
|
321
|
+
<button class="run-row" data-suite="${escapeHtml(suite.name)}" data-scenario="${escapeHtml(scenario.name)}"${scenario.skip ? " disabled" : ""}>Run selected hosts</button>
|
|
117
322
|
</div>
|
|
118
|
-
</
|
|
119
|
-
${
|
|
120
|
-
|
|
323
|
+
</div>
|
|
324
|
+
<div class="scenario-live" hidden data-live-row="${key}" data-live-slot="${key}">
|
|
325
|
+
<div class="host-panels">${renderHostPanels(suite, selectedHost)}</div>
|
|
326
|
+
</div>
|
|
327
|
+
</article>`;
|
|
121
328
|
}
|
|
122
329
|
function renderSuiteSection(suite) {
|
|
123
|
-
const
|
|
124
|
-
const rows = suite.scenarios.map((scenario) => renderScenarioRow(suite, scenario)).join("");
|
|
330
|
+
const cards = suite.scenarios.map((scenario) => renderScenarioCard(suite, scenario)).join("");
|
|
125
331
|
return `
|
|
126
332
|
<section class="suite" data-suite="${escapeHtml(suite.name)}">
|
|
127
333
|
<header class="suite-header">
|
|
@@ -131,10 +337,7 @@ function renderSuiteSection(suite) {
|
|
|
131
337
|
</div>
|
|
132
338
|
<button class="run-suite" data-suite="${escapeHtml(suite.name)}">Run suite</button>
|
|
133
339
|
</header>
|
|
134
|
-
<
|
|
135
|
-
<thead><tr><th>Scenario</th>${columns}</tr></thead>
|
|
136
|
-
<tbody>${rows}</tbody>
|
|
137
|
-
</table>
|
|
340
|
+
<div class="scenario-list">${cards}</div>
|
|
138
341
|
</section>`;
|
|
139
342
|
}
|
|
140
343
|
function clientScript() {
|
|
@@ -143,6 +346,8 @@ function clientScript() {
|
|
|
143
346
|
var catalog = JSON.parse(document.getElementById("catalog-data").textContent);
|
|
144
347
|
var source = null;
|
|
145
348
|
var runId = null;
|
|
349
|
+
var cancelRequested = false;
|
|
350
|
+
var progress = { total: 0, completed: 0, passed: 0, failed: 0, skipped: 0, seen: {}, experiments: {} };
|
|
146
351
|
|
|
147
352
|
function selectedHosts() {
|
|
148
353
|
return Array.prototype.map.call(document.querySelectorAll("[data-host-toggle]:checked"), function (box) {
|
|
@@ -158,6 +363,134 @@ function clientScript() {
|
|
|
158
363
|
document.getElementById("run-banner").textContent = text;
|
|
159
364
|
}
|
|
160
365
|
|
|
366
|
+
function runnableTargets(body) {
|
|
367
|
+
var requestedHosts = body.hosts && body.hosts.length ? body.hosts : catalog.defaultSelectedHosts;
|
|
368
|
+
var total = 0;
|
|
369
|
+
catalog.suites.forEach(function (suite) {
|
|
370
|
+
if (body.suite && suite.name !== body.suite) {
|
|
371
|
+
return;
|
|
372
|
+
}
|
|
373
|
+
suite.scenarios.forEach(function (scenario) {
|
|
374
|
+
if (body.scenario && scenario.name !== body.scenario) {
|
|
375
|
+
return;
|
|
376
|
+
}
|
|
377
|
+
if (scenario.skip) {
|
|
378
|
+
return;
|
|
379
|
+
}
|
|
380
|
+
requestedHosts.forEach(function (host) {
|
|
381
|
+
if (suite.hosts.indexOf(host) === -1 || (scenario.host && scenario.host !== host)) {
|
|
382
|
+
return;
|
|
383
|
+
}
|
|
384
|
+
total += 1;
|
|
385
|
+
});
|
|
386
|
+
});
|
|
387
|
+
});
|
|
388
|
+
return total;
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
function renderProgress() {
|
|
392
|
+
var box = document.getElementById("run-progress");
|
|
393
|
+
var total = Math.max(progress.total, progress.completed);
|
|
394
|
+
var remaining = Math.max(0, total - progress.completed);
|
|
395
|
+
box.hidden = total === 0;
|
|
396
|
+
box.setAttribute("data-has-failures", progress.failed > 0 ? "true" : "false");
|
|
397
|
+
var testLabel = total === 1 ? "test" : "tests";
|
|
398
|
+
box.querySelector(".run-progress-title").textContent = progress.completed + " of " + total + " " + testLabel + " finished";
|
|
399
|
+
box.querySelector(".progress-passed strong").textContent = String(progress.passed);
|
|
400
|
+
box.querySelector(".progress-failed strong").textContent = String(progress.failed);
|
|
401
|
+
box.querySelector(".progress-skipped strong").textContent = String(progress.skipped);
|
|
402
|
+
box.querySelector(".progress-remaining strong").textContent = String(remaining);
|
|
403
|
+
var percent = total > 0 ? Math.round((progress.completed / total) * 100) : 0;
|
|
404
|
+
var fill = box.querySelector(".run-progress-fill");
|
|
405
|
+
fill.style.width = percent + "%";
|
|
406
|
+
fill.parentNode.setAttribute("aria-valuenow", String(progress.completed));
|
|
407
|
+
fill.parentNode.setAttribute("aria-valuemax", String(total));
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
function resetProgress(body) {
|
|
411
|
+
progress = { total: runnableTargets(body), completed: 0, passed: 0, failed: 0, skipped: 0, seen: {}, experiments: {} };
|
|
412
|
+
renderProgress();
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
function recordProgress(event) {
|
|
416
|
+
var key = cellKey(event);
|
|
417
|
+
if (progress.seen[key]) {
|
|
418
|
+
return;
|
|
419
|
+
}
|
|
420
|
+
progress.seen[key] = true;
|
|
421
|
+
var experiment = experimentState(event);
|
|
422
|
+
experiment.cells[event.arm || "_"] = event;
|
|
423
|
+
var scenario = findScenario(event.suite, event.scenario);
|
|
424
|
+
var expectedArms = scenario && scenario.compare && scenario.compare.length ? scenario.compare.length : 1;
|
|
425
|
+
if (Object.keys(experiment.cells).length < expectedArms || experiment.counted) {
|
|
426
|
+
renderProgress();
|
|
427
|
+
return;
|
|
428
|
+
}
|
|
429
|
+
experiment.counted = true;
|
|
430
|
+
progress.completed += 1;
|
|
431
|
+
var outcome = experimentOutcome(scenario, experiment.cells);
|
|
432
|
+
if (outcome === "skipped") progress.skipped += 1;
|
|
433
|
+
else if (outcome === "passed") progress.passed += 1;
|
|
434
|
+
else progress.failed += 1;
|
|
435
|
+
renderProgress();
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
function finishProgress(event) {
|
|
439
|
+
// The stream reports one cell per comparison arm. Progress is based on
|
|
440
|
+
// the completed scenario/host experiment, so expected failed controls do
|
|
441
|
+
// not turn a passing comparison into a red suite result.
|
|
442
|
+
progress.total = Math.max(progress.total, progress.completed);
|
|
443
|
+
renderProgress();
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
function experimentKey(event) {
|
|
447
|
+
return [event.suite, event.scenario, event.host].join("::");
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
function experimentState(event) {
|
|
451
|
+
var key = experimentKey(event);
|
|
452
|
+
if (!progress.experiments[key]) {
|
|
453
|
+
progress.experiments[key] = { cells: {}, counted: false };
|
|
454
|
+
}
|
|
455
|
+
return progress.experiments[key];
|
|
456
|
+
}
|
|
457
|
+
|
|
458
|
+
function experimentOutcome(scenario, cells) {
|
|
459
|
+
var events = Object.keys(cells).map(function (id) { return cells[id]; });
|
|
460
|
+
if (events.some(function (event) { return event.skipped; })) return "skipped";
|
|
461
|
+
var byArm = {};
|
|
462
|
+
events.forEach(function (event) { byArm[event.arm || "_"] = event; });
|
|
463
|
+
if (!scenario || !scenario.compare || !scenario.compare.length) {
|
|
464
|
+
return events.every(function (event) { return event.passed; }) ? "passed" : "failed";
|
|
465
|
+
}
|
|
466
|
+
var gates = scenario.gates || [];
|
|
467
|
+
var behaviorOk = events.every(function (event) {
|
|
468
|
+
return event.passed || (event.failures || []).every(function (failure) {
|
|
469
|
+
return gates.some(function (gate) {
|
|
470
|
+
return gate.metric === "outcome" && gate.arm === event.arm && gate.operator === "equal" && gate.value === "fail";
|
|
471
|
+
});
|
|
472
|
+
});
|
|
473
|
+
});
|
|
474
|
+
var gatesOk = gates.every(function (gate) {
|
|
475
|
+
if (gate.metric === "outcome" && gate.arm) {
|
|
476
|
+
var event = byArm[gate.arm];
|
|
477
|
+
if (!event) return false;
|
|
478
|
+
var actual = event.passed ? "pass" : "fail";
|
|
479
|
+
return gate.operator === "equal" ? actual === gate.value : false;
|
|
480
|
+
}
|
|
481
|
+
if (["turns", "tokens", "tools", "durationMs"].indexOf(gate.metric) !== -1 && gate.winner && gate.loser) {
|
|
482
|
+
var winner = byArm[gate.winner];
|
|
483
|
+
var loser = byArm[gate.loser];
|
|
484
|
+
var field = gate.metric === "durationMs" ? "durationMs" : gate.metric;
|
|
485
|
+
var winnerValue = winner && winner.metrics && winner.metrics[field];
|
|
486
|
+
var loserValue = loser && loser.metrics && loser.metrics[field];
|
|
487
|
+
return typeof winnerValue === "number" && typeof loserValue === "number" && winnerValue < loserValue;
|
|
488
|
+
}
|
|
489
|
+
return true;
|
|
490
|
+
});
|
|
491
|
+
return behaviorOk && gatesOk ? "passed" : "failed";
|
|
492
|
+
}
|
|
493
|
+
|
|
161
494
|
function cellKey(event) {
|
|
162
495
|
return [event.suite, event.scenario, event.host, event.arm || "_"].join("::");
|
|
163
496
|
}
|
|
@@ -166,41 +499,372 @@ function clientScript() {
|
|
|
166
499
|
return "live-" + cellKey(event).replace(/[^a-z0-9]+/gi, "-");
|
|
167
500
|
}
|
|
168
501
|
|
|
502
|
+
function compareWrapId(event) {
|
|
503
|
+
return ("live-wrap-" + event.suite + "-" + event.scenario + "-" + event.host).replace(/[^a-z0-9]+/gi, "-");
|
|
504
|
+
}
|
|
505
|
+
|
|
506
|
+
function selectCompareTab(wrap, armId) {
|
|
507
|
+
wrap.querySelectorAll(".compare-tab").forEach(function (tab) {
|
|
508
|
+
var on = tab.getAttribute("data-arm") === armId;
|
|
509
|
+
tab.setAttribute("aria-selected", on ? "true" : "false");
|
|
510
|
+
});
|
|
511
|
+
wrap.querySelectorAll(".live-cell").forEach(function (panel) {
|
|
512
|
+
panel.hidden = panel.getAttribute("data-arm") !== armId;
|
|
513
|
+
});
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
function scenarioCard(event) {
|
|
517
|
+
return document.querySelector('[data-scenario-card="' + event.suite + "::" + event.scenario + '"]');
|
|
518
|
+
}
|
|
519
|
+
|
|
520
|
+
function selectHostTab(card, host) {
|
|
521
|
+
if (!card || !host) {
|
|
522
|
+
return;
|
|
523
|
+
}
|
|
524
|
+
card.querySelectorAll(".host-tab").forEach(function (tab) {
|
|
525
|
+
var on = tab.getAttribute("data-host") === host;
|
|
526
|
+
tab.setAttribute("aria-selected", on ? "true" : "false");
|
|
527
|
+
});
|
|
528
|
+
card.querySelectorAll("[data-host-panel]").forEach(function (panel) {
|
|
529
|
+
panel.hidden = panel.getAttribute("data-host-panel") !== host;
|
|
530
|
+
});
|
|
531
|
+
var run = card.querySelector("button.run-cell");
|
|
532
|
+
if (run) {
|
|
533
|
+
run.setAttribute("data-host", host);
|
|
534
|
+
var tab = card.querySelector('.host-tab[data-host="' + host + '"]');
|
|
535
|
+
if (tab) {
|
|
536
|
+
run.disabled = tab.disabled;
|
|
537
|
+
}
|
|
538
|
+
}
|
|
539
|
+
}
|
|
540
|
+
|
|
541
|
+
function revealLive(event) {
|
|
542
|
+
var card = scenarioCard(event);
|
|
543
|
+
if (!card) {
|
|
544
|
+
return;
|
|
545
|
+
}
|
|
546
|
+
var live = card.querySelector(".scenario-live");
|
|
547
|
+
if (live) {
|
|
548
|
+
live.hidden = false;
|
|
549
|
+
}
|
|
550
|
+
if (card.dataset.userPickedHost === "1") {
|
|
551
|
+
return;
|
|
552
|
+
}
|
|
553
|
+
if (card.dataset.followHost && card.dataset.followHost !== event.host) {
|
|
554
|
+
return;
|
|
555
|
+
}
|
|
556
|
+
card.dataset.followHost = event.host;
|
|
557
|
+
selectHostTab(card, event.host);
|
|
558
|
+
}
|
|
559
|
+
|
|
560
|
+
function hostPanel(event) {
|
|
561
|
+
var card = scenarioCard(event);
|
|
562
|
+
return card ? card.querySelector('[data-host-panel="' + event.host + '"]') : null;
|
|
563
|
+
}
|
|
564
|
+
|
|
565
|
+
function markArmTab(wrap, armId, status) {
|
|
566
|
+
var tab = wrap.querySelector('.compare-tab[data-arm="' + armId + '"]');
|
|
567
|
+
if (tab) {
|
|
568
|
+
tab.setAttribute("data-status", status);
|
|
569
|
+
}
|
|
570
|
+
}
|
|
571
|
+
|
|
572
|
+
function createLiveArticle(event, title) {
|
|
573
|
+
var article = document.createElement("article");
|
|
574
|
+
article.className = "live-cell";
|
|
575
|
+
article.id = paneId(event);
|
|
576
|
+
article.dataset.chatKey = cellKey(event);
|
|
577
|
+
article.innerHTML = "<h3></h3><ol class=\\"live-status\\"></ol><div class=\\"chat\\"></div>";
|
|
578
|
+
article.querySelector("h3").textContent = title;
|
|
579
|
+
return article;
|
|
580
|
+
}
|
|
581
|
+
|
|
582
|
+
var openChats = {};
|
|
583
|
+
|
|
584
|
+
function resetArticle(article) {
|
|
585
|
+
if (!article) {
|
|
586
|
+
return;
|
|
587
|
+
}
|
|
588
|
+
var status = article.querySelector(".live-status");
|
|
589
|
+
if (status) {
|
|
590
|
+
status.replaceChildren();
|
|
591
|
+
}
|
|
592
|
+
var chat = article.querySelector(".chat");
|
|
593
|
+
if (chat) {
|
|
594
|
+
chat.replaceChildren();
|
|
595
|
+
}
|
|
596
|
+
var result = article.querySelector(".cell-result");
|
|
597
|
+
if (result) {
|
|
598
|
+
result.remove();
|
|
599
|
+
}
|
|
600
|
+
var panel = article.closest("[data-host-panel]");
|
|
601
|
+
var empty = panel ? panel.querySelector(".host-empty") : null;
|
|
602
|
+
if (empty) {
|
|
603
|
+
empty.hidden = false;
|
|
604
|
+
}
|
|
605
|
+
}
|
|
606
|
+
|
|
607
|
+
function chatIsOpen(event) {
|
|
608
|
+
return !!openChats[cellKey(event)];
|
|
609
|
+
}
|
|
610
|
+
|
|
611
|
+
function startChat(event) {
|
|
612
|
+
openChats[cellKey(event)] = true;
|
|
613
|
+
delete assistantBodies[cellKey(event)];
|
|
614
|
+
delete finishedMetrics[cellKey(event)];
|
|
615
|
+
ensurePane(event);
|
|
616
|
+
resetArticle(document.getElementById(paneId(event)));
|
|
617
|
+
var panel = hostPanel(event);
|
|
618
|
+
var empty = panel ? panel.querySelector(".host-empty") : null;
|
|
619
|
+
if (empty) {
|
|
620
|
+
empty.hidden = true;
|
|
621
|
+
}
|
|
622
|
+
}
|
|
623
|
+
|
|
624
|
+
function markRunningIdle() {
|
|
625
|
+
document.querySelectorAll("[data-cell]").forEach(function (node) {
|
|
626
|
+
var status = node.querySelector(".cell-status");
|
|
627
|
+
if (!status) {
|
|
628
|
+
return;
|
|
629
|
+
}
|
|
630
|
+
var label = status.textContent;
|
|
631
|
+
if (label !== "running" && label !== "starting") {
|
|
632
|
+
return;
|
|
633
|
+
}
|
|
634
|
+
status.textContent = "idle";
|
|
635
|
+
status.className = "cell-status muted";
|
|
636
|
+
if (node.classList.contains("host-tab")) {
|
|
637
|
+
node.setAttribute("data-status", "idle");
|
|
638
|
+
}
|
|
639
|
+
});
|
|
640
|
+
document.querySelectorAll(".compare-tab[data-status='running']").forEach(function (tab) {
|
|
641
|
+
tab.setAttribute("data-status", "idle");
|
|
642
|
+
});
|
|
643
|
+
}
|
|
644
|
+
|
|
645
|
+
function closeAllChats() {
|
|
646
|
+
Object.keys(openChats).forEach(function (key) {
|
|
647
|
+
resetArticle(document.getElementById("live-" + key.replace(/[^a-z0-9]+/gi, "-")));
|
|
648
|
+
});
|
|
649
|
+
openChats = {};
|
|
650
|
+
assistantBodies = {};
|
|
651
|
+
hideAllRunning();
|
|
652
|
+
markRunningIdle();
|
|
653
|
+
}
|
|
654
|
+
|
|
655
|
+
function setCancelButton(label, disabled) {
|
|
656
|
+
var button = document.getElementById("cancel-run");
|
|
657
|
+
button.textContent = label;
|
|
658
|
+
button.disabled = disabled;
|
|
659
|
+
}
|
|
660
|
+
|
|
661
|
+
function showCancelledNote(article, text) {
|
|
662
|
+
if (!article) {
|
|
663
|
+
return;
|
|
664
|
+
}
|
|
665
|
+
var existing = article.querySelector(".cell-result");
|
|
666
|
+
if (existing) {
|
|
667
|
+
existing.remove();
|
|
668
|
+
}
|
|
669
|
+
var box = document.createElement("div");
|
|
670
|
+
box.className = "cell-result status-skipped";
|
|
671
|
+
var verdict = document.createElement("p");
|
|
672
|
+
verdict.className = "cell-result-verdict";
|
|
673
|
+
verdict.textContent = text;
|
|
674
|
+
box.appendChild(verdict);
|
|
675
|
+
article.appendChild(box);
|
|
676
|
+
var panel = article.closest("[data-host-panel]");
|
|
677
|
+
var empty = panel ? panel.querySelector(".host-empty") : null;
|
|
678
|
+
if (empty) {
|
|
679
|
+
empty.hidden = true;
|
|
680
|
+
}
|
|
681
|
+
}
|
|
682
|
+
|
|
683
|
+
function cancelOpenChats() {
|
|
684
|
+
document.querySelectorAll(".cell-status").forEach(function (status) {
|
|
685
|
+
if (status.textContent !== "running" && status.textContent !== "starting") {
|
|
686
|
+
return;
|
|
687
|
+
}
|
|
688
|
+
status.textContent = "cancelling";
|
|
689
|
+
status.className = "cell-status status-skipped";
|
|
690
|
+
var tab = status.closest(".host-tab");
|
|
691
|
+
if (tab) {
|
|
692
|
+
tab.setAttribute("data-status", "cancelling");
|
|
693
|
+
}
|
|
694
|
+
});
|
|
695
|
+
document.querySelectorAll(".compare-tab[data-status='running']").forEach(function (tab) {
|
|
696
|
+
tab.setAttribute("data-status", "cancelling");
|
|
697
|
+
});
|
|
698
|
+
hideAllRunning();
|
|
699
|
+
document.querySelectorAll("[data-cell]").forEach(function (node) {
|
|
700
|
+
var status = node.querySelector(".cell-status");
|
|
701
|
+
if (!status || status.textContent !== "cancelling") {
|
|
702
|
+
return;
|
|
703
|
+
}
|
|
704
|
+
var card = node.closest("[data-scenario-card]");
|
|
705
|
+
if (!card) {
|
|
706
|
+
return;
|
|
707
|
+
}
|
|
708
|
+
var live = card.querySelector(".scenario-live");
|
|
709
|
+
if (live) {
|
|
710
|
+
live.hidden = false;
|
|
711
|
+
}
|
|
712
|
+
var host = node.getAttribute("data-host") || (node.getAttribute("data-cell") || "").split("::")[2];
|
|
713
|
+
var panel = card.querySelector('[data-host-panel="' + host + '"]');
|
|
714
|
+
if (!panel) {
|
|
715
|
+
return;
|
|
716
|
+
}
|
|
717
|
+
var articles = panel.querySelectorAll("article.live-cell");
|
|
718
|
+
if (articles.length === 0) {
|
|
719
|
+
showCancelledNote(panel, "Cancelled. Stopping the host agent.");
|
|
720
|
+
return;
|
|
721
|
+
}
|
|
722
|
+
articles.forEach(function (article) {
|
|
723
|
+
var key = article.dataset.chatKey;
|
|
724
|
+
resetArticle(article);
|
|
725
|
+
showCancelledNote(article, "Cancelled. Stopping the host agent.");
|
|
726
|
+
if (key) {
|
|
727
|
+
delete openChats[key];
|
|
728
|
+
delete assistantBodies[key];
|
|
729
|
+
}
|
|
730
|
+
});
|
|
731
|
+
});
|
|
732
|
+
}
|
|
733
|
+
|
|
734
|
+
function markCancelledCells() {
|
|
735
|
+
document.querySelectorAll(".cell-status").forEach(function (status) {
|
|
736
|
+
if (status.textContent === "cancelling") {
|
|
737
|
+
status.textContent = "cancelled";
|
|
738
|
+
status.className = "cell-status status-skipped";
|
|
739
|
+
var tab = status.closest(".host-tab");
|
|
740
|
+
if (tab) {
|
|
741
|
+
tab.setAttribute("data-status", "cancelled");
|
|
742
|
+
}
|
|
743
|
+
}
|
|
744
|
+
});
|
|
745
|
+
document.querySelectorAll(".compare-tab[data-status='cancelling']").forEach(function (tab) {
|
|
746
|
+
tab.setAttribute("data-status", "cancelled");
|
|
747
|
+
});
|
|
748
|
+
document.querySelectorAll(".cell-result-verdict").forEach(function (node) {
|
|
749
|
+
if (node.textContent === "Cancelled. Stopping the host agent.") {
|
|
750
|
+
node.textContent = "Cancelled.";
|
|
751
|
+
}
|
|
752
|
+
});
|
|
753
|
+
}
|
|
754
|
+
|
|
755
|
+
function sealAllChats() {
|
|
756
|
+
openChats = {};
|
|
757
|
+
}
|
|
758
|
+
|
|
759
|
+
function addArmTab(wrap, list, article, armId, label, selected) {
|
|
760
|
+
article.setAttribute("role", "tabpanel");
|
|
761
|
+
article.setAttribute("data-arm", armId);
|
|
762
|
+
article.hidden = !selected;
|
|
763
|
+
var tab = document.createElement("button");
|
|
764
|
+
tab.type = "button";
|
|
765
|
+
tab.className = "compare-tab";
|
|
766
|
+
tab.setAttribute("role", "tab");
|
|
767
|
+
tab.setAttribute("data-arm", armId);
|
|
768
|
+
tab.setAttribute("aria-controls", article.id);
|
|
769
|
+
tab.setAttribute("aria-selected", selected ? "true" : "false");
|
|
770
|
+
tab.textContent = label;
|
|
771
|
+
tab.addEventListener("click", function () {
|
|
772
|
+
wrap.dataset.userPicked = "1";
|
|
773
|
+
selectCompareTab(wrap, armId);
|
|
774
|
+
});
|
|
775
|
+
list.appendChild(tab);
|
|
776
|
+
}
|
|
777
|
+
|
|
778
|
+
function ensureCompareWrap(event, scenario, slot) {
|
|
779
|
+
var wrapId = compareWrapId(event);
|
|
780
|
+
var wrap = document.getElementById(wrapId);
|
|
781
|
+
if (!wrap) {
|
|
782
|
+
wrap = document.createElement("div");
|
|
783
|
+
wrap.className = "compare-layout compare-tabs";
|
|
784
|
+
wrap.id = wrapId;
|
|
785
|
+
var list = document.createElement("div");
|
|
786
|
+
list.className = "compare-tablist";
|
|
787
|
+
list.setAttribute("role", "tablist");
|
|
788
|
+
list.setAttribute("aria-label", "Comparison arms");
|
|
789
|
+
wrap.appendChild(list);
|
|
790
|
+
var arms = document.createElement("div");
|
|
791
|
+
arms.className = "compare-arms";
|
|
792
|
+
if (scenario && scenario.compare) {
|
|
793
|
+
arms.setAttribute("data-arm-count", String(scenario.compare.length));
|
|
794
|
+
}
|
|
795
|
+
wrap.appendChild(arms);
|
|
796
|
+
if (scenario && scenario.compare) {
|
|
797
|
+
scenario.compare.forEach(function (arm, index) {
|
|
798
|
+
var article = createLiveArticle({
|
|
799
|
+
suite: event.suite,
|
|
800
|
+
scenario: event.scenario,
|
|
801
|
+
host: event.host,
|
|
802
|
+
arm: arm.id,
|
|
803
|
+
}, arm.label);
|
|
804
|
+
addArmTab(wrap, list, article, arm.id, arm.label, index === 0);
|
|
805
|
+
arms.appendChild(article);
|
|
806
|
+
});
|
|
807
|
+
}
|
|
808
|
+
}
|
|
809
|
+
if (!wrap.parentNode) {
|
|
810
|
+
slot.appendChild(wrap);
|
|
811
|
+
}
|
|
812
|
+
return wrap;
|
|
813
|
+
}
|
|
814
|
+
|
|
169
815
|
function ensurePane(event) {
|
|
170
|
-
var
|
|
171
|
-
var existing = document.getElementById(id);
|
|
816
|
+
var existing = document.getElementById(paneId(event));
|
|
172
817
|
if (existing) {
|
|
818
|
+
revealLive(event);
|
|
173
819
|
return existing.querySelector(".chat");
|
|
174
820
|
}
|
|
175
|
-
var
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
article.id = id;
|
|
179
|
-
var title = event.suite + " / " + event.scenario + " / " + event.host;
|
|
180
|
-
if (event.arm) {
|
|
181
|
-
title += " / arm " + event.arm;
|
|
821
|
+
var slot = hostPanel(event);
|
|
822
|
+
if (!slot) {
|
|
823
|
+
return null;
|
|
182
824
|
}
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
825
|
+
revealLive(event);
|
|
826
|
+
var empty = slot.querySelector(".host-empty");
|
|
827
|
+
if (empty) {
|
|
828
|
+
empty.hidden = true;
|
|
829
|
+
}
|
|
830
|
+
var leftover = slot.querySelector(":scope > .cell-result");
|
|
831
|
+
if (leftover) {
|
|
832
|
+
leftover.remove();
|
|
833
|
+
}
|
|
834
|
+
var scenario = findScenario(event.suite, event.scenario);
|
|
186
835
|
if (event.arm) {
|
|
187
|
-
var
|
|
188
|
-
|
|
189
|
-
if (
|
|
190
|
-
wrap
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
}
|
|
200
|
-
parent.appendChild(article);
|
|
836
|
+
var wrap = ensureCompareWrap(event, scenario, slot);
|
|
837
|
+
existing = document.getElementById(paneId(event));
|
|
838
|
+
if (existing) {
|
|
839
|
+
if (wrap.dataset.userPicked !== "1") {
|
|
840
|
+
selectCompareTab(wrap, event.arm);
|
|
841
|
+
}
|
|
842
|
+
return existing.querySelector(".chat");
|
|
843
|
+
}
|
|
844
|
+
return null;
|
|
845
|
+
}
|
|
846
|
+
var article = createLiveArticle(event, event.host);
|
|
847
|
+
slot.appendChild(article);
|
|
201
848
|
return article.querySelector(".chat");
|
|
202
849
|
}
|
|
203
850
|
|
|
851
|
+
function appendStatus(event, text) {
|
|
852
|
+
var article = document.getElementById(paneId(event));
|
|
853
|
+
var list = article ? article.querySelector(".live-status") : null;
|
|
854
|
+
if (!list) {
|
|
855
|
+
ensurePane(event);
|
|
856
|
+
article = document.getElementById(paneId(event));
|
|
857
|
+
list = article ? article.querySelector(".live-status") : null;
|
|
858
|
+
}
|
|
859
|
+
if (!list) {
|
|
860
|
+
return;
|
|
861
|
+
}
|
|
862
|
+
var item = document.createElement("li");
|
|
863
|
+
item.textContent = text;
|
|
864
|
+
list.appendChild(item);
|
|
865
|
+
list.scrollTop = list.scrollHeight;
|
|
866
|
+
}
|
|
867
|
+
|
|
204
868
|
function appendBubble(chat, role, text) {
|
|
205
869
|
var row = document.createElement("div");
|
|
206
870
|
row.className = "chat-row " + (role === "user" ? "side-right" : "side-left");
|
|
@@ -219,6 +883,117 @@ function clientScript() {
|
|
|
219
883
|
return body;
|
|
220
884
|
}
|
|
221
885
|
|
|
886
|
+
function runningRow(article) {
|
|
887
|
+
return article.querySelector(".chat-running-row");
|
|
888
|
+
}
|
|
889
|
+
|
|
890
|
+
function showRunning(event) {
|
|
891
|
+
var article = document.getElementById(paneId(event));
|
|
892
|
+
if (!article) {
|
|
893
|
+
ensurePane(event);
|
|
894
|
+
article = document.getElementById(paneId(event));
|
|
895
|
+
}
|
|
896
|
+
if (!article) {
|
|
897
|
+
return;
|
|
898
|
+
}
|
|
899
|
+
var chat = article.querySelector(".chat");
|
|
900
|
+
if (!chat) {
|
|
901
|
+
return;
|
|
902
|
+
}
|
|
903
|
+
var row = runningRow(article);
|
|
904
|
+
if (!row) {
|
|
905
|
+
row = document.createElement("div");
|
|
906
|
+
row.className = "chat-row side-left chat-running-row";
|
|
907
|
+
row.innerHTML = '<div class="chat-running" aria-live="polite">Running<span class="chat-progress" aria-hidden="true"><span></span><span></span><span></span></span></div>';
|
|
908
|
+
chat.appendChild(row);
|
|
909
|
+
}
|
|
910
|
+
row.hidden = false;
|
|
911
|
+
chat.appendChild(row);
|
|
912
|
+
}
|
|
913
|
+
|
|
914
|
+
function hideRunning(event) {
|
|
915
|
+
var article = document.getElementById(paneId(event));
|
|
916
|
+
var row = article ? runningRow(article) : null;
|
|
917
|
+
if (row) {
|
|
918
|
+
row.remove();
|
|
919
|
+
}
|
|
920
|
+
}
|
|
921
|
+
|
|
922
|
+
function hideAllRunning() {
|
|
923
|
+
document.querySelectorAll(".chat-running-row").forEach(function (row) {
|
|
924
|
+
row.remove();
|
|
925
|
+
});
|
|
926
|
+
document.querySelectorAll(".bubble.is-streaming").forEach(function (bubble) {
|
|
927
|
+
bubble.classList.remove("is-streaming");
|
|
928
|
+
});
|
|
929
|
+
}
|
|
930
|
+
|
|
931
|
+
function setStreaming(event, on) {
|
|
932
|
+
var body = assistantBodies[cellKey(event)];
|
|
933
|
+
var bubble = body ? body.parentNode : null;
|
|
934
|
+
if (bubble) {
|
|
935
|
+
if (on) {
|
|
936
|
+
bubble.classList.add("is-streaming");
|
|
937
|
+
} else {
|
|
938
|
+
bubble.classList.remove("is-streaming");
|
|
939
|
+
}
|
|
940
|
+
}
|
|
941
|
+
}
|
|
942
|
+
|
|
943
|
+
function appendContextFiles(chat, files) {
|
|
944
|
+
(files || []).forEach(function (file) {
|
|
945
|
+
var row = document.createElement("div");
|
|
946
|
+
row.className = "chat-row side-left";
|
|
947
|
+
var bubble = document.createElement("div");
|
|
948
|
+
bubble.className = "bubble role-context";
|
|
949
|
+
var label = document.createElement("div");
|
|
950
|
+
label.className = "bubble-label";
|
|
951
|
+
label.textContent = "Context";
|
|
952
|
+
var path = document.createElement("p");
|
|
953
|
+
path.className = "context-path";
|
|
954
|
+
path.textContent = file.path || "context";
|
|
955
|
+
var why = document.createElement("p");
|
|
956
|
+
why.className = "context-why";
|
|
957
|
+
why.textContent = file.why || "The runner loaded this file into the host preamble.";
|
|
958
|
+
var body = document.createElement("div");
|
|
959
|
+
body.className = "bubble-text";
|
|
960
|
+
body.textContent = file.text || "";
|
|
961
|
+
bubble.appendChild(label);
|
|
962
|
+
bubble.appendChild(path);
|
|
963
|
+
bubble.appendChild(why);
|
|
964
|
+
bubble.appendChild(body);
|
|
965
|
+
row.appendChild(bubble);
|
|
966
|
+
chat.appendChild(row);
|
|
967
|
+
});
|
|
968
|
+
}
|
|
969
|
+
|
|
970
|
+
function displayToolPath(path) {
|
|
971
|
+
var stripped = String(path).replace(/^file:\\/\\//, "").replace(/\\/+$/, "");
|
|
972
|
+
var workspace = stripped.match(/\\/agent-harness-seal-[^/]+(?:\\/(.*))?$/);
|
|
973
|
+
if (workspace) {
|
|
974
|
+
return workspace[1] ? workspace[1] : ".";
|
|
975
|
+
}
|
|
976
|
+
var sealed = stripped.match(/\\/(?:\\.agents|\\.claude|\\.codex|AGENTS\\.md|CLAUDE\\.md)(?:\\/|$)/);
|
|
977
|
+
if (sealed && sealed.index !== undefined) {
|
|
978
|
+
return stripped.slice(sealed.index + 1);
|
|
979
|
+
}
|
|
980
|
+
var parts = stripped.split("/").filter(Boolean);
|
|
981
|
+
if (parts.length > 3) {
|
|
982
|
+
return ".../" + parts.slice(-3).join("/");
|
|
983
|
+
}
|
|
984
|
+
return stripped;
|
|
985
|
+
}
|
|
986
|
+
|
|
987
|
+
function formatArgValue(key, value) {
|
|
988
|
+
var pathKeys = { path: 1, file_path: 1, filePath: 1, target_file: 1, uri: 1, cwd: 1 };
|
|
989
|
+
if (typeof value === "string" && (pathKeys[key] || value.indexOf("/") === 0 || value.indexOf("file://") === 0)) {
|
|
990
|
+
return { text: displayToolPath(value), title: value };
|
|
991
|
+
}
|
|
992
|
+
var raw = typeof value === "string" ? value : JSON.stringify(value);
|
|
993
|
+
var oneLine = raw.replace(/\\n/g, " ↵ ");
|
|
994
|
+
return { text: oneLine.length > 140 ? oneLine.slice(0, 140) + "…" : oneLine };
|
|
995
|
+
}
|
|
996
|
+
|
|
222
997
|
function appendTool(chat, name, args) {
|
|
223
998
|
var row = document.createElement("div");
|
|
224
999
|
row.className = "chat-row side-left";
|
|
@@ -241,7 +1016,11 @@ function clientScript() {
|
|
|
241
1016
|
keyEl.className = "tool-arg-key";
|
|
242
1017
|
keyEl.textContent = key;
|
|
243
1018
|
var value = document.createElement("code");
|
|
244
|
-
|
|
1019
|
+
var formatted = formatArgValue(key, args[key]);
|
|
1020
|
+
value.textContent = formatted.text;
|
|
1021
|
+
if (formatted.title && formatted.title !== formatted.text) {
|
|
1022
|
+
value.title = formatted.title;
|
|
1023
|
+
}
|
|
245
1024
|
line.appendChild(keyEl);
|
|
246
1025
|
line.appendChild(value);
|
|
247
1026
|
box.appendChild(line);
|
|
@@ -253,101 +1032,542 @@ function clientScript() {
|
|
|
253
1032
|
}
|
|
254
1033
|
|
|
255
1034
|
var assistantBodies = {};
|
|
1035
|
+
var finishedMetrics = {};
|
|
1036
|
+
|
|
1037
|
+
function findScenario(suiteName, scenarioName) {
|
|
1038
|
+
for (var i = 0; i < catalog.suites.length; i++) {
|
|
1039
|
+
if (catalog.suites[i].name !== suiteName) {
|
|
1040
|
+
continue;
|
|
1041
|
+
}
|
|
1042
|
+
var scenarios = catalog.suites[i].scenarios;
|
|
1043
|
+
for (var j = 0; j < scenarios.length; j++) {
|
|
1044
|
+
if (scenarios[j].name === scenarioName) {
|
|
1045
|
+
return scenarios[j];
|
|
1046
|
+
}
|
|
1047
|
+
}
|
|
1048
|
+
}
|
|
1049
|
+
return null;
|
|
1050
|
+
}
|
|
1051
|
+
|
|
1052
|
+
function joinLabels(labels) {
|
|
1053
|
+
if (labels.length <= 1) {
|
|
1054
|
+
return labels[0] || "";
|
|
1055
|
+
}
|
|
1056
|
+
if (labels.length === 2) {
|
|
1057
|
+
return labels[0] + " and " + labels[1];
|
|
1058
|
+
}
|
|
1059
|
+
return labels.slice(0, -1).join(", ") + ", and " + labels[labels.length - 1];
|
|
1060
|
+
}
|
|
1061
|
+
|
|
1062
|
+
function summarizeMetric(arms, id, label) {
|
|
1063
|
+
var values = arms.map(function (arm) { return arm[id]; });
|
|
1064
|
+
var numeric = values.filter(function (value) { return typeof value === "number"; });
|
|
1065
|
+
if (numeric.length < 2) {
|
|
1066
|
+
return { line: label + ": n/a." };
|
|
1067
|
+
}
|
|
1068
|
+
var lowest = Math.min.apply(null, numeric);
|
|
1069
|
+
var winners = arms.filter(function (arm) { return arm[id] === lowest; });
|
|
1070
|
+
var shown = values.map(function (value) { return typeof value === "number" ? String(value) : "n/a"; }).join(" vs ");
|
|
1071
|
+
if (winners.length > 1) {
|
|
1072
|
+
return {
|
|
1073
|
+
line: arms.length === 2
|
|
1074
|
+
? label + ": tie (" + shown + ")."
|
|
1075
|
+
: label + ": lowest is a tie between " + joinLabels(winners.map(function (arm) { return arm.label; })) + " (" + shown + ").",
|
|
1076
|
+
};
|
|
1077
|
+
}
|
|
1078
|
+
var winner = winners[0];
|
|
1079
|
+
if (!winner) {
|
|
1080
|
+
return { line: label + ": n/a." };
|
|
1081
|
+
}
|
|
1082
|
+
if (arms.length === 2) {
|
|
1083
|
+
return { line: label + ": " + winner.label + " wins (" + shown + ")." };
|
|
1084
|
+
}
|
|
1085
|
+
return { line: label + ": lowest is " + winner.label + " (" + shown + ")." };
|
|
1086
|
+
}
|
|
1087
|
+
|
|
1088
|
+
function summarizeGates(arms, gates) {
|
|
1089
|
+
var byId = {};
|
|
1090
|
+
arms.forEach(function (arm) { byId[arm.id] = arm; });
|
|
1091
|
+
return (gates || []).map(function (gate) {
|
|
1092
|
+
var field = gate.metric === "durationMs" ? "durationMs" : gate.metric;
|
|
1093
|
+
var winner = byId[gate.winner];
|
|
1094
|
+
var loser = byId[gate.loser];
|
|
1095
|
+
if (!winner || !loser || ["turns", "tokens", "tools", "durationMs"].indexOf(field) === -1) {
|
|
1096
|
+
return { passed: true, line: gate.metric + ": scored in the final result." };
|
|
1097
|
+
}
|
|
1098
|
+
var winnerValue = winner[field];
|
|
1099
|
+
var loserValue = loser[field];
|
|
1100
|
+
if (typeof winnerValue !== "number" || typeof loserValue !== "number") {
|
|
1101
|
+
return { passed: false, line: gate.metric + ": one arm did not report a value. Fail." };
|
|
1102
|
+
}
|
|
1103
|
+
var passed = winnerValue < loserValue;
|
|
1104
|
+
return { passed: passed, line: winner.label + " must beat " + loser.label + " on " + gate.metric + " (" + winnerValue + " vs " + loserValue + "). " + (passed ? "Pass." : "Fail.") };
|
|
1105
|
+
});
|
|
1106
|
+
}
|
|
1107
|
+
|
|
1108
|
+
function renderCompareWinners(event) {
|
|
1109
|
+
var scenario = findScenario(event.suite, event.scenario);
|
|
1110
|
+
if (!scenario || !scenario.compare || !scenario.compare.length) {
|
|
1111
|
+
return;
|
|
1112
|
+
}
|
|
1113
|
+
var wrap = document.getElementById(compareWrapId(event));
|
|
1114
|
+
if (!wrap) {
|
|
1115
|
+
return;
|
|
1116
|
+
}
|
|
1117
|
+
var rows = [];
|
|
1118
|
+
for (var i = 0; i < scenario.compare.length; i++) {
|
|
1119
|
+
var arm = scenario.compare[i];
|
|
1120
|
+
var metrics = finishedMetrics[[event.suite, event.scenario, event.host, arm.id].join("::")];
|
|
1121
|
+
if (!metrics) {
|
|
1122
|
+
return;
|
|
1123
|
+
}
|
|
1124
|
+
rows.push({
|
|
1125
|
+
id: arm.id,
|
|
1126
|
+
label: arm.label,
|
|
1127
|
+
turns: metrics.turns,
|
|
1128
|
+
tokens: metrics.tokens,
|
|
1129
|
+
tools: metrics.tools,
|
|
1130
|
+
durationMs: metrics.durationMs,
|
|
1131
|
+
passed: metrics.passed,
|
|
1132
|
+
});
|
|
1133
|
+
}
|
|
1134
|
+
var metricLines = [
|
|
1135
|
+
summarizeMetric(rows, "turns", "Turns"),
|
|
1136
|
+
summarizeMetric(rows, "tokens", "Tokens"),
|
|
1137
|
+
summarizeMetric(rows, "tools", "Tools"),
|
|
1138
|
+
summarizeMetric(rows, "durationMs", "Duration"),
|
|
1139
|
+
];
|
|
1140
|
+
var gateLines = summarizeGates(rows, scenario.gates);
|
|
1141
|
+
var existing = wrap.querySelector(".compare-winners");
|
|
1142
|
+
if (existing) {
|
|
1143
|
+
existing.remove();
|
|
1144
|
+
}
|
|
1145
|
+
var section = document.createElement("section");
|
|
1146
|
+
section.className = "compare-winners";
|
|
1147
|
+
var heading = document.createElement("h3");
|
|
1148
|
+
heading.textContent = "Winners";
|
|
1149
|
+
var list = document.createElement("ul");
|
|
1150
|
+
metricLines.forEach(function (metric) {
|
|
1151
|
+
var item = document.createElement("li");
|
|
1152
|
+
item.textContent = metric.line;
|
|
1153
|
+
list.appendChild(item);
|
|
1154
|
+
});
|
|
1155
|
+
gateLines.forEach(function (gate) {
|
|
1156
|
+
var item = document.createElement("li");
|
|
1157
|
+
item.className = gate.passed ? "compare-winner-pass" : "compare-winner-fail";
|
|
1158
|
+
item.textContent = gate.line;
|
|
1159
|
+
list.appendChild(item);
|
|
1160
|
+
});
|
|
1161
|
+
section.appendChild(heading);
|
|
1162
|
+
section.appendChild(list);
|
|
1163
|
+
wrap.appendChild(section);
|
|
1164
|
+
}
|
|
256
1165
|
|
|
257
1166
|
function markCell(event, label, statusClass) {
|
|
258
|
-
var selector = '[data-cell="' + event.suite + "::" + event.scenario + "::" + event.host + '"]
|
|
1167
|
+
var selector = '[data-cell="' + event.suite + "::" + event.scenario + "::" + event.host + '"]';
|
|
259
1168
|
document.querySelectorAll(selector).forEach(function (node) {
|
|
260
|
-
|
|
261
|
-
|
|
1169
|
+
var status = node.querySelector(".cell-status");
|
|
1170
|
+
if (status) {
|
|
1171
|
+
status.textContent = label;
|
|
1172
|
+
status.className = "cell-status " + statusClass;
|
|
1173
|
+
}
|
|
1174
|
+
if (node.classList.contains("host-tab")) {
|
|
1175
|
+
node.setAttribute("data-status", label);
|
|
1176
|
+
}
|
|
262
1177
|
});
|
|
263
1178
|
}
|
|
264
1179
|
|
|
1180
|
+
function metricPart(count, one, many) {
|
|
1181
|
+
if (typeof count !== "number") {
|
|
1182
|
+
return "";
|
|
1183
|
+
}
|
|
1184
|
+
return count + " " + (count === 1 ? one : many);
|
|
1185
|
+
}
|
|
1186
|
+
|
|
1187
|
+
function formatMetrics(metrics) {
|
|
1188
|
+
if (!metrics) {
|
|
1189
|
+
return "";
|
|
1190
|
+
}
|
|
1191
|
+
return [metricPart(metrics.turns, "turn", "turns"), metricPart(metrics.tokens, "token", "tokens"), metricPart(metrics.tools, "tool", "tools")]
|
|
1192
|
+
.filter(Boolean)
|
|
1193
|
+
.join(" · ");
|
|
1194
|
+
}
|
|
1195
|
+
|
|
1196
|
+
function clearCellResult(event) {
|
|
1197
|
+
var article = document.getElementById(paneId(event));
|
|
1198
|
+
var existing = article ? article.querySelector(".cell-result") : null;
|
|
1199
|
+
if (existing) {
|
|
1200
|
+
existing.remove();
|
|
1201
|
+
}
|
|
1202
|
+
}
|
|
1203
|
+
|
|
1204
|
+
function renderCellResult(event) {
|
|
1205
|
+
ensurePane(event);
|
|
1206
|
+
var article = document.getElementById(paneId(event));
|
|
1207
|
+
if (!article) {
|
|
1208
|
+
return;
|
|
1209
|
+
}
|
|
1210
|
+
var existing = article.querySelector(".cell-result");
|
|
1211
|
+
if (existing) {
|
|
1212
|
+
existing.remove();
|
|
1213
|
+
}
|
|
1214
|
+
var skipped = !!event.skipped;
|
|
1215
|
+
var passed = !!event.passed && !skipped;
|
|
1216
|
+
var box = document.createElement("div");
|
|
1217
|
+
box.className = "cell-result " + (skipped ? "status-skipped" : passed ? "status-passed" : "status-failed");
|
|
1218
|
+
var verdict = document.createElement("p");
|
|
1219
|
+
verdict.className = "cell-result-verdict";
|
|
1220
|
+
verdict.textContent = skipped ? "Skipped." : passed ? "Passed." : "Failed.";
|
|
1221
|
+
box.appendChild(verdict);
|
|
1222
|
+
var metrics = formatMetrics(event.metrics);
|
|
1223
|
+
if (metrics) {
|
|
1224
|
+
var line = document.createElement("p");
|
|
1225
|
+
line.className = "cell-result-metrics";
|
|
1226
|
+
line.textContent = metrics;
|
|
1227
|
+
box.appendChild(line);
|
|
1228
|
+
}
|
|
1229
|
+
if (!skipped && !passed) {
|
|
1230
|
+
var list = document.createElement("ul");
|
|
1231
|
+
list.className = "cell-result-failures";
|
|
1232
|
+
var failures = event.failures || [];
|
|
1233
|
+
if (failures.length === 0) {
|
|
1234
|
+
var item = document.createElement("li");
|
|
1235
|
+
item.textContent = "The run failed. The host did not report a reason.";
|
|
1236
|
+
list.appendChild(item);
|
|
1237
|
+
} else {
|
|
1238
|
+
failures.forEach(function (failure) {
|
|
1239
|
+
var item = document.createElement("li");
|
|
1240
|
+
item.textContent = failure.message || failure.matcher || "The run failed.";
|
|
1241
|
+
list.appendChild(item);
|
|
1242
|
+
});
|
|
1243
|
+
}
|
|
1244
|
+
box.appendChild(list);
|
|
1245
|
+
}
|
|
1246
|
+
article.appendChild(box);
|
|
1247
|
+
}
|
|
1248
|
+
|
|
1249
|
+
function appendJudgeVerdicts(event) {
|
|
1250
|
+
ensurePane(event);
|
|
1251
|
+
var article = document.getElementById(paneId(event));
|
|
1252
|
+
if (!article) {
|
|
1253
|
+
return;
|
|
1254
|
+
}
|
|
1255
|
+
var box = article.querySelector(".cell-result");
|
|
1256
|
+
if (!box) {
|
|
1257
|
+
box = document.createElement("div");
|
|
1258
|
+
box.className = "cell-result";
|
|
1259
|
+
article.appendChild(box);
|
|
1260
|
+
}
|
|
1261
|
+
var existing = box.querySelector(".cell-result-judges");
|
|
1262
|
+
if (existing) {
|
|
1263
|
+
existing.remove();
|
|
1264
|
+
}
|
|
1265
|
+
var list = document.createElement("ul");
|
|
1266
|
+
list.className = "cell-result-judges";
|
|
1267
|
+
(event.verdicts || []).forEach(function (verdict) {
|
|
1268
|
+
var item = document.createElement("li");
|
|
1269
|
+
item.className = verdict.pass ? "judge-pass" : "judge-fail";
|
|
1270
|
+
var text = (verdict.pass ? "Pass. " : "Fail. ") + (verdict.question || "");
|
|
1271
|
+
if (verdict.rationale) {
|
|
1272
|
+
text += " " + verdict.rationale;
|
|
1273
|
+
}
|
|
1274
|
+
item.textContent = text;
|
|
1275
|
+
list.appendChild(item);
|
|
1276
|
+
});
|
|
1277
|
+
box.appendChild(list);
|
|
1278
|
+
}
|
|
1279
|
+
|
|
265
1280
|
function handleEvent(event) {
|
|
266
1281
|
if (event.type === "run_started") {
|
|
1282
|
+
if (cancelRequested) {
|
|
1283
|
+
return;
|
|
1284
|
+
}
|
|
267
1285
|
setRunBanner("Run " + event.runId + " started.");
|
|
268
|
-
|
|
1286
|
+
setCancelButton("Cancel run", false);
|
|
269
1287
|
return;
|
|
270
1288
|
}
|
|
271
1289
|
if (event.type === "run_finished") {
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
);
|
|
275
|
-
|
|
1290
|
+
finishProgress(event);
|
|
1291
|
+
hideAllRunning();
|
|
1292
|
+
sealAllChats();
|
|
1293
|
+
if (cancelRequested) {
|
|
1294
|
+
markCancelledCells();
|
|
1295
|
+
setRunBanner("Run cancelled.");
|
|
1296
|
+
} else {
|
|
1297
|
+
setRunBanner(
|
|
1298
|
+
"Run finished. " + progress.passed + " passed. " + progress.failed + " failed. " + progress.skipped + " skipped."
|
|
1299
|
+
);
|
|
1300
|
+
}
|
|
1301
|
+
setCancelButton("Cancel run", true);
|
|
1302
|
+
cancelRequested = false;
|
|
276
1303
|
runId = null;
|
|
1304
|
+
if (source) {
|
|
1305
|
+
source.close();
|
|
1306
|
+
source = null;
|
|
1307
|
+
}
|
|
1308
|
+
return;
|
|
1309
|
+
}
|
|
1310
|
+
if (cancelRequested) {
|
|
277
1311
|
return;
|
|
278
1312
|
}
|
|
279
1313
|
if (event.type === "error") {
|
|
280
1314
|
setRunBanner(event.message);
|
|
1315
|
+
if (event.suite && chatIsOpen(event)) {
|
|
1316
|
+
appendStatus(event, event.message);
|
|
1317
|
+
var errorChat = ensurePane(event);
|
|
1318
|
+
if (errorChat) {
|
|
1319
|
+
appendBubble(errorChat, "system", event.message);
|
|
1320
|
+
}
|
|
1321
|
+
}
|
|
281
1322
|
return;
|
|
282
1323
|
}
|
|
283
1324
|
if (event.type === "cell_started") {
|
|
284
1325
|
markCell(event, "running", "status-skipped");
|
|
285
|
-
|
|
1326
|
+
startChat(event);
|
|
1327
|
+
if (event.arm) {
|
|
1328
|
+
var startedWrap = document.getElementById(compareWrapId(event));
|
|
1329
|
+
if (startedWrap) {
|
|
1330
|
+
markArmTab(startedWrap, event.arm, "running");
|
|
1331
|
+
if (startedWrap.classList.contains("compare-tabs") && startedWrap.dataset.userPicked !== "1") {
|
|
1332
|
+
selectCompareTab(startedWrap, event.arm);
|
|
1333
|
+
}
|
|
1334
|
+
}
|
|
1335
|
+
}
|
|
1336
|
+
return;
|
|
1337
|
+
}
|
|
1338
|
+
if (event.type === "status") {
|
|
1339
|
+
if (!chatIsOpen(event)) {
|
|
1340
|
+
return;
|
|
1341
|
+
}
|
|
1342
|
+
appendStatus(event, event.text);
|
|
1343
|
+
return;
|
|
1344
|
+
}
|
|
1345
|
+
if (event.type === "context") {
|
|
1346
|
+
if (!chatIsOpen(event)) {
|
|
1347
|
+
return;
|
|
1348
|
+
}
|
|
1349
|
+
var contextChat = ensurePane(event);
|
|
1350
|
+
if (contextChat) {
|
|
1351
|
+
appendContextFiles(contextChat, event.files);
|
|
1352
|
+
}
|
|
286
1353
|
return;
|
|
287
1354
|
}
|
|
288
1355
|
if (event.type === "prompt") {
|
|
289
|
-
|
|
1356
|
+
if (!chatIsOpen(event)) {
|
|
1357
|
+
return;
|
|
1358
|
+
}
|
|
1359
|
+
var promptChat = ensurePane(event);
|
|
1360
|
+
if (promptChat) {
|
|
1361
|
+
appendBubble(promptChat, "user", event.text);
|
|
1362
|
+
showRunning(event);
|
|
1363
|
+
}
|
|
290
1364
|
return;
|
|
291
1365
|
}
|
|
292
1366
|
if (event.type === "text") {
|
|
1367
|
+
if (!chatIsOpen(event)) {
|
|
1368
|
+
return;
|
|
1369
|
+
}
|
|
293
1370
|
var key = cellKey(event);
|
|
294
1371
|
var chat = ensurePane(event);
|
|
1372
|
+
if (!chat) {
|
|
1373
|
+
return;
|
|
1374
|
+
}
|
|
1375
|
+
hideRunning(event);
|
|
295
1376
|
if (!assistantBodies[key]) {
|
|
296
1377
|
assistantBodies[key] = appendBubble(chat, "assistant", event.text);
|
|
297
1378
|
} else {
|
|
298
1379
|
assistantBodies[key].textContent = event.text;
|
|
299
1380
|
}
|
|
1381
|
+
setStreaming(event, true);
|
|
300
1382
|
return;
|
|
301
1383
|
}
|
|
302
1384
|
if (event.type === "tool") {
|
|
1385
|
+
if (!chatIsOpen(event)) {
|
|
1386
|
+
return;
|
|
1387
|
+
}
|
|
1388
|
+
var toolChat = ensurePane(event);
|
|
1389
|
+
if (!toolChat) {
|
|
1390
|
+
return;
|
|
1391
|
+
}
|
|
1392
|
+
setStreaming(event, false);
|
|
303
1393
|
assistantBodies[cellKey(event)] = null;
|
|
304
|
-
|
|
1394
|
+
hideRunning(event);
|
|
1395
|
+
appendTool(toolChat, event.name, event.args);
|
|
1396
|
+
showRunning(event);
|
|
305
1397
|
return;
|
|
306
1398
|
}
|
|
307
1399
|
if (event.type === "cell_finished") {
|
|
1400
|
+
recordProgress(event);
|
|
1401
|
+
if (!chatIsOpen(event)) {
|
|
1402
|
+
return;
|
|
1403
|
+
}
|
|
1404
|
+
hideRunning(event);
|
|
1405
|
+
setStreaming(event, false);
|
|
308
1406
|
markCell(event, event.skipped ? "skipped" : event.passed ? "passed" : "failed", event.skipped ? "status-skipped" : event.passed ? "status-passed" : "status-failed");
|
|
1407
|
+
finishedMetrics[cellKey(event)] = Object.assign({}, event.metrics || {}, {
|
|
1408
|
+
durationMs: event.durationMs,
|
|
1409
|
+
passed: event.passed,
|
|
1410
|
+
});
|
|
1411
|
+
renderCellResult(event);
|
|
1412
|
+
if (event.arm) {
|
|
1413
|
+
var finishedWrap = document.getElementById(compareWrapId(event));
|
|
1414
|
+
if (finishedWrap) {
|
|
1415
|
+
markArmTab(
|
|
1416
|
+
finishedWrap,
|
|
1417
|
+
event.arm,
|
|
1418
|
+
event.skipped ? "skipped" : event.passed ? "passed" : "failed"
|
|
1419
|
+
);
|
|
1420
|
+
}
|
|
1421
|
+
renderCompareWinners(event);
|
|
1422
|
+
}
|
|
1423
|
+
return;
|
|
1424
|
+
}
|
|
1425
|
+
if (event.type === "judge") {
|
|
1426
|
+
if (!chatIsOpen(event)) {
|
|
1427
|
+
return;
|
|
1428
|
+
}
|
|
1429
|
+
appendJudgeVerdicts(event);
|
|
309
1430
|
}
|
|
310
1431
|
}
|
|
311
1432
|
|
|
1433
|
+
function markRunTargets(body, label, statusClass) {
|
|
1434
|
+
var hosts = body.hosts && body.hosts.length ? body.hosts : catalog.defaultSelectedHosts;
|
|
1435
|
+
catalog.suites.forEach(function (suite) {
|
|
1436
|
+
if (body.suite && suite.name !== body.suite) {
|
|
1437
|
+
return;
|
|
1438
|
+
}
|
|
1439
|
+
suite.scenarios.forEach(function (scenario) {
|
|
1440
|
+
if (body.scenario && scenario.name !== body.scenario) {
|
|
1441
|
+
return;
|
|
1442
|
+
}
|
|
1443
|
+
if (scenario.skip) {
|
|
1444
|
+
return;
|
|
1445
|
+
}
|
|
1446
|
+
hosts.forEach(function (host) {
|
|
1447
|
+
if (scenario.host && scenario.host !== host) {
|
|
1448
|
+
return;
|
|
1449
|
+
}
|
|
1450
|
+
markCell({ suite: suite.name, scenario: scenario.name, host: host }, label, statusClass);
|
|
1451
|
+
});
|
|
1452
|
+
});
|
|
1453
|
+
});
|
|
1454
|
+
}
|
|
1455
|
+
|
|
312
1456
|
async function startRun(body) {
|
|
313
1457
|
if (runId) {
|
|
314
|
-
setRunBanner(
|
|
1458
|
+
setRunBanner(
|
|
1459
|
+
cancelRequested
|
|
1460
|
+
? "Cancelling the run. Wait for it to stop."
|
|
1461
|
+
: "A run is already in progress. Cancel it first."
|
|
1462
|
+
);
|
|
315
1463
|
return;
|
|
316
1464
|
}
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
body
|
|
1465
|
+
resetProgress(body);
|
|
1466
|
+
markRunTargets(body, "starting", "status-skipped");
|
|
1467
|
+
var firstCard = document.querySelector(
|
|
1468
|
+
body.suite && body.scenario
|
|
1469
|
+
? '[data-scenario-card="' + body.suite + "::" + body.scenario + '"]'
|
|
1470
|
+
: body.suite
|
|
1471
|
+
? '[data-suite="' + body.suite + '"] [data-scenario-card]'
|
|
1472
|
+
: "[data-scenario-card]"
|
|
1473
|
+
);
|
|
1474
|
+
document.querySelectorAll("[data-scenario-card]").forEach(function (card) {
|
|
1475
|
+
if (body.suite && card.getAttribute("data-scenario-card").indexOf(body.suite + "::") !== 0) {
|
|
1476
|
+
return;
|
|
1477
|
+
}
|
|
1478
|
+
if (body.scenario && card.getAttribute("data-scenario-card") !== body.suite + "::" + body.scenario) {
|
|
1479
|
+
return;
|
|
1480
|
+
}
|
|
1481
|
+
delete card.dataset.followHost;
|
|
321
1482
|
});
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
1483
|
+
if (firstCard) {
|
|
1484
|
+
var live = firstCard.querySelector(".scenario-live");
|
|
1485
|
+
if (live) {
|
|
1486
|
+
live.hidden = false;
|
|
1487
|
+
}
|
|
1488
|
+
if (body.hosts && body.hosts.length === 1 && firstCard.dataset.userPickedHost !== "1") {
|
|
1489
|
+
firstCard.dataset.followHost = body.hosts[0];
|
|
1490
|
+
selectHostTab(firstCard, body.hosts[0]);
|
|
1491
|
+
}
|
|
1492
|
+
if (live) {
|
|
1493
|
+
live.scrollIntoView({ block: "nearest" });
|
|
1494
|
+
}
|
|
326
1495
|
}
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
1496
|
+
try {
|
|
1497
|
+
var response = await fetch("/api/runs", {
|
|
1498
|
+
method: "POST",
|
|
1499
|
+
headers: { "content-type": "application/json" },
|
|
1500
|
+
body: JSON.stringify(body),
|
|
1501
|
+
});
|
|
1502
|
+
var payload = await response.json();
|
|
1503
|
+
if (!response.ok) {
|
|
1504
|
+
setRunBanner(payload.error || "Run failed to start.");
|
|
1505
|
+
progress = { total: 0, completed: 0, passed: 0, failed: 0, skipped: 0, seen: {} };
|
|
1506
|
+
renderProgress();
|
|
1507
|
+
markRunTargets(body, "idle", "muted");
|
|
1508
|
+
return;
|
|
1509
|
+
}
|
|
1510
|
+
runId = payload.runId;
|
|
1511
|
+
cancelRequested = false;
|
|
1512
|
+
setRunBanner("Run " + runId + " started.");
|
|
1513
|
+
setCancelButton("Cancel run", false);
|
|
1514
|
+
if (source) {
|
|
1515
|
+
source.close();
|
|
1516
|
+
}
|
|
1517
|
+
source = new EventSource("/api/runs/" + runId + "/events");
|
|
1518
|
+
source.onmessage = function (message) {
|
|
1519
|
+
handleEvent(JSON.parse(message.data));
|
|
1520
|
+
};
|
|
1521
|
+
source.onerror = function () {
|
|
1522
|
+
source.close();
|
|
1523
|
+
if (!runId) {
|
|
1524
|
+
return;
|
|
1525
|
+
}
|
|
1526
|
+
setRunBanner("Live event stream dropped.");
|
|
1527
|
+
};
|
|
1528
|
+
} catch (error) {
|
|
1529
|
+
setRunBanner(error instanceof Error ? error.message : String(error));
|
|
1530
|
+
markRunTargets(body, "idle", "muted");
|
|
330
1531
|
}
|
|
331
|
-
source = new EventSource("/api/runs/" + runId + "/events");
|
|
332
|
-
source.onmessage = function (message) {
|
|
333
|
-
handleEvent(JSON.parse(message.data));
|
|
334
|
-
};
|
|
335
1532
|
}
|
|
336
1533
|
|
|
337
1534
|
document.getElementById("run-selection").addEventListener("click", function () {
|
|
338
1535
|
startRun({ hosts: selectedHosts(), parallelHosts: parallelHosts() });
|
|
339
1536
|
});
|
|
340
1537
|
document.getElementById("cancel-run").addEventListener("click", function () {
|
|
341
|
-
if (!runId) {
|
|
1538
|
+
if (!runId || cancelRequested) {
|
|
342
1539
|
return;
|
|
343
1540
|
}
|
|
344
|
-
|
|
1541
|
+
cancelRequested = true;
|
|
1542
|
+
setRunBanner("Cancelling the run.");
|
|
1543
|
+
setCancelButton("Cancelling", true);
|
|
1544
|
+
cancelOpenChats();
|
|
1545
|
+
fetch("/api/runs/" + runId + "/cancel", { method: "POST" }).then(function (response) {
|
|
1546
|
+
if (!response.ok && cancelRequested) {
|
|
1547
|
+
setRunBanner("Cancel failed.");
|
|
1548
|
+
setCancelButton("Cancel run", false);
|
|
1549
|
+
cancelRequested = false;
|
|
1550
|
+
}
|
|
1551
|
+
}).catch(function () {
|
|
1552
|
+
if (cancelRequested) {
|
|
1553
|
+
setRunBanner("Cancel failed.");
|
|
1554
|
+
setCancelButton("Cancel run", false);
|
|
1555
|
+
cancelRequested = false;
|
|
1556
|
+
}
|
|
1557
|
+
});
|
|
345
1558
|
});
|
|
346
1559
|
document.body.addEventListener("click", function (event) {
|
|
347
1560
|
var button = event.target.closest("button");
|
|
348
1561
|
if (!button) {
|
|
349
1562
|
return;
|
|
350
1563
|
}
|
|
1564
|
+
if (button.classList.contains("host-tab")) {
|
|
1565
|
+
var card = button.closest("[data-scenario-card]");
|
|
1566
|
+
if (card) {
|
|
1567
|
+
card.dataset.userPickedHost = "1";
|
|
1568
|
+
selectHostTab(card, button.getAttribute("data-host"));
|
|
1569
|
+
}
|
|
1570
|
+
}
|
|
351
1571
|
if (button.classList.contains("run-cell")) {
|
|
352
1572
|
startRun({
|
|
353
1573
|
suite: button.getAttribute("data-suite"),
|
|
@@ -408,7 +1628,7 @@ ${viewerCss()}
|
|
|
408
1628
|
<div class="host-toggles">${hostToggles}</div>
|
|
409
1629
|
</div>
|
|
410
1630
|
<div class="toolbar-block">
|
|
411
|
-
<p class="toolbar-label">
|
|
1631
|
+
<p class="toolbar-label">Run order</p>
|
|
412
1632
|
<label><input type="checkbox" id="parallel-hosts"> Run hosts together</label>
|
|
413
1633
|
</div>
|
|
414
1634
|
<div class="toolbar-actions">
|
|
@@ -417,8 +1637,21 @@ ${viewerCss()}
|
|
|
417
1637
|
</div>
|
|
418
1638
|
<p class="run-banner" id="run-banner"></p>
|
|
419
1639
|
</section>
|
|
1640
|
+
<section class="run-progress" id="run-progress" aria-live="polite" hidden>
|
|
1641
|
+
<div class="run-progress-head">
|
|
1642
|
+
<p class="run-progress-title">0 of 0 tests finished</p>
|
|
1643
|
+
</div>
|
|
1644
|
+
<div class="run-progress-track" role="progressbar" aria-label="Suite run progress" aria-valuemin="0" aria-valuenow="0" aria-valuemax="0">
|
|
1645
|
+
<span class="run-progress-fill"></span>
|
|
1646
|
+
</div>
|
|
1647
|
+
<div class="run-progress-counts">
|
|
1648
|
+
<span class="progress-passed">Passed <strong>0</strong></span>
|
|
1649
|
+
<span class="progress-failed">Failed <strong>0</strong></span>
|
|
1650
|
+
<span class="progress-skipped">Skipped <strong>0</strong></span>
|
|
1651
|
+
<span class="progress-remaining">Remaining <strong>0</strong></span>
|
|
1652
|
+
</div>
|
|
1653
|
+
</section>
|
|
420
1654
|
${suites}
|
|
421
|
-
<section class="live-dock" id="live-dock"></section>
|
|
422
1655
|
</main>
|
|
423
1656
|
<script type="application/json" id="catalog-data">${embedJson(catalog)}</script>
|
|
424
1657
|
<script>
|