@post-print/agent-test 1.1.1 → 1.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -2
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +34 -6
- package/dist/cli.js.map +1 -1
- package/dist/compare-scenario.d.ts +10 -0
- package/dist/compare-scenario.d.ts.map +1 -1
- package/dist/compare-scenario.js +48 -3
- package/dist/compare-scenario.js.map +1 -1
- package/dist/expect.d.ts +1 -0
- package/dist/expect.d.ts.map +1 -1
- package/dist/expect.js +24 -0
- package/dist/expect.js.map +1 -1
- package/dist/html-report.d.ts +4 -1
- package/dist/html-report.d.ts.map +1 -1
- package/dist/html-report.js +223 -142
- package/dist/html-report.js.map +1 -1
- package/dist/index.d.ts +3 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -2
- package/dist/index.js.map +1 -1
- package/dist/live-isolation.d.ts +2 -0
- package/dist/live-isolation.d.ts.map +1 -1
- package/dist/live-isolation.js +8 -2
- package/dist/live-isolation.js.map +1 -1
- package/dist/report-preview.d.ts +24 -0
- package/dist/report-preview.d.ts.map +1 -1
- package/dist/report-preview.js +46 -2
- package/dist/report-preview.js.map +1 -1
- package/dist/run-suite.d.ts +10 -0
- package/dist/run-suite.d.ts.map +1 -1
- package/dist/run-suite.js +147 -126
- package/dist/run-suite.js.map +1 -1
- package/dist/scenario-finalizer.d.ts +27 -0
- package/dist/scenario-finalizer.d.ts.map +1 -0
- package/dist/scenario-finalizer.js +39 -0
- package/dist/scenario-finalizer.js.map +1 -0
- package/dist/scenario-story.d.ts.map +1 -1
- package/dist/scenario-story.js +14 -8
- package/dist/scenario-story.js.map +1 -1
- package/dist/types.d.ts +9 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/validate-suite.d.ts.map +1 -1
- package/dist/validate-suite.js +4 -0
- package/dist/validate-suite.js.map +1 -1
- package/dist/viewer/catalog.d.ts +14 -3
- package/dist/viewer/catalog.d.ts.map +1 -1
- package/dist/viewer/catalog.js +37 -6
- package/dist/viewer/catalog.js.map +1 -1
- package/dist/viewer/compare-metrics.d.ts.map +1 -1
- package/dist/viewer/compare-metrics.js +8 -9
- package/dist/viewer/compare-metrics.js.map +1 -1
- package/dist/viewer/events.d.ts +22 -1
- package/dist/viewer/events.d.ts.map +1 -1
- package/dist/viewer/events.js +1 -0
- package/dist/viewer/events.js.map +1 -1
- package/dist/viewer/live-runner.d.ts +7 -0
- package/dist/viewer/live-runner.d.ts.map +1 -1
- package/dist/viewer/live-runner.js +36 -23
- package/dist/viewer/live-runner.js.map +1 -1
- package/dist/viewer/page.d.ts +14 -1
- package/dist/viewer/page.d.ts.map +1 -1
- package/dist/viewer/page.js +848 -141
- package/dist/viewer/page.js.map +1 -1
- package/dist/viewer/run-controller.d.ts +13 -6
- package/dist/viewer/run-controller.d.ts.map +1 -1
- package/dist/viewer/run-controller.js +246 -74
- package/dist/viewer/run-controller.js.map +1 -1
- package/dist/viewer/server.d.ts +5 -0
- package/dist/viewer/server.d.ts.map +1 -1
- package/dist/viewer/server.js +72 -17
- package/dist/viewer/server.js.map +1 -1
- package/dist/viewer/session-server.d.ts +2 -0
- package/dist/viewer/session-server.d.ts.map +1 -0
- package/dist/viewer/session-server.js +71 -0
- package/dist/viewer/session-server.js.map +1 -0
- package/package.json +2 -2
package/dist/viewer/page.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { describeCompareGate } from "../compare-scenario.js";
|
|
2
2
|
function escapeHtml(value) {
|
|
3
3
|
return value
|
|
4
4
|
.replaceAll("&", "&")
|
|
@@ -25,6 +25,7 @@ function viewerCss() {
|
|
|
25
25
|
margin-bottom: 1.25rem;
|
|
26
26
|
}
|
|
27
27
|
.toolbar-block { display: grid; gap: 0.35rem; }
|
|
28
|
+
.run-history { min-width: min(26rem, 80vw); }
|
|
28
29
|
.toolbar-label { color: var(--muted); font-size: 0.75rem; font-weight: 650; text-transform: uppercase; letter-spacing: 0.04em; }
|
|
29
30
|
.host-toggles, .toolbar-actions { display: flex; flex-wrap: wrap; gap: 0.45rem; align-items: center; }
|
|
30
31
|
.host-toggles label { display: inline-flex; gap: 0.35rem; align-items: center; font-size: 0.85rem; }
|
|
@@ -72,6 +73,8 @@ function viewerCss() {
|
|
|
72
73
|
.host-tab[data-status="passed"] { color: var(--pass); }
|
|
73
74
|
.host-tab[data-status="failed"] { color: var(--fail); }
|
|
74
75
|
.host-tab[data-status="skipped"], .host-tab[data-status="skip"] { color: var(--muted); }
|
|
76
|
+
.host-tab[hidden], .host-empty-selection[hidden] { display: none; }
|
|
77
|
+
.host-empty-selection { color: var(--muted); font-size: .78rem; }
|
|
75
78
|
.host-status { display: inline-flex; align-items: center; }
|
|
76
79
|
.cell-status { font-size: 0.72rem; font-weight: 700; text-transform: uppercase; }
|
|
77
80
|
.scenario-live { display: grid; gap: 0.75rem; }
|
|
@@ -86,6 +89,14 @@ function viewerCss() {
|
|
|
86
89
|
border-left: 3px solid var(--border);
|
|
87
90
|
padding: 0.45rem 0.7rem;
|
|
88
91
|
}
|
|
92
|
+
.cell-result-head, .failure-section-head {
|
|
93
|
+
display: flex;
|
|
94
|
+
align-items: baseline;
|
|
95
|
+
justify-content: space-between;
|
|
96
|
+
gap: 0.75rem;
|
|
97
|
+
}
|
|
98
|
+
.cell-result-head .cell-result-verdict, .failure-section-head h3 { margin: 0; }
|
|
99
|
+
.copy-error { flex: none; padding: 0.2rem 0.45rem; font-size: 0.72rem; }
|
|
89
100
|
.cell-result.status-passed { border-left-color: var(--pass); }
|
|
90
101
|
.cell-result.status-failed { border-left-color: var(--fail); }
|
|
91
102
|
.cell-result.status-skipped { border-left-color: var(--skip); }
|
|
@@ -268,52 +279,324 @@ function viewerCss() {
|
|
|
268
279
|
@media (prefers-reduced-motion: reduce) {
|
|
269
280
|
.chat-progress span, .bubble.is-streaming .bubble-text::after { animation: none; }
|
|
270
281
|
}
|
|
282
|
+
.scenario-card {
|
|
283
|
+
background: linear-gradient(135deg, color-mix(in oklch, var(--panel) 94%, white), var(--panel));
|
|
284
|
+
padding: 1rem 1.05rem;
|
|
285
|
+
}
|
|
286
|
+
.scenario-detail { gap: 0.4rem; }
|
|
287
|
+
.scenario-detail strong { font-size: 1.05rem; letter-spacing: -0.01em; }
|
|
288
|
+
.scenario-toolbar { gap: 0.45rem 0.85rem; padding-top: 0.6rem; border-top: 1px solid var(--border); }
|
|
289
|
+
.host-runs { display: flex; gap: 0.4rem; align-items: center; }
|
|
290
|
+
.host-runs-label { color: var(--muted); font-size: 0.7rem; font-weight: 700; text-transform: uppercase; letter-spacing: 0.05em; }
|
|
291
|
+
.host-tablist { gap: 0.25rem; }
|
|
292
|
+
.host-tab { gap: 0.28rem; padding: 0.25rem 0.45rem; font-size: 0.82rem; }
|
|
293
|
+
body { background: oklch(0.155 0.015 250); }
|
|
294
|
+
.viewer-shell { max-width: none; min-height: 100dvh; padding: 0; }
|
|
295
|
+
.viewer-topbar {
|
|
296
|
+
position: sticky; inset-block-start: 0; z-index: 10;
|
|
297
|
+
display: grid; grid-template-columns: minmax(12rem, .7fr) minmax(18rem, 1.2fr) minmax(14rem, 1fr);
|
|
298
|
+
align-items: center; gap: 1rem; min-height: 4.5rem;
|
|
299
|
+
padding: .75rem clamp(1rem, 2.5vw, 2rem);
|
|
300
|
+
background: color-mix(in oklch, var(--bg) 92%, transparent);
|
|
301
|
+
border-block-end: 1px solid var(--border); backdrop-filter: blur(18px);
|
|
302
|
+
}
|
|
303
|
+
.viewer-identity { display: flex; align-items: baseline; gap: .65rem; min-width: 0; }
|
|
304
|
+
.viewer-identity .brand { color: var(--pass); }
|
|
305
|
+
.viewer-identity h1 { font-size: 1.05rem; white-space: nowrap; }
|
|
306
|
+
.run-context { display: grid; grid-template-columns: auto minmax(0, 1fr); align-items: center; gap: .5rem; min-width: 0; }
|
|
307
|
+
.run-history { width: 100%; min-width: 0; border-radius: 7px; background: var(--panel-2); color: var(--text); border: 1px solid var(--border); padding: .42rem .6rem; }
|
|
308
|
+
.run-banner { margin: 0; overflow: hidden; color: var(--muted); font-size: .78rem; text-align: end; text-overflow: ellipsis; white-space: nowrap; }
|
|
309
|
+
.viewer-command-row { display: flex; flex-wrap: wrap; align-items: center; gap: .55rem 1rem; padding: .55rem clamp(1rem, 2.5vw, 2rem); background: var(--panel-2); border-block-end: 1px solid var(--border); }
|
|
310
|
+
.viewer-command-row .toolbar-block { display: flex; flex-wrap: wrap; align-items: center; gap: .4rem .55rem; }
|
|
311
|
+
.viewer-command-row .host-toggles { gap: .25rem .55rem; }
|
|
312
|
+
.viewer-command-row .parallel-control { display: inline-flex; align-items: center; gap: .35rem; color: var(--muted); font-size: .78rem; white-space: nowrap; }
|
|
313
|
+
.viewer-command-row .run-toggle { min-width: 5.75rem; margin-inline-start: auto; padding-block: .3rem; font-size: .78rem; }
|
|
314
|
+
.run-progress { margin: 0; padding: .7rem clamp(1rem, 2.5vw, 2rem); border: 0; border-block-end: 1px solid var(--border); border-radius: 0; background: var(--bg); }
|
|
315
|
+
.run-progress-track { height: .35rem; }
|
|
316
|
+
.viewer-workspace { display: grid; grid-template-columns: minmax(15rem, 19rem) minmax(0, 1fr); min-height: calc(100dvh - 7.5rem); }
|
|
317
|
+
.test-navigator { position: sticky; inset-block-start: 7.5rem; align-self: start; height: calc(100dvh - 7.5rem); overflow: auto; padding: 1.15rem .75rem 2rem; background: color-mix(in oklch, var(--panel-2) 72%, var(--bg)); border-inline-end: 1px solid var(--border); }
|
|
318
|
+
.test-navigator-heading { display: flex; justify-content: space-between; align-items: end; padding-inline: .5rem; margin-block-end: 1rem; }
|
|
319
|
+
.section-label, .scenario-path { margin: 0; color: var(--muted); font-size: .68rem; font-weight: 700; letter-spacing: .08em; text-transform: uppercase; }
|
|
320
|
+
.test-navigator-heading h2 { margin: .1rem 0 0; font-size: 1rem; }
|
|
321
|
+
.test-picker, .test-picker-label { display: none; }
|
|
322
|
+
.test-tree { display: grid; gap: 1.1rem; }
|
|
323
|
+
.test-nav-suite { display: grid; gap: .35rem; }
|
|
324
|
+
.test-nav-suite > header { display: flex; align-items: center; justify-content: space-between; gap: .5rem; padding-inline: .45rem; }
|
|
325
|
+
.test-nav-suite > header h2 { margin: 0; color: var(--muted); font-size: .72rem; letter-spacing: .05em; text-transform: uppercase; }
|
|
326
|
+
.test-nav-suite > header p { display: none; }
|
|
327
|
+
.test-nav-suite > header .run-suite { padding: .12rem .35rem; color: var(--muted); border: 0; background: transparent; font-size: .7rem; }
|
|
328
|
+
.test-nav-items { display: grid; gap: .18rem; }
|
|
329
|
+
.test-nav-item { display: grid; grid-template-columns: .6rem minmax(0, 1fr); align-items: start; gap: .55rem; width: 100%; padding: .5rem .55rem; border-color: transparent; background: transparent; text-align: start; }
|
|
330
|
+
.test-nav-item:hover { background: color-mix(in oklch, var(--text) 5%, transparent); }
|
|
331
|
+
.test-nav-item[aria-current="true"] { background: color-mix(in oklch, var(--pass) 9%, var(--panel)); border-color: color-mix(in oklch, var(--pass) 28%, var(--border)); }
|
|
332
|
+
.test-nav-status { width: .48rem; height: .48rem; margin-block-start: .28rem; border: 1px solid var(--muted); border-radius: 50%; }
|
|
333
|
+
.test-nav-item[data-status="running"] .test-nav-status { border-color: var(--skip); background: var(--skip); box-shadow: 0 0 0 3px color-mix(in oklch, var(--skip) 14%, transparent); }
|
|
334
|
+
.test-nav-item[data-status="passed"] .test-nav-status { border-color: var(--pass); background: var(--pass); }
|
|
335
|
+
.test-nav-item[data-status="failed"] .test-nav-status { border-color: var(--fail); background: var(--fail); }
|
|
336
|
+
.test-nav-item[data-status="skipped"] .test-nav-status { border-color: var(--skip); }
|
|
337
|
+
.test-nav-copy { display: grid; min-width: 0; }
|
|
338
|
+
.test-nav-copy strong { overflow: hidden; font-size: .82rem; font-weight: 600; text-overflow: ellipsis; white-space: nowrap; }
|
|
339
|
+
.test-nav-copy small { overflow: hidden; color: var(--muted); font-size: .7rem; text-overflow: ellipsis; white-space: nowrap; }
|
|
340
|
+
.test-stage { min-width: 0; padding: clamp(1rem, 3vw, 2.25rem); }
|
|
341
|
+
.test-stage > .suite { display: block; margin: 0; }
|
|
342
|
+
.test-stage > .suite > .suite-header { display: none; }
|
|
343
|
+
.test-stage .scenario-list { display: contents; }
|
|
344
|
+
.scenario-card, .scenario-card[data-selected="true"] { max-width: 76rem; margin-inline: auto; padding: 0; gap: 0; overflow: hidden; background: var(--panel); border: 1px solid var(--border); border-radius: 12px; box-shadow: 0 12px 36px oklch(0 0 0 / .16); }
|
|
345
|
+
.scenario-card[hidden] { display: none; }
|
|
346
|
+
.scenario-focus-header { display: flex; align-items: flex-start; justify-content: space-between; gap: 1rem; padding: clamp(1rem, 2vw, 1.5rem); border-block-end: 1px solid var(--border); }
|
|
347
|
+
.scenario-heading-copy { display: grid; gap: .3rem; min-width: 0; }
|
|
348
|
+
.scenario-heading-copy h2 { margin: 0; font-size: clamp(1.25rem, 2.5vw, 1.75rem); letter-spacing: -.025em; }
|
|
349
|
+
.scenario-heading-copy .suite-description { max-width: 62ch; color: var(--muted); font-size: .9rem; }
|
|
350
|
+
.scenario-heading-copy .scenario-lede { max-width: 62ch; font-size: .9rem; }
|
|
351
|
+
.focus-verdict { display: inline-flex; align-items: center; gap: .4rem; flex: none; padding: .28rem .55rem; color: var(--muted); border: 1px solid var(--border); border-radius: 999px; font-size: .72rem; font-weight: 700; text-transform: uppercase; }
|
|
352
|
+
.focus-verdict > span { width: .45rem; height: .45rem; border-radius: 50%; background: currentColor; }
|
|
353
|
+
.focus-verdict[data-status="running"] { color: var(--skip); }
|
|
354
|
+
.focus-verdict[data-status="passed"] { color: var(--pass); }
|
|
355
|
+
.focus-verdict[data-status="failed"] { color: var(--fail); }
|
|
356
|
+
.scenario-definition { display: grid; grid-template-columns: minmax(0, 1.5fr) minmax(15rem, .7fr); gap: .8rem; padding: clamp(1rem, 2vw, 1.5rem); background: color-mix(in oklch, var(--panel) 88%, var(--bg)); }
|
|
357
|
+
.test-intent, .criteria-panel, .compare-definition, .comparison-criteria { min-width: 0; padding: .9rem; background: var(--panel-2); border: 1px solid var(--border); border-radius: 8px; }
|
|
358
|
+
.prompt-preview { margin: .45rem 0 0; color: var(--text); font-size: .94rem; line-height: 1.55; }
|
|
359
|
+
.task-metadata { display: flex; flex-wrap: wrap; align-items: baseline; gap: .3rem .55rem; margin-block-start: .75rem; padding-block-start: .65rem; border-block-start: 1px solid var(--border); }
|
|
360
|
+
.task-metadata + .task-metadata { margin-block-start: .55rem; padding-block-start: 0; border-block-start: 0; }
|
|
361
|
+
.task-metadata .task-meta-label { color: var(--muted); font-size: .66rem; font-weight: 700; letter-spacing: .07em; text-transform: uppercase; }
|
|
362
|
+
.task-metadata p { margin: 0; color: var(--muted); font-size: .76rem; line-height: 1.6; }
|
|
363
|
+
.task-metadata code { padding: .08rem .28rem; background: var(--bg); border: 1px solid var(--border); border-radius: 4px; overflow-wrap: anywhere; }
|
|
364
|
+
.criteria-panel > summary { display: flex; justify-content: space-between; gap: .5rem; cursor: pointer; font-size: .82rem; font-weight: 650; }
|
|
365
|
+
.criteria-summary { color: var(--muted); font-size: .72rem; font-weight: 400; }
|
|
366
|
+
.criterion-groups { display: grid; gap: .65rem; margin-block-start: .75rem; }
|
|
367
|
+
.criterion-group { display: grid; gap: .35rem; padding-block-start: .6rem; border-block-start: 1px solid var(--border); }
|
|
368
|
+
.criterion-group:first-child { padding-block-start: 0; border-block-start: 0; }
|
|
369
|
+
.criterion-group h4 { margin: 0; color: var(--muted); font-size: .66rem; font-weight: 700; letter-spacing: .07em; text-transform: uppercase; }
|
|
370
|
+
.criterion-group p { margin: 0; color: var(--muted); font-size: .76rem; line-height: 1.4; }
|
|
371
|
+
.criterion-list { display: grid; gap: .28rem; margin: 0; padding: 0; list-style: none; }
|
|
372
|
+
.criterion-list li { color: var(--text); font-size: .78rem; line-height: 1.4; }
|
|
373
|
+
.criterion-list code { padding: .08rem .28rem; background: var(--bg); border: 1px solid var(--border); border-radius: 4px; overflow-wrap: anywhere; }
|
|
374
|
+
.criteria-empty { margin: .7rem 0 0; color: var(--muted); font-size: .78rem; line-height: 1.45; }
|
|
375
|
+
.compare-definition { grid-column: 1 / -1; }
|
|
376
|
+
.comparison-intro { margin: .35rem 0 0; color: var(--muted); font-size: .78rem; }
|
|
377
|
+
.compare-definition { padding: 0; background: transparent; border: 0; }
|
|
378
|
+
.compare-definition-tabs { display: flex; flex-wrap: wrap; gap: .35rem; margin-block-start: .55rem; }
|
|
379
|
+
.compare-definition-tab-input { position: absolute; inline-size: 1px; block-size: 1px; opacity: 0; pointer-events: none; }
|
|
380
|
+
.compare-definition-tab { padding: .35rem .65rem; color: var(--muted); background: var(--panel-2); border: 1px solid var(--border); border-radius: 7px; cursor: pointer; font-size: .78rem; }
|
|
381
|
+
.compare-definition-tab-input:checked + .compare-definition-tab { color: var(--text); background: color-mix(in oklch, var(--pass) 12%, var(--panel-2)); border-color: color-mix(in oklch, var(--pass) 36%, var(--border)); }
|
|
382
|
+
.compare-definition-tab-input:focus-visible + .compare-definition-tab { outline: 2px solid var(--pass); outline-offset: 2px; }
|
|
383
|
+
.compare-definition-grid { display: grid; grid-template-columns: minmax(0, 1fr); margin-block-start: .55rem; }
|
|
384
|
+
.compare-arm-definition { min-width: 0; overflow: hidden; background: var(--panel-2); border: 1px solid var(--border); border-radius: 8px; }
|
|
385
|
+
.compare-definition-tabbed .compare-arm-definition { display: none; }
|
|
386
|
+
.compare-arm-header { display: grid; gap: .2rem; padding: .8rem .9rem; border-block-end: 1px solid var(--border); }
|
|
387
|
+
.compare-arm-header h3 { margin: 0; font-size: .95rem; }
|
|
388
|
+
.compare-arm-description { margin: 0; color: var(--muted); font-size: .76rem; line-height: 1.45; }
|
|
389
|
+
.compare-arm-definition .test-intent, .compare-arm-definition .criteria-panel { border: 0; border-radius: 0; }
|
|
390
|
+
.compare-arm-definition .test-intent { background: transparent; }
|
|
391
|
+
.compare-arm-definition .criteria-panel { border-block-start: 1px solid var(--border); background: color-mix(in oklch, var(--panel-2) 82%, var(--bg)); }
|
|
392
|
+
.scenario-toolbar { justify-content: space-between; gap: .75rem 1rem; padding: .8rem clamp(1rem, 2vw, 1.5rem); background: var(--panel); border-block: 1px solid var(--border); }
|
|
393
|
+
.host-runs-label { font-size: .65rem; }
|
|
394
|
+
.host-tab { padding: .3rem .5rem; background: transparent; }
|
|
395
|
+
.host-tab[aria-selected="true"] { color: var(--text); background: color-mix(in oklch, var(--pass) 11%, var(--panel-2)); }
|
|
396
|
+
.scenario-actions .run-toggle { min-width: 5.6rem; padding: .28rem .55rem; font-size: .78rem; }
|
|
397
|
+
.scenario-live { padding: clamp(1rem, 2vw, 1.5rem); background: var(--bg); }
|
|
398
|
+
.scenario-live .host-empty { padding: 2rem; text-align: center; border: 1px dashed var(--border); border-radius: 8px; }
|
|
399
|
+
@container (max-width: 48rem) { .scenario-definition { grid-template-columns: minmax(0, 1fr); } .compare-definition { grid-column: auto; } }
|
|
400
|
+
@media (max-width: 800px) {
|
|
401
|
+
.viewer-topbar { position: static; grid-template-columns: minmax(0, 1fr); gap: .6rem; padding-block: .85rem; }
|
|
402
|
+
.viewer-identity { justify-content: space-between; }
|
|
403
|
+
.run-context { grid-template-columns: minmax(0, 1fr); gap: .25rem; }
|
|
404
|
+
.run-banner { text-align: start; }
|
|
405
|
+
.viewer-command-row { position: sticky; inset-block-start: 0; z-index: 9; }
|
|
406
|
+
.viewer-workspace { display: block; min-height: 0; }
|
|
407
|
+
.test-navigator { position: static; height: auto; padding: .75rem 1rem; border-inline-end: 0; border-block-end: 1px solid var(--border); }
|
|
408
|
+
.test-navigator-heading, .test-tree { display: none; }
|
|
409
|
+
.test-picker-label { display: block; margin-block-end: .25rem; color: var(--muted); font-size: .68rem; font-weight: 700; text-transform: uppercase; }
|
|
410
|
+
.test-picker { display: block; width: 100%; padding: .5rem .6rem; border: 1px solid var(--border); border-radius: 7px; background: var(--panel); color: var(--text); }
|
|
411
|
+
.test-stage { padding: .8rem; }
|
|
412
|
+
.scenario-focus-header { align-items: center; }
|
|
413
|
+
.scenario-definition { grid-template-columns: minmax(0, 1fr); }
|
|
414
|
+
.compare-definition { grid-column: auto; }
|
|
415
|
+
.scenario-toolbar { align-items: flex-start; }
|
|
416
|
+
.host-runs { align-items: flex-start; flex-direction: column; }
|
|
417
|
+
}
|
|
418
|
+
@media (max-width: 480px) { .scenario-focus-header { display: grid; } .focus-verdict { justify-self: start; } .scenario-toolbar { display: grid; } .scenario-actions { width: 100%; } .scenario-actions .primary { flex: 1; } .viewer-command-row .run-toggle { width: 100%; margin-inline-start: 0; } }
|
|
271
419
|
`;
|
|
272
420
|
}
|
|
273
|
-
function
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
const
|
|
278
|
-
if (
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
}
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
}
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
421
|
+
function criterionCode(value) {
|
|
422
|
+
return `<code>${escapeHtml(value)}</code>`;
|
|
423
|
+
}
|
|
424
|
+
function renderAllowedCommands(target) {
|
|
425
|
+
const commands = target.rubric.allowedCommands;
|
|
426
|
+
if (commands === undefined)
|
|
427
|
+
return "";
|
|
428
|
+
const list = commands.length
|
|
429
|
+
? commands.map(criterionCode).join('<span aria-hidden="true">, </span>')
|
|
430
|
+
: '<span class="task-command-empty">None</span>';
|
|
431
|
+
return `<div class="task-metadata task-allowed-commands"><span class="task-meta-label">Allowed commands</span><p>${list}</p></div>`;
|
|
432
|
+
}
|
|
433
|
+
function renderProvidedContext(target) {
|
|
434
|
+
if (!target.contextSources?.length)
|
|
435
|
+
return "";
|
|
436
|
+
const sources = target.contextSources
|
|
437
|
+
.map(criterionCode)
|
|
438
|
+
.join('<span aria-hidden="true">, </span>');
|
|
439
|
+
return `<div class="task-metadata task-provided-context"><span class="task-meta-label">Provided context</span><p>${sources}</p></div>`;
|
|
440
|
+
}
|
|
441
|
+
function renderSuppliedResources(target, showEmpty = false) {
|
|
442
|
+
const servers = target.suppliedMcp ?? [];
|
|
443
|
+
const tools = [...new Set(servers.flatMap((server) => server.tools))];
|
|
444
|
+
const skills = target.suppliedSkills ?? [];
|
|
445
|
+
const value = (items) => items.length
|
|
446
|
+
? items.map(criterionCode).join('<span aria-hidden="true">, </span>')
|
|
447
|
+
: '<span class="task-resource-empty">None</span>';
|
|
448
|
+
if (!showEmpty && servers.length === 0 && skills.length === 0)
|
|
449
|
+
return "";
|
|
450
|
+
return `<div class="task-resources">
|
|
451
|
+
<div class="task-metadata"><span class="task-meta-label">Supplied MCP servers</span><p>${value(servers.map((server) => server.name))}</p></div>
|
|
452
|
+
<div class="task-metadata"><span class="task-meta-label">Supplied MCP tools</span><p>${value(tools)}</p></div>
|
|
453
|
+
<div class="task-metadata"><span class="task-meta-label">Supplied skills</span><p>${value(skills)}</p></div>
|
|
454
|
+
</div>`;
|
|
455
|
+
}
|
|
456
|
+
function renderPassCriteria(target) {
|
|
457
|
+
const groups = [];
|
|
458
|
+
const rubric = target.rubric;
|
|
459
|
+
const reply = [];
|
|
460
|
+
for (const text of rubric.must ?? []) {
|
|
461
|
+
reply.push(`Final reply must include ${criterionCode(text)}.`);
|
|
462
|
+
}
|
|
463
|
+
for (const text of rubric.mustNot ?? []) {
|
|
464
|
+
reply.push(`The run must not contain ${criterionCode(text)}.`);
|
|
465
|
+
}
|
|
466
|
+
if (reply.length)
|
|
467
|
+
groups.push({ title: "Reply", items: reply });
|
|
468
|
+
if (rubric.mustRun?.length || rubric.mustRunSuccessfully?.length) {
|
|
469
|
+
groups.push({
|
|
470
|
+
title: "Required commands",
|
|
471
|
+
items: [
|
|
472
|
+
...(rubric.mustRun ?? []).map((command) => `Must run a command matching ${criterionCode(command)}.`),
|
|
473
|
+
...(rubric.mustRunSuccessfully ?? []).map((command) => `Must run a command matching ${criterionCode(command)} successfully.`),
|
|
474
|
+
],
|
|
475
|
+
});
|
|
300
476
|
}
|
|
301
|
-
|
|
302
|
-
|
|
477
|
+
const tools = [
|
|
478
|
+
...(rubric.mustCallTool ?? []).map((tool) => `Must call ${criterionCode(tool)}.`),
|
|
479
|
+
...(rubric.mustCallToolsInOrder ?? []).map((tool, index) => `${index + 1}. Call ${criterionCode(tool)}.`),
|
|
480
|
+
...(rubric.mustNotCallTool ?? []).map((tool) => `Must not call ${criterionCode(tool)}.`),
|
|
481
|
+
];
|
|
482
|
+
if (tools.length) {
|
|
483
|
+
groups.push({
|
|
484
|
+
title: "Tools",
|
|
485
|
+
intro: rubric.mustCallToolsInOrder?.length ? "Ordered tool calls are numbered." : undefined,
|
|
486
|
+
items: tools,
|
|
487
|
+
});
|
|
303
488
|
}
|
|
304
|
-
|
|
305
|
-
|
|
489
|
+
const files = [
|
|
490
|
+
...(rubric.mustReadPath ?? []).map((path) => `Must read a path matching ${criterionCode(path)}.`),
|
|
491
|
+
...(rubric.mustNotReadPath ?? []).map((path) => `Must not read a path matching ${criterionCode(path)}.`),
|
|
492
|
+
];
|
|
493
|
+
if (files.length)
|
|
494
|
+
groups.push({ title: "Files", items: files });
|
|
495
|
+
const skills = [
|
|
496
|
+
...(rubric.mustInvokeSkill ?? []).map((skill) => `Must use ${criterionCode(skill)}.`),
|
|
497
|
+
...(rubric.mustNotInvokeSkill ?? []).map((skill) => `Must not use ${criterionCode(skill)}.`),
|
|
498
|
+
];
|
|
499
|
+
if (skills.length)
|
|
500
|
+
groups.push({ title: "Skills", items: skills });
|
|
501
|
+
const routing = [
|
|
502
|
+
...(rubric.handsOnRouting
|
|
503
|
+
? ["Must announce hands-on routing before the first tool call."]
|
|
504
|
+
: []),
|
|
505
|
+
...(rubric.tier ? [`Must announce the ${criterionCode(rubric.tier)} tier.`] : []),
|
|
506
|
+
...(rubric.reviewDepth
|
|
507
|
+
? [`Must announce ${criterionCode(rubric.reviewDepth)} review depth.`]
|
|
508
|
+
: []),
|
|
509
|
+
...(rubric.routingBlock ? ["Must announce a routing block."] : []),
|
|
510
|
+
];
|
|
511
|
+
if (routing.length)
|
|
512
|
+
groups.push({ title: "Routing", items: routing });
|
|
513
|
+
if (rubric.judge?.length) {
|
|
514
|
+
groups.push({
|
|
515
|
+
title: "Judge",
|
|
516
|
+
intro: "A judge evaluates these questions after the run.",
|
|
517
|
+
items: rubric.judge.map((item) => escapeHtml(typeof item === "string" ? item : item.question)),
|
|
518
|
+
});
|
|
306
519
|
}
|
|
307
|
-
|
|
520
|
+
const count = groups.reduce((total, group) => total + Math.max(group.items.length, 1), 0);
|
|
521
|
+
const html = groups
|
|
522
|
+
.map((group) => `<section class="criterion-group"><h4>${escapeHtml(group.title)}</h4>${group.intro ? `<p>${escapeHtml(group.intro)}</p>` : ""}${group.items.length ? `<ul class="criterion-list">${group.items.map((item) => `<li>${item}</li>`).join("")}</ul>` : ""}</section>`)
|
|
523
|
+
.join("");
|
|
524
|
+
return { count, html };
|
|
525
|
+
}
|
|
526
|
+
function armLabel(scenario, id) {
|
|
527
|
+
return scenario.compare?.find((arm) => arm.id === id)?.label ?? id;
|
|
528
|
+
}
|
|
529
|
+
function renderComparisonCriteria(scenario) {
|
|
530
|
+
const items = scenario.gates?.length
|
|
531
|
+
? scenario.gates.map((gate) => `${escapeHtml(describeCompareGate(gate, (id) => armLabel(scenario, id)))}.`)
|
|
532
|
+
: (scenario.compare ?? []).map((arm) => `${escapeHtml(arm.label)} must pass all of its criteria.`);
|
|
533
|
+
const acceptableBehavior = scenario.judgeMetrics?.length
|
|
534
|
+
? `<section class="criterion-group comparison-acceptable-behavior"><h4>Acceptable behavior</h4><p>The same questions are judged separately for every arm. The arm-result rules below decide which verdicts are required.</p><ul class="criterion-list">${scenario.judgeMetrics.map((metric) => `<li>${escapeHtml(metric.question)}</li>`).join("")}</ul></section>`
|
|
535
|
+
: "";
|
|
536
|
+
const sharedJudge = scenario.rubric.judge?.length
|
|
537
|
+
? `<section class="criterion-group comparison-shared-judge"><h4>Comparison judge</h4><p>One judge sees all completed arms and evaluates the comparison as a whole.</p><ul class="criterion-list">${scenario.rubric.judge.map((item) => `<li>${escapeHtml(typeof item === "string" ? item : item.question)}</li>`).join("")}</ul></section>`
|
|
538
|
+
: "";
|
|
539
|
+
return {
|
|
540
|
+
count: items.length + (scenario.judgeMetrics?.length ?? 0) + (scenario.rubric.judge?.length ?? 0),
|
|
541
|
+
html: `${acceptableBehavior}${sharedJudge}<section class="criterion-group"><h4>Arm results</h4><ul class="criterion-list">${items.map((item) => `<li>${item}</li>`).join("")}</ul></section>`,
|
|
542
|
+
};
|
|
543
|
+
}
|
|
544
|
+
function renderCompareArm(arm) {
|
|
545
|
+
const criteria = renderPassCriteria(arm);
|
|
546
|
+
return `<article class="compare-arm-definition" data-compare-arm-definition="${escapeHtml(arm.id)}">
|
|
547
|
+
<header class="compare-arm-header">
|
|
548
|
+
<p class="section-label">Arm</p>
|
|
549
|
+
<h3>${escapeHtml(arm.label)}</h3>
|
|
550
|
+
${arm.description ? `<p class="compare-arm-description">${escapeHtml(arm.description)}</p>` : ""}
|
|
551
|
+
</header>
|
|
552
|
+
<section class="test-intent">
|
|
553
|
+
<p class="section-label">Task</p>
|
|
554
|
+
<p class="prompt-preview">${escapeHtml(arm.prompt)}</p>
|
|
555
|
+
${renderAllowedCommands(arm)}
|
|
556
|
+
${renderProvidedContext(arm)}
|
|
557
|
+
${renderSuppliedResources(arm, true)}
|
|
558
|
+
</section>
|
|
559
|
+
<details class="criteria-panel" open>
|
|
560
|
+
<summary><span>Pass criteria</span><span class="criteria-summary">${criteria.count ? `${criteria.count} ${criteria.count === 1 ? "check" : "checks"}` : "Completion only"}</span></summary>
|
|
561
|
+
${criteria.html ? `<div class="criterion-groups">${criteria.html}</div>` : '<p class="criteria-empty">This arm passes when it completes without a runner error.</p>'}
|
|
562
|
+
</details>
|
|
563
|
+
</article>`;
|
|
564
|
+
}
|
|
565
|
+
function comparisonDefinitionGroupId(suite, scenario) {
|
|
566
|
+
return `cd-${suite.name}-${scenario.name}`
|
|
567
|
+
.toLowerCase()
|
|
568
|
+
.replace(/[^a-z0-9]+/g, "-")
|
|
569
|
+
.replace(/^-+|-+$/g, "");
|
|
570
|
+
}
|
|
571
|
+
function renderCompareDefinition(suite, scenario) {
|
|
572
|
+
const arms = scenario.compare ?? [];
|
|
573
|
+
const group = comparisonDefinitionGroupId(suite, scenario);
|
|
574
|
+
const tabs = arms
|
|
575
|
+
.map((arm, index) => {
|
|
576
|
+
const id = `${group}-${arm.id}`;
|
|
577
|
+
return `<input type="radio" class="compare-definition-tab-input" name="${escapeHtml(group)}" id="${escapeHtml(id)}"${index === 0 ? " checked" : ""}>
|
|
578
|
+
<label class="compare-definition-tab" role="tab" for="${escapeHtml(id)}">${escapeHtml(arm.label)}</label>`;
|
|
579
|
+
})
|
|
580
|
+
.join("");
|
|
581
|
+
const rules = arms
|
|
582
|
+
.map((arm) => `.${group}:has(#${group}-${arm.id}:checked) [data-compare-arm-definition="${arm.id}"]{display:block}`)
|
|
583
|
+
.join("");
|
|
584
|
+
return `<style>${rules}</style>
|
|
585
|
+
<section class="compare-definition compare-definition-tabbed ${escapeHtml(group)}">
|
|
586
|
+
<p class="section-label">Arms</p>
|
|
587
|
+
<div class="compare-definition-tabs" role="tablist" aria-label="Comparison arm definitions">${tabs}</div>
|
|
588
|
+
<div class="compare-definition-grid">${arms.map(renderCompareArm).join("")}</div>
|
|
589
|
+
</section>`;
|
|
308
590
|
}
|
|
309
|
-
function defaultScenarioHost(suite, scenario) {
|
|
591
|
+
function defaultScenarioHost(suite, scenario, defaultSelectedHosts) {
|
|
310
592
|
if (scenario.host && suite.hosts.includes(scenario.host)) {
|
|
311
593
|
return scenario.host;
|
|
312
594
|
}
|
|
313
|
-
|
|
314
|
-
|
|
595
|
+
const configured = suite.hosts.find((host) => defaultSelectedHosts.includes(host));
|
|
596
|
+
if (configured) {
|
|
597
|
+
return configured;
|
|
315
598
|
}
|
|
316
|
-
return suite.hosts[0] ?? "
|
|
599
|
+
return suite.hosts.length === 1 ? (suite.hosts[0] ?? "") : "";
|
|
317
600
|
}
|
|
318
601
|
function hostIsSkipped(scenario, host) {
|
|
319
602
|
return Boolean(scenario.skip || (scenario.host && scenario.host !== host));
|
|
@@ -333,67 +616,159 @@ function renderHostChrome(suite, scenario, selectedHost) {
|
|
|
333
616
|
return `<button type="button" class="host-tab" role="tab" aria-label="${escapeHtml(host)}" data-cell="${escapeHtml(`${suite.name}::${scenario.name}::${host}`)}" data-host="${escapeHtml(host)}" aria-selected="${selected ? "true" : "false"}"${skipped ? " disabled" : ""}>${escapeHtml(host)} <span class="cell-status muted" aria-hidden="true">${skipped ? "skip" : "idle"}</span></button>`;
|
|
334
617
|
})
|
|
335
618
|
.join("");
|
|
336
|
-
return `<div class="host-tablist" role="tablist">${tabs}</div>`;
|
|
619
|
+
return `<div class="host-runs"><span class="host-runs-label">Run host</span><div class="host-tablist" role="tablist">${tabs}</div><span class="host-empty-selection" hidden>No selected host can run this test.</span></div>`;
|
|
337
620
|
}
|
|
338
|
-
function renderHostPanels(suite, selectedHost) {
|
|
621
|
+
function renderHostPanels(suite, scenario, selectedHost, results, canRun) {
|
|
339
622
|
return suite.hosts
|
|
340
623
|
.map((host) => {
|
|
341
624
|
const selected = host === selectedHost;
|
|
342
|
-
|
|
625
|
+
const result = results.find((item) => item.suite === suite.name && item.scenario === scenario.name && item.host === host);
|
|
626
|
+
const hostHeading = !canRun && suite.hosts.length > 1 ? `<h3>${escapeHtml(host)}</h3>` : "";
|
|
627
|
+
return `<div class="host-panel" data-host-panel="${escapeHtml(host)}" role="tabpanel"${canRun && !selected ? " hidden" : ""}>${hostHeading}${result?.html ?? '<p class="host-empty">No run yet.</p>'}</div>`;
|
|
343
628
|
})
|
|
344
629
|
.join("");
|
|
345
630
|
}
|
|
346
|
-
function renderScenarioCard(suite, scenario) {
|
|
347
|
-
const selectedHost = defaultScenarioHost(suite, scenario);
|
|
348
|
-
const runDisabled = hostIsSkipped(scenario, selectedHost);
|
|
349
|
-
const
|
|
350
|
-
? scenario.compare
|
|
351
|
-
.map((arm) => `<p class="arm-note">${escapeHtml(arm.label)}: ${escapeHtml(arm.description ?? arm.prompt ?? "")}</p>`)
|
|
352
|
-
.join("")
|
|
353
|
-
: "";
|
|
631
|
+
function renderScenarioCard(suite, scenario, results, canRun, defaultSelectedHosts, selected) {
|
|
632
|
+
const selectedHost = defaultScenarioHost(suite, scenario, defaultSelectedHosts);
|
|
633
|
+
const runDisabled = !selectedHost || hostIsSkipped(scenario, selectedHost);
|
|
634
|
+
const hasResult = results.some((item) => item.suite === suite.name && item.scenario === scenario.name);
|
|
354
635
|
const key = escapeHtml(`${suite.name}::${scenario.name}`);
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
636
|
+
const criteria = renderPassCriteria(scenario);
|
|
637
|
+
const comparisonCriteria = scenario.compare ? renderComparisonCriteria(scenario) : undefined;
|
|
638
|
+
const definition = comparisonCriteria
|
|
639
|
+
? `<section class="test-intent comparison-task">
|
|
640
|
+
<p class="section-label">Comparison task</p>
|
|
641
|
+
<p class="prompt-preview">${escapeHtml(scenario.prompt)}</p>
|
|
642
|
+
</section>
|
|
643
|
+
<section class="comparison-criteria">
|
|
644
|
+
<p class="section-label">Comparison pass criteria</p>
|
|
645
|
+
<p class="comparison-intro">Each arm runs independently. These checks decide whether the comparison passes.</p>
|
|
646
|
+
<div class="criterion-groups">${comparisonCriteria.html}</div>
|
|
647
|
+
</section>
|
|
648
|
+
${renderCompareDefinition(suite, scenario)}`
|
|
649
|
+
: `<section class="test-intent">
|
|
650
|
+
<p class="section-label">Task</p>
|
|
651
|
+
<p class="prompt-preview">${escapeHtml(scenario.prompt)}</p>
|
|
652
|
+
${renderAllowedCommands(scenario)}
|
|
653
|
+
${renderProvidedContext(scenario)}
|
|
654
|
+
${renderSuppliedResources(scenario)}
|
|
655
|
+
</section>
|
|
656
|
+
<details class="criteria-panel" open>
|
|
657
|
+
<summary><span>Pass criteria</span><span class="criteria-summary">${criteria.count ? `${criteria.count} ${criteria.count === 1 ? "check" : "checks"}` : "Completion only"}</span></summary>
|
|
658
|
+
${criteria.html ? `<div class="criterion-groups">${criteria.html}</div>` : '<p class="criteria-empty">The scenario passes when it completes without a runner error.</p>'}
|
|
659
|
+
</details>`;
|
|
660
|
+
return `<article class="scenario-card" data-scenario-card="${key}" data-selected="${selected ? "true" : "false"}"${selected ? "" : " hidden"}>
|
|
661
|
+
<header class="scenario-focus-header">
|
|
662
|
+
<div class="scenario-heading-copy">
|
|
663
|
+
<p class="scenario-path">${escapeHtml(suite.name)} <span aria-hidden="true">/</span> test</p>
|
|
664
|
+
${suite.description ? `<p class="suite-description">${escapeHtml(suite.description)}</p>` : ""}
|
|
665
|
+
<h2>${escapeHtml(scenario.name)}</h2>
|
|
666
|
+
${scenario.description ? `<p class="scenario-lede">${escapeHtml(scenario.description)}</p>` : ""}
|
|
667
|
+
</div>
|
|
668
|
+
<span class="focus-verdict" data-focus-verdict="${key}" data-status="idle"><span aria-hidden="true"></span>Not run</span>
|
|
669
|
+
</header>
|
|
670
|
+
<div class="scenario-definition">
|
|
671
|
+
${definition}
|
|
362
672
|
</div>
|
|
363
673
|
<div class="scenario-toolbar">
|
|
364
674
|
${renderHostChrome(suite, scenario, selectedHost)}
|
|
365
675
|
<div class="scenario-actions">
|
|
366
|
-
|
|
367
|
-
<button class="run-row" data-suite="${escapeHtml(suite.name)}" data-scenario="${escapeHtml(scenario.name)}"${scenario.skip ? " disabled" : ""}>Run selected hosts</button>
|
|
676
|
+
<button class="primary run-toggle run-cell" data-start-label="Start test" data-suite="${escapeHtml(suite.name)}" data-scenario="${escapeHtml(scenario.name)}" data-host="${escapeHtml(selectedHost)}"${scenario.skip || !canRun ? ' disabled data-fixed-disabled="1"' : runDisabled ? " disabled" : ""}>Start test</button>
|
|
368
677
|
</div>
|
|
369
678
|
</div>
|
|
370
|
-
<div class="scenario-live" hidden data-live-row="${key}" data-live-slot="${key}">
|
|
371
|
-
<div class="host-panels">${renderHostPanels(suite, selectedHost)}</div>
|
|
679
|
+
<div class="scenario-live"${hasResult ? "" : " hidden"} data-live-row="${key}" data-live-slot="${key}">
|
|
680
|
+
<div class="host-panels">${renderHostPanels(suite, scenario, selectedHost, results, canRun)}</div>
|
|
372
681
|
</div>
|
|
373
682
|
</article>`;
|
|
374
683
|
}
|
|
375
|
-
function renderSuiteSection(suite) {
|
|
376
|
-
const cards = suite.scenarios
|
|
684
|
+
function renderSuiteSection(suite, results, canRun, defaultSelectedHosts, firstScenarioKey) {
|
|
685
|
+
const cards = suite.scenarios
|
|
686
|
+
.map((scenario) => renderScenarioCard(suite, scenario, results, canRun, defaultSelectedHosts, `${suite.name}::${scenario.name}` === firstScenarioKey))
|
|
687
|
+
.join("");
|
|
377
688
|
return `
|
|
378
689
|
<section class="suite" data-suite="${escapeHtml(suite.name)}">
|
|
379
|
-
<header class="suite-header">
|
|
380
|
-
<div class="suite-title">
|
|
381
|
-
<h2>${escapeHtml(suite.name)}</h2>
|
|
382
|
-
${suite.description ? `<p class="muted">${escapeHtml(suite.description)}</p>` : ""}
|
|
383
|
-
</div>
|
|
384
|
-
<button class="run-suite" data-suite="${escapeHtml(suite.name)}">Run suite</button>
|
|
385
|
-
</header>
|
|
386
690
|
<div class="scenario-list">${cards}</div>
|
|
387
691
|
</section>`;
|
|
388
692
|
}
|
|
693
|
+
function renderTestNavigator(catalog, firstScenarioKey, canRun) {
|
|
694
|
+
const groups = catalog.suites
|
|
695
|
+
.map((suite) => {
|
|
696
|
+
const tests = suite.scenarios
|
|
697
|
+
.map((scenario) => {
|
|
698
|
+
const key = `${suite.name}::${scenario.name}`;
|
|
699
|
+
const selected = key === firstScenarioKey;
|
|
700
|
+
const status = scenario.skip ? "skipped" : "idle";
|
|
701
|
+
return `<button type="button" class="test-nav-item" data-select-scenario="${escapeHtml(key)}" data-status="${status}" aria-current="${selected ? "true" : "false"}">
|
|
702
|
+
<span class="test-nav-status" aria-hidden="true"></span>
|
|
703
|
+
<span class="test-nav-copy"><strong>${escapeHtml(scenario.name)}</strong>${scenario.description ? `<small>${escapeHtml(scenario.description)}</small>` : ""}</span>
|
|
704
|
+
</button>`;
|
|
705
|
+
})
|
|
706
|
+
.join("");
|
|
707
|
+
return `<section class="test-nav-suite" data-nav-suite="${escapeHtml(suite.name)}">
|
|
708
|
+
<header><div><h2>${escapeHtml(suite.name)}</h2>${suite.description ? `<p>${escapeHtml(suite.description)}</p>` : ""}</div><button class="run-toggle run-suite" data-start-label="Start suite" data-suite="${escapeHtml(suite.name)}"${canRun ? "" : ' disabled data-fixed-disabled="1"'}>Start suite</button></header>
|
|
709
|
+
<div class="test-nav-items">${tests}</div>
|
|
710
|
+
</section>`;
|
|
711
|
+
})
|
|
712
|
+
.join("");
|
|
713
|
+
const options = catalog.suites
|
|
714
|
+
.flatMap((suite) => suite.scenarios.map((scenario) => {
|
|
715
|
+
const key = `${suite.name}::${scenario.name}`;
|
|
716
|
+
return `<option value="${escapeHtml(key)}"${key === firstScenarioKey ? " selected" : ""}>${escapeHtml(suite.name)} / ${escapeHtml(scenario.name)}</option>`;
|
|
717
|
+
}))
|
|
718
|
+
.join("");
|
|
719
|
+
return `<aside class="test-navigator" aria-label="Tests">
|
|
720
|
+
<div class="test-navigator-heading"><div><p class="section-label">Test catalog</p><h2>${catalog.suites.reduce((sum, suite) => sum + suite.scenarios.length, 0)} tests</h2></div></div>
|
|
721
|
+
<label class="test-picker-label" for="test-picker">Selected test</label>
|
|
722
|
+
<select class="test-picker" id="test-picker">${options}</select>
|
|
723
|
+
<nav class="test-tree" aria-label="Test catalog">${groups}</nav>
|
|
724
|
+
</aside>`;
|
|
725
|
+
}
|
|
389
726
|
function clientScript() {
|
|
390
727
|
return `
|
|
391
728
|
(function () {
|
|
392
|
-
var
|
|
729
|
+
var bootstrap = JSON.parse(document.getElementById("bootstrap-data").textContent);
|
|
730
|
+
var catalog = bootstrap.catalog;
|
|
393
731
|
var source = null;
|
|
394
732
|
var runId = null;
|
|
733
|
+
var selectedRunId = bootstrap.selectedRunId || null;
|
|
395
734
|
var cancelRequested = false;
|
|
735
|
+
var activeRunRequest = null;
|
|
736
|
+
var viewedRunHosts = null;
|
|
396
737
|
var progress = { total: 0, completed: 0, passed: 0, failed: 0, skipped: 0, seen: {}, experiments: {} };
|
|
738
|
+
var selectedScenarioKey = document.querySelector("[data-scenario-card][data-selected='true']")?.getAttribute("data-scenario-card") || null;
|
|
739
|
+
|
|
740
|
+
function selectScenario(key) {
|
|
741
|
+
if (!key) return;
|
|
742
|
+
selectedScenarioKey = key;
|
|
743
|
+
document.querySelectorAll("[data-scenario-card]").forEach(function (card) {
|
|
744
|
+
var selected = card.getAttribute("data-scenario-card") === key;
|
|
745
|
+
card.dataset.selected = selected ? "true" : "false";
|
|
746
|
+
card.hidden = !selected;
|
|
747
|
+
});
|
|
748
|
+
document.querySelectorAll("[data-select-scenario]").forEach(function (item) {
|
|
749
|
+
item.setAttribute("aria-current", item.getAttribute("data-select-scenario") === key ? "true" : "false");
|
|
750
|
+
});
|
|
751
|
+
var picker = document.getElementById("test-picker");
|
|
752
|
+
if (picker) picker.value = key;
|
|
753
|
+
}
|
|
754
|
+
|
|
755
|
+
function syncScenarioStatus(suite, scenario) {
|
|
756
|
+
var key = suite + "::" + scenario;
|
|
757
|
+
var card = document.querySelector('[data-scenario-card="' + key + '"]');
|
|
758
|
+
if (!card) return;
|
|
759
|
+
var labels = Array.prototype.map.call(card.querySelectorAll(".cell-status"), function (node) { return node.textContent; });
|
|
760
|
+
var status = labels.indexOf("failed") !== -1 ? "failed"
|
|
761
|
+
: labels.some(function (label) { return label === "running" || label === "starting" || label === "cancelling"; }) ? "running"
|
|
762
|
+
: labels.indexOf("passed") !== -1 ? "passed"
|
|
763
|
+
: labels.length > 0 && labels.every(function (label) { return label === "skip" || label === "skipped" || label === "cancelled"; }) ? "skipped"
|
|
764
|
+
: "idle";
|
|
765
|
+
document.querySelectorAll('[data-select-scenario="' + key + '"]').forEach(function (item) { item.dataset.status = status; });
|
|
766
|
+
var verdict = document.querySelector('[data-focus-verdict="' + key + '"]');
|
|
767
|
+
if (verdict) {
|
|
768
|
+
verdict.dataset.status = status;
|
|
769
|
+
verdict.lastChild.textContent = status === "idle" ? "Not run" : status.charAt(0).toUpperCase() + status.slice(1);
|
|
770
|
+
}
|
|
771
|
+
}
|
|
397
772
|
|
|
398
773
|
function selectedHosts() {
|
|
399
774
|
return Array.prototype.map.call(document.querySelectorAll("[data-host-toggle]:checked"), function (box) {
|
|
@@ -401,6 +776,80 @@ function clientScript() {
|
|
|
401
776
|
});
|
|
402
777
|
}
|
|
403
778
|
|
|
779
|
+
function updateHostSelectionControls() {
|
|
780
|
+
var hasSelectedHost = selectedHosts().length > 0;
|
|
781
|
+
syncScenarioHostChoices();
|
|
782
|
+
document.querySelectorAll(".run-toggle").forEach(function (button) {
|
|
783
|
+
if (button.dataset.runAction !== "stop") {
|
|
784
|
+
var hasTarget = button.classList.contains("run-cell")
|
|
785
|
+
? !!button.getAttribute("data-host")
|
|
786
|
+
: hasSelectedHost;
|
|
787
|
+
button.disabled = !hasTarget || button.dataset.fixedDisabled === "1";
|
|
788
|
+
}
|
|
789
|
+
});
|
|
790
|
+
}
|
|
791
|
+
|
|
792
|
+
function syncScenarioHostChoices() {
|
|
793
|
+
var hosts = Array.isArray(viewedRunHosts) ? viewedRunHosts : selectedHosts();
|
|
794
|
+
var isFiltered = hosts.length > 0;
|
|
795
|
+
document.querySelectorAll("[data-scenario-card]").forEach(function (card) {
|
|
796
|
+
var tabs = Array.prototype.slice.call(card.querySelectorAll(".host-tab"));
|
|
797
|
+
if (tabs.length === 0) return;
|
|
798
|
+
tabs.forEach(function (tab) {
|
|
799
|
+
var eligible = !tab.disabled && (!isFiltered || hosts.indexOf(tab.getAttribute("data-host")) !== -1);
|
|
800
|
+
tab.hidden = isFiltered && !eligible;
|
|
801
|
+
});
|
|
802
|
+
var current = tabs.find(function (tab) { return tab.getAttribute("aria-selected") === "true" && !tab.hidden && !tab.disabled; });
|
|
803
|
+
var available = tabs.find(function (tab) { return !tab.hidden && !tab.disabled; });
|
|
804
|
+
if (!current && isFiltered && available) {
|
|
805
|
+
selectHostTab(card, available.getAttribute("data-host"));
|
|
806
|
+
current = available;
|
|
807
|
+
}
|
|
808
|
+
var empty = card.querySelector(".host-empty-selection");
|
|
809
|
+
if (empty) empty.hidden = !!available;
|
|
810
|
+
if (!available) {
|
|
811
|
+
tabs.forEach(function (tab) { tab.setAttribute("aria-selected", "false"); });
|
|
812
|
+
card.querySelectorAll("[data-host-panel]").forEach(function (panel) { panel.hidden = true; });
|
|
813
|
+
var run = card.querySelector("button.run-cell");
|
|
814
|
+
if (run) {
|
|
815
|
+
run.setAttribute("data-host", "");
|
|
816
|
+
run.disabled = true;
|
|
817
|
+
}
|
|
818
|
+
}
|
|
819
|
+
});
|
|
820
|
+
}
|
|
821
|
+
|
|
822
|
+
function scopedRunControl(request) {
|
|
823
|
+
if (!request) return null;
|
|
824
|
+
if (request.scenario) {
|
|
825
|
+
return document.querySelector('.run-cell[data-suite="' + request.suite + '"][data-scenario="' + request.scenario + '"]');
|
|
826
|
+
}
|
|
827
|
+
if (request.suite) return document.querySelector('.run-suite[data-suite="' + request.suite + '"]');
|
|
828
|
+
return null;
|
|
829
|
+
}
|
|
830
|
+
|
|
831
|
+
function setRunControls(status, request) {
|
|
832
|
+
activeRunRequest = status === "idle" ? null : request || activeRunRequest;
|
|
833
|
+
document.querySelectorAll(".run-toggle").forEach(function (button) {
|
|
834
|
+
button.textContent = button.dataset.startLabel || "Start run";
|
|
835
|
+
button.dataset.runAction = "start";
|
|
836
|
+
button.classList.remove("danger");
|
|
837
|
+
});
|
|
838
|
+
updateHostSelectionControls();
|
|
839
|
+
if (status === "idle") return;
|
|
840
|
+
document.querySelectorAll(".run-toggle").forEach(function (button) { button.disabled = true; });
|
|
841
|
+
var controls = [document.getElementById("run-selection"), scopedRunControl(activeRunRequest)];
|
|
842
|
+
controls.forEach(function (button) {
|
|
843
|
+
if (!button) return;
|
|
844
|
+
button.dataset.runAction = "stop";
|
|
845
|
+
button.textContent = status === "cancelling"
|
|
846
|
+
? "Stopping…"
|
|
847
|
+
: button.id === "run-selection" ? "Stop run" : (button.classList.contains("run-suite") ? "Stop suite" : "Stop test");
|
|
848
|
+
button.classList.add("danger");
|
|
849
|
+
button.disabled = status === "cancelling";
|
|
850
|
+
});
|
|
851
|
+
}
|
|
852
|
+
|
|
404
853
|
function parallelHosts() {
|
|
405
854
|
return document.getElementById("parallel-hosts").checked;
|
|
406
855
|
}
|
|
@@ -410,7 +859,7 @@ function clientScript() {
|
|
|
410
859
|
}
|
|
411
860
|
|
|
412
861
|
function runnableTargets(body) {
|
|
413
|
-
var requestedHosts =
|
|
862
|
+
var requestedHosts = Array.isArray(body.hosts) ? body.hosts : catalog.defaultSelectedHosts;
|
|
414
863
|
var total = 0;
|
|
415
864
|
catalog.suites.forEach(function (suite) {
|
|
416
865
|
if (body.suite && suite.name !== body.suite) {
|
|
@@ -702,11 +1151,130 @@ function clientScript() {
|
|
|
702
1151
|
markRunningIdle();
|
|
703
1152
|
}
|
|
704
1153
|
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
1154
|
+
function resetViewerCards() {
|
|
1155
|
+
closeAllChats();
|
|
1156
|
+
document.querySelectorAll("[data-host-panel]").forEach(function (panel) {
|
|
1157
|
+
panel.innerHTML = '<p class="host-empty">No run yet.</p>';
|
|
1158
|
+
});
|
|
1159
|
+
document.querySelectorAll(".scenario-live").forEach(function (live) { live.hidden = true; });
|
|
1160
|
+
document.querySelectorAll(".cell-status").forEach(function (status) {
|
|
1161
|
+
if (status.textContent !== "skip") {
|
|
1162
|
+
status.textContent = "idle";
|
|
1163
|
+
status.className = "cell-status muted";
|
|
1164
|
+
}
|
|
1165
|
+
});
|
|
1166
|
+
catalog.suites.forEach(function (suite) {
|
|
1167
|
+
suite.scenarios.forEach(function (scenario) { syncScenarioStatus(suite.name, scenario.name); });
|
|
1168
|
+
});
|
|
1169
|
+
}
|
|
1170
|
+
|
|
1171
|
+
function runLabel(run) {
|
|
1172
|
+
var scope = run.request && (run.request.scenario || run.request.suite) || "all tests";
|
|
1173
|
+
var time = new Date(run.startedAt).toLocaleTimeString([], { hour: "2-digit", minute: "2-digit" });
|
|
1174
|
+
return time + " · " + scope + " · " + run.status;
|
|
1175
|
+
}
|
|
1176
|
+
|
|
1177
|
+
function renderRunHistory(runs) {
|
|
1178
|
+
bootstrap.runs = runs;
|
|
1179
|
+
var select = document.getElementById("run-history");
|
|
1180
|
+
select.replaceChildren();
|
|
1181
|
+
var idle = document.createElement("option");
|
|
1182
|
+
idle.value = "";
|
|
1183
|
+
idle.textContent = "Tests · no run selected";
|
|
1184
|
+
select.appendChild(idle);
|
|
1185
|
+
runs.forEach(function (run) {
|
|
1186
|
+
var option = document.createElement("option");
|
|
1187
|
+
option.value = run.id;
|
|
1188
|
+
option.textContent = runLabel(run);
|
|
1189
|
+
select.appendChild(option);
|
|
1190
|
+
});
|
|
1191
|
+
select.value = selectedRunId || "";
|
|
1192
|
+
}
|
|
1193
|
+
|
|
1194
|
+
function completedRunBanner(run) {
|
|
1195
|
+
if (run.status === "cancelled") return "Run cancelled.";
|
|
1196
|
+
if (run.status !== "completed") return "Run " + run.id + " · " + run.status + ".";
|
|
1197
|
+
var counts = (run.reports || []).reduce(function (total, report) {
|
|
1198
|
+
total.passed += report.passed || 0;
|
|
1199
|
+
total.failed += report.failed || 0;
|
|
1200
|
+
total.skipped += report.skipped || 0;
|
|
1201
|
+
return total;
|
|
1202
|
+
}, { passed: 0, failed: 0, skipped: 0 });
|
|
1203
|
+
return "Run finished. " + counts.passed + " passed. " + counts.failed + " failed. " + counts.skipped + " skipped.";
|
|
1204
|
+
}
|
|
1205
|
+
|
|
1206
|
+
async function refreshRunHistory() {
|
|
1207
|
+
if (!bootstrap.capabilities.canRun) return;
|
|
1208
|
+
var response = await fetch("/api/runs");
|
|
1209
|
+
if (!response.ok) return;
|
|
1210
|
+
renderRunHistory(await response.json());
|
|
1211
|
+
}
|
|
1212
|
+
|
|
1213
|
+
async function selectRun(id) {
|
|
1214
|
+
selectedRunId = id || null;
|
|
1215
|
+
viewedRunHosts = null;
|
|
1216
|
+
if (source) { source.close(); source = null; }
|
|
1217
|
+
runId = null;
|
|
1218
|
+
cancelRequested = false;
|
|
1219
|
+
resetViewerCards();
|
|
1220
|
+
if (!selectedRunId) {
|
|
1221
|
+
setRunBanner("Choose tests to run.");
|
|
1222
|
+
setRunControls("idle");
|
|
1223
|
+
renderRunHistory(bootstrap.runs || []);
|
|
1224
|
+
return;
|
|
1225
|
+
}
|
|
1226
|
+
var response = await fetch("/api/runs/" + selectedRunId);
|
|
1227
|
+
if (!response.ok) return;
|
|
1228
|
+
var payload = await response.json();
|
|
1229
|
+
viewedRunHosts = Array.isArray(payload.run.request && payload.run.request.hosts) ? payload.run.request.hosts : null;
|
|
1230
|
+
syncScenarioHostChoices();
|
|
1231
|
+
(payload.fragments || []).forEach(function (fragment) {
|
|
1232
|
+
var card = document.querySelector('[data-scenario-card="' + fragment.suite + "::" + fragment.scenario + '"]');
|
|
1233
|
+
var panel = card && card.querySelector('[data-host-panel="' + fragment.host + '"]');
|
|
1234
|
+
if (!card || !panel) return;
|
|
1235
|
+
panel.innerHTML = fragment.html;
|
|
1236
|
+
installCopyErrorButtons(panel);
|
|
1237
|
+
var live = card.querySelector(".scenario-live");
|
|
1238
|
+
if (live) live.hidden = false;
|
|
1239
|
+
markCell(fragment, fragment.skipped ? "skipped" : fragment.passed ? "passed" : "failed", fragment.skipped ? "status-skipped" : fragment.passed ? "status-passed" : "status-failed");
|
|
1240
|
+
});
|
|
1241
|
+
var preferred = (payload.fragments || []).find(function (fragment) { return !fragment.skipped && !fragment.passed; }) || (payload.fragments || [])[0];
|
|
1242
|
+
if (preferred) selectScenario(preferred.suite + "::" + preferred.scenario, false);
|
|
1243
|
+
if (payload.run.status === "cancelled") {
|
|
1244
|
+
markRunTargets(payload.run.request || {}, "cancelled", "status-skipped");
|
|
1245
|
+
}
|
|
1246
|
+
setRunBanner(completedRunBanner(payload.run));
|
|
1247
|
+
renderRunHistory((bootstrap.runs || []).map(function (item) { return item.id === payload.run.id ? payload.run : item; }));
|
|
1248
|
+
if (payload.run.status === "running" || payload.run.status === "cancelling") {
|
|
1249
|
+
runId = payload.run.id;
|
|
1250
|
+
setRunControls(payload.run.status === "cancelling" ? "cancelling" : "running", payload.run.request);
|
|
1251
|
+
source = new EventSource("/api/runs/" + runId + "/events");
|
|
1252
|
+
source.onmessage = function (message) { handleEvent(JSON.parse(message.data)); };
|
|
1253
|
+
} else {
|
|
1254
|
+
setRunControls("idle");
|
|
1255
|
+
}
|
|
1256
|
+
}
|
|
1257
|
+
|
|
1258
|
+
function requestRunStop() {
|
|
1259
|
+
if (!runId || cancelRequested) return;
|
|
1260
|
+
cancelRequested = true;
|
|
1261
|
+
setRunBanner("Stopping the run.");
|
|
1262
|
+
setRunControls("cancelling", activeRunRequest);
|
|
1263
|
+
cancelOpenChats();
|
|
1264
|
+
fetch("/api/runs/" + runId + "/cancel", { method: "POST" }).then(function (response) {
|
|
1265
|
+
if (!response.ok && cancelRequested) {
|
|
1266
|
+
setRunBanner("Stop failed.");
|
|
1267
|
+
cancelRequested = false;
|
|
1268
|
+
setRunControls("running", activeRunRequest);
|
|
1269
|
+
}
|
|
1270
|
+
}).catch(function () {
|
|
1271
|
+
if (cancelRequested) {
|
|
1272
|
+
setRunBanner("Stop failed.");
|
|
1273
|
+
cancelRequested = false;
|
|
1274
|
+
setRunControls("running", activeRunRequest);
|
|
1275
|
+
}
|
|
1276
|
+
});
|
|
1277
|
+
}
|
|
710
1278
|
|
|
711
1279
|
function showCancelledNote(article, text) {
|
|
712
1280
|
if (!article) {
|
|
@@ -1238,7 +1806,7 @@ function clientScript() {
|
|
|
1238
1806
|
return { passed: false, line: gate.metric + ": one arm did not report a value. Fail." };
|
|
1239
1807
|
}
|
|
1240
1808
|
var passed = winnerValue < loserValue;
|
|
1241
|
-
return { passed: passed, line: winner.label + " must
|
|
1809
|
+
return { passed: passed, line: winner.label + " must use fewer " + gate.metric + " than " + loser.label + " (" + winnerValue + " vs " + loserValue + "). " + (passed ? "Pass." : "Fail.") };
|
|
1242
1810
|
});
|
|
1243
1811
|
}
|
|
1244
1812
|
|
|
@@ -1272,7 +1840,6 @@ function clientScript() {
|
|
|
1272
1840
|
summarizeMetric(rows, "turns", "Turns"),
|
|
1273
1841
|
summarizeMetric(rows, "tokens", "Tokens"),
|
|
1274
1842
|
summarizeMetric(rows, "tools", "Tools"),
|
|
1275
|
-
summarizeMetric(rows, "durationMs", "Duration"),
|
|
1276
1843
|
];
|
|
1277
1844
|
var gateLines = summarizeGates(rows, scenario.gates);
|
|
1278
1845
|
var existing = wrap.querySelector(".compare-winners");
|
|
@@ -1312,6 +1879,7 @@ function clientScript() {
|
|
|
1312
1879
|
node.setAttribute("data-status", label);
|
|
1313
1880
|
}
|
|
1314
1881
|
});
|
|
1882
|
+
syncScenarioStatus(event.suite, event.scenario);
|
|
1315
1883
|
}
|
|
1316
1884
|
|
|
1317
1885
|
function metricPart(count, one, many) {
|
|
@@ -1330,6 +1898,78 @@ function clientScript() {
|
|
|
1330
1898
|
.join(" · ");
|
|
1331
1899
|
}
|
|
1332
1900
|
|
|
1901
|
+
function failureText(container) {
|
|
1902
|
+
var messages = Array.prototype.map.call(
|
|
1903
|
+
container.querySelectorAll(".failure-message, .cell-result-failures li"),
|
|
1904
|
+
function (node) { return node.textContent.trim(); }
|
|
1905
|
+
).filter(Boolean);
|
|
1906
|
+
var evidence = Array.prototype.map.call(
|
|
1907
|
+
container.querySelectorAll(".failure-evidence"),
|
|
1908
|
+
function (node) { return node.textContent.trim(); }
|
|
1909
|
+
).filter(Boolean);
|
|
1910
|
+
return messages.concat(evidence).join("\\n\\n");
|
|
1911
|
+
}
|
|
1912
|
+
|
|
1913
|
+
function copyText(text) {
|
|
1914
|
+
if (navigator.clipboard && window.isSecureContext) {
|
|
1915
|
+
return navigator.clipboard.writeText(text);
|
|
1916
|
+
}
|
|
1917
|
+
return new Promise(function (resolve, reject) {
|
|
1918
|
+
var input = document.createElement("textarea");
|
|
1919
|
+
input.value = text;
|
|
1920
|
+
input.setAttribute("readonly", "");
|
|
1921
|
+
input.style.position = "fixed";
|
|
1922
|
+
input.style.opacity = "0";
|
|
1923
|
+
document.body.appendChild(input);
|
|
1924
|
+
input.select();
|
|
1925
|
+
var copied = document.execCommand("copy");
|
|
1926
|
+
input.remove();
|
|
1927
|
+
if (copied) resolve();
|
|
1928
|
+
else reject(new Error("Clipboard copy failed"));
|
|
1929
|
+
});
|
|
1930
|
+
}
|
|
1931
|
+
|
|
1932
|
+
function copyErrorButton(container) {
|
|
1933
|
+
var button = document.createElement("button");
|
|
1934
|
+
button.type = "button";
|
|
1935
|
+
button.className = "copy-error";
|
|
1936
|
+
button.textContent = "Copy error";
|
|
1937
|
+
button.addEventListener("click", function () {
|
|
1938
|
+
var text = failureText(container);
|
|
1939
|
+
copyText(text).then(function () {
|
|
1940
|
+
button.textContent = "Copied";
|
|
1941
|
+
window.setTimeout(function () { button.textContent = "Copy error"; }, 1500);
|
|
1942
|
+
}).catch(function () {
|
|
1943
|
+
button.textContent = "Copy failed";
|
|
1944
|
+
window.setTimeout(function () { button.textContent = "Copy error"; }, 1500);
|
|
1945
|
+
});
|
|
1946
|
+
});
|
|
1947
|
+
return button;
|
|
1948
|
+
}
|
|
1949
|
+
|
|
1950
|
+
function installCopyErrorButtons(root) {
|
|
1951
|
+
root.querySelectorAll(".cell-result.status-failed").forEach(function (box) {
|
|
1952
|
+
if (box.querySelector(".copy-error") || !box.querySelector(".cell-result-failures")) return;
|
|
1953
|
+
var verdict = box.querySelector(".cell-result-verdict");
|
|
1954
|
+
if (!verdict) return;
|
|
1955
|
+
var head = document.createElement("div");
|
|
1956
|
+
head.className = "cell-result-head";
|
|
1957
|
+
box.insertBefore(head, verdict);
|
|
1958
|
+
head.appendChild(verdict);
|
|
1959
|
+
head.appendChild(copyErrorButton(box));
|
|
1960
|
+
});
|
|
1961
|
+
root.querySelectorAll("details.scenario.status-failed .diagnostics > section").forEach(function (section) {
|
|
1962
|
+
if (section.querySelector(".copy-error") || !section.querySelector(".failures")) return;
|
|
1963
|
+
var heading = section.querySelector("h3");
|
|
1964
|
+
if (!heading || heading.textContent.trim() !== "What went wrong") return;
|
|
1965
|
+
var head = document.createElement("div");
|
|
1966
|
+
head.className = "failure-section-head";
|
|
1967
|
+
section.insertBefore(head, heading);
|
|
1968
|
+
head.appendChild(heading);
|
|
1969
|
+
head.appendChild(copyErrorButton(section));
|
|
1970
|
+
});
|
|
1971
|
+
}
|
|
1972
|
+
|
|
1333
1973
|
function clearCellResult(event) {
|
|
1334
1974
|
var article = document.getElementById(paneId(event));
|
|
1335
1975
|
var existing = article ? article.querySelector(".cell-result") : null;
|
|
@@ -1381,6 +2021,7 @@ function clientScript() {
|
|
|
1381
2021
|
box.appendChild(list);
|
|
1382
2022
|
}
|
|
1383
2023
|
article.appendChild(box);
|
|
2024
|
+
installCopyErrorButtons(article);
|
|
1384
2025
|
}
|
|
1385
2026
|
|
|
1386
2027
|
function appendJudgeVerdicts(event) {
|
|
@@ -1420,10 +2061,11 @@ function clientScript() {
|
|
|
1420
2061
|
return;
|
|
1421
2062
|
}
|
|
1422
2063
|
setRunBanner("Run " + event.runId + " started.");
|
|
1423
|
-
|
|
2064
|
+
setRunControls("running", activeRunRequest);
|
|
1424
2065
|
return;
|
|
1425
2066
|
}
|
|
1426
2067
|
if (event.type === "run_finished") {
|
|
2068
|
+
var wasCancelled = cancelRequested;
|
|
1427
2069
|
finishProgress(event);
|
|
1428
2070
|
hideAllRunning();
|
|
1429
2071
|
sealAllChats();
|
|
@@ -1435,13 +2077,16 @@ function clientScript() {
|
|
|
1435
2077
|
"Run finished. " + progress.passed + " passed. " + progress.failed + " failed. " + progress.skipped + " skipped."
|
|
1436
2078
|
);
|
|
1437
2079
|
}
|
|
1438
|
-
|
|
2080
|
+
setRunControls("idle");
|
|
1439
2081
|
cancelRequested = false;
|
|
1440
2082
|
runId = null;
|
|
1441
2083
|
if (source) {
|
|
1442
2084
|
source.close();
|
|
1443
2085
|
source = null;
|
|
1444
2086
|
}
|
|
2087
|
+
void refreshRunHistory().then(function () {
|
|
2088
|
+
if (!wasCancelled) return selectRun(event.runId);
|
|
2089
|
+
});
|
|
1445
2090
|
return;
|
|
1446
2091
|
}
|
|
1447
2092
|
if (cancelRequested) {
|
|
@@ -1458,6 +2103,22 @@ function clientScript() {
|
|
|
1458
2103
|
}
|
|
1459
2104
|
return;
|
|
1460
2105
|
}
|
|
2106
|
+
if (event.type === "scenario_result") {
|
|
2107
|
+
fetch("/api/runs/" + (runId || selectedRunId)).then(function (response) {
|
|
2108
|
+
return response.ok ? response.json() : null;
|
|
2109
|
+
}).then(function (payload) {
|
|
2110
|
+
if (!payload || selectedRunId !== payload.run.id) return;
|
|
2111
|
+
var fragment = (payload.fragments || []).find(function (item) {
|
|
2112
|
+
return item.suite === event.suite && item.scenario === event.scenario && item.host === event.host;
|
|
2113
|
+
});
|
|
2114
|
+
if (!fragment) return;
|
|
2115
|
+
var card = scenarioCard(event);
|
|
2116
|
+
var panel = card && card.querySelector('[data-host-panel="' + event.host + '"]');
|
|
2117
|
+
if (panel) panel.innerHTML = fragment.html;
|
|
2118
|
+
if (panel) installCopyErrorButtons(panel);
|
|
2119
|
+
}).catch(function () {});
|
|
2120
|
+
return;
|
|
2121
|
+
}
|
|
1461
2122
|
if (event.type === "cell_started") {
|
|
1462
2123
|
markCell(event, "running", "status-skipped");
|
|
1463
2124
|
startChat(event);
|
|
@@ -1568,7 +2229,7 @@ function clientScript() {
|
|
|
1568
2229
|
}
|
|
1569
2230
|
|
|
1570
2231
|
function markRunTargets(body, label, statusClass) {
|
|
1571
|
-
var hosts =
|
|
2232
|
+
var hosts = Array.isArray(body.hosts) ? body.hosts : catalog.defaultSelectedHosts;
|
|
1572
2233
|
catalog.suites.forEach(function (suite) {
|
|
1573
2234
|
if (body.suite && suite.name !== body.suite) {
|
|
1574
2235
|
return;
|
|
@@ -1591,6 +2252,11 @@ function clientScript() {
|
|
|
1591
2252
|
}
|
|
1592
2253
|
|
|
1593
2254
|
async function startRun(body) {
|
|
2255
|
+
if (!bootstrap.capabilities.canRun) return;
|
|
2256
|
+
if (Array.isArray(body.hosts) && body.hosts.length === 0) {
|
|
2257
|
+
setRunBanner("Select at least one host.");
|
|
2258
|
+
return;
|
|
2259
|
+
}
|
|
1594
2260
|
if (runId) {
|
|
1595
2261
|
setRunBanner(
|
|
1596
2262
|
cancelRequested
|
|
@@ -1599,6 +2265,11 @@ function clientScript() {
|
|
|
1599
2265
|
);
|
|
1600
2266
|
return;
|
|
1601
2267
|
}
|
|
2268
|
+
resetViewerCards();
|
|
2269
|
+
activeRunRequest = body;
|
|
2270
|
+
viewedRunHosts = Array.isArray(body.hosts) ? body.hosts : null;
|
|
2271
|
+
syncScenarioHostChoices();
|
|
2272
|
+
setRunControls("running", body);
|
|
1602
2273
|
resetProgress(body);
|
|
1603
2274
|
markRunTargets(body, "starting", "status-skipped");
|
|
1604
2275
|
var firstCard = document.querySelector(
|
|
@@ -1618,6 +2289,7 @@ function clientScript() {
|
|
|
1618
2289
|
delete card.dataset.followHost;
|
|
1619
2290
|
});
|
|
1620
2291
|
if (firstCard) {
|
|
2292
|
+
selectScenario(firstCard.getAttribute("data-scenario-card"), false);
|
|
1621
2293
|
var live = firstCard.querySelector(".scenario-live");
|
|
1622
2294
|
if (live) {
|
|
1623
2295
|
live.hidden = false;
|
|
@@ -1642,12 +2314,15 @@ function clientScript() {
|
|
|
1642
2314
|
progress = { total: 0, completed: 0, passed: 0, failed: 0, skipped: 0, seen: {} };
|
|
1643
2315
|
renderProgress();
|
|
1644
2316
|
markRunTargets(body, "idle", "muted");
|
|
2317
|
+
setRunControls("idle");
|
|
1645
2318
|
return;
|
|
1646
2319
|
}
|
|
1647
2320
|
runId = payload.runId;
|
|
2321
|
+
selectedRunId = runId;
|
|
2322
|
+
await refreshRunHistory();
|
|
1648
2323
|
cancelRequested = false;
|
|
1649
2324
|
setRunBanner("Run " + runId + " started.");
|
|
1650
|
-
|
|
2325
|
+
setRunControls("running", body);
|
|
1651
2326
|
if (source) {
|
|
1652
2327
|
source.close();
|
|
1653
2328
|
}
|
|
@@ -1665,39 +2340,29 @@ function clientScript() {
|
|
|
1665
2340
|
} catch (error) {
|
|
1666
2341
|
setRunBanner(error instanceof Error ? error.message : String(error));
|
|
1667
2342
|
markRunTargets(body, "idle", "muted");
|
|
2343
|
+
setRunControls("idle");
|
|
1668
2344
|
}
|
|
1669
2345
|
}
|
|
1670
2346
|
|
|
1671
|
-
document.getElementById("run-selection").addEventListener("click", function () {
|
|
2347
|
+
document.getElementById("run-selection").addEventListener("click", function (event) {
|
|
2348
|
+
if (event.currentTarget.dataset.runAction === "stop") return requestRunStop();
|
|
1672
2349
|
startRun({ hosts: selectedHosts(), parallelHosts: parallelHosts() });
|
|
1673
2350
|
});
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
setCancelButton("Cancelling", true);
|
|
1681
|
-
cancelOpenChats();
|
|
1682
|
-
fetch("/api/runs/" + runId + "/cancel", { method: "POST" }).then(function (response) {
|
|
1683
|
-
if (!response.ok && cancelRequested) {
|
|
1684
|
-
setRunBanner("Cancel failed.");
|
|
1685
|
-
setCancelButton("Cancel run", false);
|
|
1686
|
-
cancelRequested = false;
|
|
1687
|
-
}
|
|
1688
|
-
}).catch(function () {
|
|
1689
|
-
if (cancelRequested) {
|
|
1690
|
-
setRunBanner("Cancel failed.");
|
|
1691
|
-
setCancelButton("Cancel run", false);
|
|
1692
|
-
cancelRequested = false;
|
|
1693
|
-
}
|
|
1694
|
-
});
|
|
1695
|
-
});
|
|
2351
|
+
document.querySelectorAll("[data-host-toggle]").forEach(function (toggle) {
|
|
2352
|
+
toggle.addEventListener("change", updateHostSelectionControls);
|
|
2353
|
+
});
|
|
2354
|
+
document.getElementById("run-history").addEventListener("change", function (event) {
|
|
2355
|
+
void selectRun(event.target.value);
|
|
2356
|
+
});
|
|
1696
2357
|
document.body.addEventListener("click", function (event) {
|
|
1697
2358
|
var button = event.target.closest("button");
|
|
1698
2359
|
if (!button) {
|
|
1699
2360
|
return;
|
|
1700
2361
|
}
|
|
2362
|
+
if (button.classList.contains("test-nav-item")) {
|
|
2363
|
+
selectScenario(button.getAttribute("data-select-scenario"), false);
|
|
2364
|
+
return;
|
|
2365
|
+
}
|
|
1701
2366
|
if (button.classList.contains("host-tab")) {
|
|
1702
2367
|
var card = button.closest("[data-scenario-card]");
|
|
1703
2368
|
if (card) {
|
|
@@ -1706,6 +2371,7 @@ function clientScript() {
|
|
|
1706
2371
|
}
|
|
1707
2372
|
}
|
|
1708
2373
|
if (button.classList.contains("run-cell")) {
|
|
2374
|
+
if (button.dataset.runAction === "stop") return requestRunStop();
|
|
1709
2375
|
startRun({
|
|
1710
2376
|
suite: button.getAttribute("data-suite"),
|
|
1711
2377
|
scenario: button.getAttribute("data-scenario"),
|
|
@@ -1713,67 +2379,104 @@ function clientScript() {
|
|
|
1713
2379
|
parallelHosts: false,
|
|
1714
2380
|
});
|
|
1715
2381
|
}
|
|
1716
|
-
if (button.classList.contains("run-row")) {
|
|
1717
|
-
startRun({
|
|
1718
|
-
suite: button.getAttribute("data-suite"),
|
|
1719
|
-
scenario: button.getAttribute("data-scenario"),
|
|
1720
|
-
hosts: selectedHosts(),
|
|
1721
|
-
parallelHosts: parallelHosts(),
|
|
1722
|
-
});
|
|
1723
|
-
}
|
|
1724
2382
|
if (button.classList.contains("run-suite")) {
|
|
2383
|
+
if (button.dataset.runAction === "stop") return requestRunStop();
|
|
1725
2384
|
startRun({
|
|
1726
2385
|
suite: button.getAttribute("data-suite"),
|
|
1727
2386
|
hosts: selectedHosts(),
|
|
1728
2387
|
parallelHosts: parallelHosts(),
|
|
1729
2388
|
});
|
|
1730
2389
|
}
|
|
2390
|
+
});
|
|
2391
|
+
document.getElementById("test-picker")?.addEventListener("change", function (event) {
|
|
2392
|
+
selectScenario(event.target.value, false);
|
|
2393
|
+
});
|
|
2394
|
+
|
|
2395
|
+
renderRunHistory(bootstrap.runs || []);
|
|
2396
|
+
installCopyErrorButtons(document);
|
|
2397
|
+
selectScenario(selectedScenarioKey, false);
|
|
2398
|
+
updateHostSelectionControls();
|
|
2399
|
+
if (selectedRunId && bootstrap.capabilities.canRun) void selectRun(selectedRunId);
|
|
2400
|
+
if (bootstrap.capabilities.canRun) setInterval(function () { void refreshRunHistory(); }, 1000);
|
|
2401
|
+
})();
|
|
2402
|
+
`;
|
|
2403
|
+
}
|
|
2404
|
+
function staticNavigationScript() {
|
|
2405
|
+
return `
|
|
2406
|
+
(function () {
|
|
2407
|
+
function selectScenario(key) {
|
|
2408
|
+
document.querySelectorAll("[data-scenario-card]").forEach(function (card) {
|
|
2409
|
+
var selected = card.getAttribute("data-scenario-card") === key;
|
|
2410
|
+
card.dataset.selected = selected ? "true" : "false";
|
|
2411
|
+
card.hidden = !selected;
|
|
2412
|
+
});
|
|
2413
|
+
document.querySelectorAll("[data-select-scenario]").forEach(function (item) {
|
|
2414
|
+
item.setAttribute("aria-current", item.getAttribute("data-select-scenario") === key ? "true" : "false");
|
|
2415
|
+
});
|
|
2416
|
+
var picker = document.getElementById("test-picker");
|
|
2417
|
+
if (picker) picker.value = key;
|
|
2418
|
+
}
|
|
2419
|
+
document.body.addEventListener("click", function (event) {
|
|
2420
|
+
var item = event.target.closest("[data-select-scenario]");
|
|
2421
|
+
if (item) selectScenario(item.getAttribute("data-select-scenario"));
|
|
2422
|
+
});
|
|
2423
|
+
document.getElementById("test-picker")?.addEventListener("change", function (event) {
|
|
2424
|
+
selectScenario(event.target.value);
|
|
1731
2425
|
});
|
|
1732
2426
|
})();
|
|
1733
2427
|
`;
|
|
1734
2428
|
}
|
|
1735
|
-
export function renderViewerPage(
|
|
2429
|
+
export function renderViewerPage(input, options = {}) {
|
|
2430
|
+
const bootstrap = "catalog" in input ? input : { catalog: input, runs: [], capabilities: { canRun: true } };
|
|
2431
|
+
const catalog = bootstrap.catalog;
|
|
2432
|
+
const canRun = bootstrap.capabilities.canRun;
|
|
2433
|
+
const firstScenarioKey = catalog.suites[0]?.scenarios[0]
|
|
2434
|
+
? `${catalog.suites[0].name}::${catalog.suites[0].scenarios[0].name}`
|
|
2435
|
+
: "";
|
|
1736
2436
|
const hostToggles = Array.from(new Set(catalog.suites.flatMap((suite) => suite.hosts)))
|
|
1737
2437
|
.map((host) => {
|
|
1738
2438
|
const checked = catalog.defaultSelectedHosts.includes(host) ? " checked" : "";
|
|
1739
2439
|
return `<label><input type="checkbox" data-host-toggle value="${escapeHtml(host)}"${checked}>${escapeHtml(host)}</label>`;
|
|
1740
2440
|
})
|
|
1741
2441
|
.join("");
|
|
1742
|
-
const suites = catalog.suites
|
|
2442
|
+
const suites = catalog.suites
|
|
2443
|
+
.map((suite) => renderSuiteSection(suite, options.results ?? [], canRun, catalog.defaultSelectedHosts, firstScenarioKey))
|
|
2444
|
+
.join("\n");
|
|
2445
|
+
const navigator = renderTestNavigator(catalog, firstScenarioKey, canRun);
|
|
2446
|
+
const historyOptions = [
|
|
2447
|
+
`<option value="">Tests · no run selected</option>`,
|
|
2448
|
+
...bootstrap.runs.map((run) => {
|
|
2449
|
+
const selected = run.id === bootstrap.selectedRunId ? " selected" : "";
|
|
2450
|
+
return `<option value="${escapeHtml(run.id)}"${selected}>${escapeHtml(run.id)} · ${escapeHtml(run.status)}</option>`;
|
|
2451
|
+
}),
|
|
2452
|
+
].join("");
|
|
1743
2453
|
return `<!DOCTYPE html>
|
|
1744
2454
|
<html lang="en">
|
|
1745
2455
|
<head>
|
|
1746
2456
|
<meta charset="utf-8">
|
|
1747
2457
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
1748
|
-
<title>agent-test viewer</title>
|
|
2458
|
+
<title>agent-test ${canRun ? "viewer" : "report"}</title>
|
|
1749
2459
|
<style>
|
|
1750
|
-
${
|
|
2460
|
+
${options.baseCss ?? ""}
|
|
1751
2461
|
${viewerCss()}
|
|
1752
2462
|
</style>
|
|
1753
2463
|
</head>
|
|
1754
2464
|
<body>
|
|
1755
|
-
<main>
|
|
1756
|
-
<header class="
|
|
1757
|
-
<p class="brand">agent-test</p>
|
|
1758
|
-
|
|
1759
|
-
|
|
1760
|
-
|
|
2465
|
+
<main class="viewer-shell">
|
|
2466
|
+
<header class="viewer-topbar">
|
|
2467
|
+
<div class="viewer-identity"><p class="brand">agent-test</p><h1>${canRun ? "Test viewer" : "Run report"}</h1></div>
|
|
2468
|
+
<div class="run-context">
|
|
2469
|
+
<label class="toolbar-label" for="run-history">Viewing run</label>
|
|
2470
|
+
<select class="run-history" id="run-history" aria-label="Run history"${canRun ? "" : " disabled"}>${historyOptions}</select>
|
|
2471
|
+
</div>
|
|
2472
|
+
<p class="run-banner${canRun ? "" : " lede"}" id="run-banner" aria-live="polite">${canRun ? "Choose a test to inspect or run." : escapeHtml(options.headerLede ?? "Read-only report.")}</p>
|
|
1761
2473
|
</header>
|
|
1762
|
-
|
|
1763
|
-
<div class="toolbar-block">
|
|
1764
|
-
|
|
1765
|
-
|
|
1766
|
-
|
|
1767
|
-
|
|
1768
|
-
<p class="toolbar-label">Run order</p>
|
|
1769
|
-
<label><input type="checkbox" id="parallel-hosts"> Run hosts together</label>
|
|
1770
|
-
</div>
|
|
1771
|
-
<div class="toolbar-actions">
|
|
1772
|
-
<button class="primary" id="run-selection">Run selection</button>
|
|
1773
|
-
<button class="danger" id="cancel-run" disabled>Cancel run</button>
|
|
1774
|
-
</div>
|
|
1775
|
-
<p class="run-banner" id="run-banner"></p>
|
|
1776
|
-
</section>
|
|
2474
|
+
<div class="viewer-command-row">
|
|
2475
|
+
<div class="toolbar-block"><p class="toolbar-label">Hosts</p><div class="host-toggles">${hostToggles}</div></div>
|
|
2476
|
+
<label class="parallel-control"><input type="checkbox" id="parallel-hosts" checked> Run hosts together</label>
|
|
2477
|
+
<button class="primary run-toggle" id="run-selection" data-start-label="Start run"${canRun ? "" : ' disabled data-fixed-disabled="1"'}>Start run</button>
|
|
2478
|
+
</div>
|
|
2479
|
+
${options.overviewHtml ?? ""}
|
|
1777
2480
|
<section class="run-progress" id="run-progress" aria-live="polite" hidden>
|
|
1778
2481
|
<div class="run-progress-head">
|
|
1779
2482
|
<p class="run-progress-title">0 of 0 tests finished</p>
|
|
@@ -1788,11 +2491,15 @@ ${viewerCss()}
|
|
|
1788
2491
|
<span class="progress-remaining">Remaining <strong>0</strong></span>
|
|
1789
2492
|
</div>
|
|
1790
2493
|
</section>
|
|
1791
|
-
|
|
2494
|
+
<div class="viewer-workspace">
|
|
2495
|
+
${navigator}
|
|
2496
|
+
<section class="test-stage" aria-label="Selected test">${suites}</section>
|
|
2497
|
+
</div>
|
|
1792
2498
|
</main>
|
|
1793
|
-
|
|
1794
|
-
<script>
|
|
1795
|
-
|
|
2499
|
+
<script type="application/json" id="catalog-data">${embedJson(catalog)}</script>
|
|
2500
|
+
<script type="application/json" id="bootstrap-data">${embedJson(bootstrap)}</script>
|
|
2501
|
+
<script type="text/javascript">
|
|
2502
|
+
${canRun ? clientScript() : staticNavigationScript()}
|
|
1796
2503
|
</script>
|
|
1797
2504
|
</body>
|
|
1798
2505
|
</html>
|