@prismatic-io/lux 0.0.1 → 0.0.2-preview.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/answerers/persona/index.d.ts +2 -1
- package/lib/answerers/persona/index.d.ts.map +1 -1
- package/lib/answerers/persona/index.js +19 -5
- package/lib/answerers/persona/index.js.map +1 -1
- package/lib/answerers/terminal/index.js +1 -1
- package/lib/answerers/terminal/index.js.map +1 -1
- package/lib/assertions/authoring.d.ts +14 -0
- package/lib/assertions/authoring.d.ts.map +1 -0
- package/lib/assertions/authoring.js +3 -0
- package/lib/assertions/authoring.js.map +1 -0
- package/lib/assertions/core/helpers.d.ts +2 -7
- package/lib/assertions/core/helpers.d.ts.map +1 -1
- package/lib/assertions/core/helpers.js +2 -6
- package/lib/assertions/core/helpers.js.map +1 -1
- package/lib/assertions/core/index.d.ts.map +1 -1
- package/lib/assertions/core/index.js +2 -1
- package/lib/assertions/core/index.js.map +1 -1
- package/lib/assertions/core/resource-checks.d.ts +19 -0
- package/lib/assertions/core/resource-checks.d.ts.map +1 -1
- package/lib/assertions/core/resource-checks.js +32 -0
- package/lib/assertions/core/resource-checks.js.map +1 -1
- package/lib/authoring.d.ts +1 -0
- package/lib/authoring.d.ts.map +1 -1
- package/lib/authoring.js +1 -0
- package/lib/authoring.js.map +1 -1
- package/lib/cli/program.d.ts.map +1 -1
- package/lib/cli/program.js +12 -67
- package/lib/cli/program.js.map +1 -1
- package/lib/cli/render/reporter.d.ts.map +1 -1
- package/lib/cli/render/reporter.js +16 -3
- package/lib/cli/render/reporter.js.map +1 -1
- package/lib/cli/run-options.d.ts +1 -0
- package/lib/cli/run-options.d.ts.map +1 -1
- package/lib/cli/run-options.js.map +1 -1
- package/lib/cli/view.d.ts +13 -0
- package/lib/cli/view.d.ts.map +1 -0
- package/lib/cli/view.js +126 -0
- package/lib/cli/view.js.map +1 -0
- package/lib/core/action-events.d.ts +29 -0
- package/lib/core/action-events.d.ts.map +1 -0
- package/lib/core/action-events.js +51 -0
- package/lib/core/action-events.js.map +1 -0
- package/lib/core/agent-activity.d.ts +49 -0
- package/lib/core/agent-activity.d.ts.map +1 -0
- package/lib/core/agent-activity.js +206 -0
- package/lib/core/agent-activity.js.map +1 -0
- package/lib/core/agent-events.d.ts +26 -0
- package/lib/core/agent-events.d.ts.map +1 -0
- package/lib/core/agent-events.js +13 -0
- package/lib/core/agent-events.js.map +1 -0
- package/lib/core/artifact-evidence.d.ts +8 -0
- package/lib/core/artifact-evidence.d.ts.map +1 -1
- package/lib/core/artifact-evidence.js +36 -4
- package/lib/core/artifact-evidence.js.map +1 -1
- package/lib/core/assertion.d.ts +19 -9
- package/lib/core/assertion.d.ts.map +1 -1
- package/lib/core/assertion.js +26 -0
- package/lib/core/assertion.js.map +1 -1
- package/lib/core/case-authoring-state.d.ts +12 -0
- package/lib/core/case-authoring-state.d.ts.map +1 -0
- package/lib/core/case-authoring-state.js +43 -0
- package/lib/core/case-authoring-state.js.map +1 -0
- package/lib/core/case.d.ts +29 -1
- package/lib/core/case.d.ts.map +1 -1
- package/lib/core/case.js +14 -6
- package/lib/core/case.js.map +1 -1
- package/lib/core/driver-metrics.d.ts +98 -0
- package/lib/core/driver-metrics.d.ts.map +1 -0
- package/lib/core/driver-metrics.js +36 -0
- package/lib/core/driver-metrics.js.map +1 -0
- package/lib/core/driver.d.ts +3 -0
- package/lib/core/driver.d.ts.map +1 -1
- package/lib/core/driver.js.map +1 -1
- package/lib/core/events.d.ts +36 -0
- package/lib/core/events.d.ts.map +1 -1
- package/lib/core/events.js +12 -2
- package/lib/core/events.js.map +1 -1
- package/lib/core/experiment.d.ts +19 -0
- package/lib/core/experiment.d.ts.map +1 -1
- package/lib/core/experiment.js +2 -0
- package/lib/core/experiment.js.map +1 -1
- package/lib/core/index.d.ts +4 -0
- package/lib/core/index.d.ts.map +1 -1
- package/lib/core/index.js +4 -0
- package/lib/core/index.js.map +1 -1
- package/lib/core/lifecycle-fixtures.d.ts +3 -1
- package/lib/core/lifecycle-fixtures.d.ts.map +1 -1
- package/lib/core/lifecycle-fixtures.js +4 -2
- package/lib/core/lifecycle-fixtures.js.map +1 -1
- package/lib/core/lifecycle-hooks.d.ts +9 -0
- package/lib/core/lifecycle-hooks.d.ts.map +1 -1
- package/lib/core/lifecycle-hooks.js +8 -2
- package/lib/core/lifecycle-hooks.js.map +1 -1
- package/lib/core/run.d.ts +150 -0
- package/lib/core/run.d.ts.map +1 -1
- package/lib/core/run.js +9 -0
- package/lib/core/run.js.map +1 -1
- package/lib/core/wire.d.ts +12 -0
- package/lib/core/wire.d.ts.map +1 -1
- package/lib/core/wire.js +5 -1
- package/lib/core/wire.js.map +1 -1
- package/lib/drivers/claude-code/parse-events.d.ts.map +1 -1
- package/lib/drivers/claude-code/parse-events.js +73 -8
- package/lib/drivers/claude-code/parse-events.js.map +1 -1
- package/lib/drivers/codex/app-events.d.ts +9 -2
- package/lib/drivers/codex/app-events.d.ts.map +1 -1
- package/lib/drivers/codex/app-events.js +93 -12
- package/lib/drivers/codex/app-events.js.map +1 -1
- package/lib/drivers/codex/index.d.ts.map +1 -1
- package/lib/drivers/codex/index.js +10 -14
- package/lib/drivers/codex/index.js.map +1 -1
- package/lib/drivers/subprocess/index.d.ts.map +1 -1
- package/lib/drivers/subprocess/index.js +6 -0
- package/lib/drivers/subprocess/index.js.map +1 -1
- package/lib/index.d.ts +10 -6
- package/lib/index.d.ts.map +1 -1
- package/lib/index.js +10 -6
- package/lib/index.js.map +1 -1
- package/lib/orchestrator/annotation-store.d.ts.map +1 -1
- package/lib/orchestrator/annotation-store.js +6 -2
- package/lib/orchestrator/annotation-store.js.map +1 -1
- package/lib/orchestrator/case-extensions.d.ts +6 -0
- package/lib/orchestrator/case-extensions.d.ts.map +1 -0
- package/lib/orchestrator/case-extensions.js +28 -0
- package/lib/orchestrator/case-extensions.js.map +1 -0
- package/lib/orchestrator/compare.d.ts +13 -1
- package/lib/orchestrator/compare.d.ts.map +1 -1
- package/lib/orchestrator/compare.js +12 -0
- package/lib/orchestrator/compare.js.map +1 -1
- package/lib/orchestrator/config.d.ts.map +1 -1
- package/lib/orchestrator/config.js +0 -2
- package/lib/orchestrator/config.js.map +1 -1
- package/lib/orchestrator/doctor.d.ts.map +1 -1
- package/lib/orchestrator/doctor.js +5 -3
- package/lib/orchestrator/doctor.js.map +1 -1
- package/lib/orchestrator/experiment-contracts.d.ts +19 -0
- package/lib/orchestrator/experiment-contracts.d.ts.map +1 -1
- package/lib/orchestrator/experiment-evaluation.d.ts.map +1 -1
- package/lib/orchestrator/experiment-evaluation.js +17 -0
- package/lib/orchestrator/experiment-evaluation.js.map +1 -1
- package/lib/orchestrator/experiment-report.d.ts +19 -0
- package/lib/orchestrator/experiment-report.d.ts.map +1 -1
- package/lib/orchestrator/experiment-runs.d.ts +38 -0
- package/lib/orchestrator/experiment-runs.d.ts.map +1 -1
- package/lib/orchestrator/grader-experiment.d.ts +12 -0
- package/lib/orchestrator/grader-experiment.d.ts.map +1 -1
- package/lib/orchestrator/grading.d.ts +2 -1
- package/lib/orchestrator/grading.d.ts.map +1 -1
- package/lib/orchestrator/grading.js +16 -4
- package/lib/orchestrator/grading.js.map +1 -1
- package/lib/orchestrator/index.d.ts +1 -1
- package/lib/orchestrator/index.d.ts.map +1 -1
- package/lib/orchestrator/index.js +1 -1
- package/lib/orchestrator/index.js.map +1 -1
- package/lib/orchestrator/list-runs.d.ts +7 -1
- package/lib/orchestrator/list-runs.d.ts.map +1 -1
- package/lib/orchestrator/list-runs.js +50 -6
- package/lib/orchestrator/list-runs.js.map +1 -1
- package/lib/orchestrator/loader.d.ts.map +1 -1
- package/lib/orchestrator/loader.js +2 -0
- package/lib/orchestrator/loader.js.map +1 -1
- package/lib/orchestrator/orchestrator.d.ts +3 -0
- package/lib/orchestrator/orchestrator.d.ts.map +1 -1
- package/lib/orchestrator/orchestrator.js +42 -14
- package/lib/orchestrator/orchestrator.js.map +1 -1
- package/lib/orchestrator/owned-lock.d.ts.map +1 -1
- package/lib/orchestrator/owned-lock.js +52 -38
- package/lib/orchestrator/owned-lock.js.map +1 -1
- package/lib/orchestrator/regrade.d.ts +6 -0
- package/lib/orchestrator/regrade.d.ts.map +1 -1
- package/lib/orchestrator/regrade.js +8 -4
- package/lib/orchestrator/regrade.js.map +1 -1
- package/lib/orchestrator/run-execution.d.ts +2 -1
- package/lib/orchestrator/run-execution.d.ts.map +1 -1
- package/lib/orchestrator/run-execution.js +20 -5
- package/lib/orchestrator/run-execution.js.map +1 -1
- package/lib/orchestrator/suite-compare.d.ts +34 -0
- package/lib/orchestrator/suite-compare.d.ts.map +1 -1
- package/lib/orchestrator/suite-summary.d.ts +68 -0
- package/lib/orchestrator/suite-summary.d.ts.map +1 -1
- package/lib/orchestrator/suite-summary.js +45 -7
- package/lib/orchestrator/suite-summary.js.map +1 -1
- package/lib/previewer/artifacts.d.ts +25 -0
- package/lib/previewer/artifacts.d.ts.map +1 -0
- package/lib/previewer/artifacts.js +180 -0
- package/lib/previewer/artifacts.js.map +1 -0
- package/lib/previewer/server.d.ts +105 -0
- package/lib/previewer/server.d.ts.map +1 -0
- package/lib/previewer/server.js +542 -0
- package/lib/previewer/server.js.map +1 -0
- package/package.json +2 -2
- package/skills/lux/SKILL.md +7 -3
- package/skills/lux/references/cli.md +25 -2
- package/skills/lux/references/eval-authoring.md +158 -4
- package/skills/lux-answerer/SKILL.md +1 -1
- package/src/answerers/persona/index.ts +20 -4
- package/src/answerers/terminal/index.ts +1 -1
- package/src/assertions/authoring.ts +81 -0
- package/src/assertions/core/helpers.ts +3 -6
- package/src/assertions/core/index.ts +2 -0
- package/src/assertions/core/resource-checks.ts +35 -0
- package/src/authoring.ts +1 -0
- package/src/cli/program.ts +14 -93
- package/src/cli/render/reporter.ts +18 -3
- package/src/cli/run-options.ts +1 -0
- package/src/cli/view.ts +168 -0
- package/src/core/action-events.ts +59 -0
- package/src/core/agent-activity.ts +236 -0
- package/src/core/agent-events.ts +17 -0
- package/src/core/artifact-evidence.ts +47 -4
- package/src/core/assertion.ts +41 -22
- package/src/core/case-authoring-state.ts +65 -0
- package/src/core/case.ts +57 -6
- package/src/core/driver-metrics.ts +51 -0
- package/src/core/driver.ts +3 -0
- package/src/core/events.ts +14 -2
- package/src/core/experiment.ts +2 -0
- package/src/core/index.ts +4 -0
- package/src/core/lifecycle-fixtures.ts +7 -2
- package/src/core/lifecycle-hooks.ts +16 -2
- package/src/core/run.ts +9 -0
- package/src/core/wire.ts +5 -1
- package/src/drivers/claude-code/parse-events.ts +98 -12
- package/src/drivers/codex/app-events.ts +116 -16
- package/src/drivers/codex/index.ts +16 -14
- package/src/drivers/subprocess/index.ts +6 -0
- package/src/index.ts +50 -6
- package/src/orchestrator/annotation-store.ts +5 -1
- package/src/orchestrator/case-extensions.ts +44 -0
- package/src/orchestrator/compare.ts +18 -0
- package/src/orchestrator/config.ts +0 -2
- package/src/orchestrator/doctor.ts +5 -3
- package/src/orchestrator/experiment-evaluation.ts +23 -0
- package/src/orchestrator/grading.ts +21 -2
- package/src/orchestrator/index.ts +1 -0
- package/src/orchestrator/list-runs.ts +66 -5
- package/src/orchestrator/loader.ts +2 -0
- package/src/orchestrator/orchestrator.ts +86 -40
- package/src/orchestrator/owned-lock.ts +54 -34
- package/src/orchestrator/regrade.ts +8 -7
- package/src/orchestrator/run-execution.ts +24 -3
- package/src/orchestrator/suite-summary.ts +101 -8
- package/src/previewer/artifacts.ts +221 -0
- package/src/previewer/server.ts +765 -0
- package/viewer/app.css +2075 -0
- package/viewer/app.js +606 -0
- package/viewer/artifact-view.js +48 -0
- package/viewer/charts.js +86 -0
- package/viewer/detail.js +1214 -0
- package/viewer/format.js +129 -0
- package/viewer/index.html +151 -0
- package/lib/answerers/scripted/index.d.ts +0 -8
- package/lib/answerers/scripted/index.d.ts.map +0 -1
- package/lib/answerers/scripted/index.js +0 -163
- package/lib/answerers/scripted/index.js.map +0 -1
- package/src/answerers/scripted/index.ts +0 -186
package/viewer/detail.js
ADDED
|
@@ -0,0 +1,1214 @@
|
|
|
1
|
+
import { comparisonBars, compositionFigure, quantizedBar } from "./charts.js";
|
|
2
|
+
import {
|
|
3
|
+
badge,
|
|
4
|
+
compactNumber,
|
|
5
|
+
detailEmpty,
|
|
6
|
+
duration,
|
|
7
|
+
escapeHtml,
|
|
8
|
+
finite,
|
|
9
|
+
fullDate,
|
|
10
|
+
metricName,
|
|
11
|
+
metricValue,
|
|
12
|
+
percentage,
|
|
13
|
+
signedDuration,
|
|
14
|
+
signedPercentage,
|
|
15
|
+
statusLabel,
|
|
16
|
+
toneForDelta,
|
|
17
|
+
totalTokens,
|
|
18
|
+
} from "./format.js";
|
|
19
|
+
|
|
20
|
+
const json = (value) => {
|
|
21
|
+
if (value === undefined) return "";
|
|
22
|
+
try {
|
|
23
|
+
return JSON.stringify(value, null, 2);
|
|
24
|
+
} catch {
|
|
25
|
+
return String(value);
|
|
26
|
+
}
|
|
27
|
+
};
|
|
28
|
+
|
|
29
|
+
const eventContent = (event) => {
|
|
30
|
+
if (event.type === "lifecycle") return [event.stage, "Run lifecycle"];
|
|
31
|
+
if (event.type === "ready") {
|
|
32
|
+
return [
|
|
33
|
+
`Agent ready · ${event.agentId}`,
|
|
34
|
+
event.model ? `Model ${event.model}` : "Driver ready",
|
|
35
|
+
];
|
|
36
|
+
}
|
|
37
|
+
if (event.type === "interrupt") {
|
|
38
|
+
return [
|
|
39
|
+
`Human-in-the-loop · ${event.kind}`,
|
|
40
|
+
event.question ?? event.request ?? json(event.payload),
|
|
41
|
+
];
|
|
42
|
+
}
|
|
43
|
+
if (event.type === "responded") {
|
|
44
|
+
let answer = event.text ?? null;
|
|
45
|
+
if (!answer && typeof event.approved === "boolean") {
|
|
46
|
+
answer = event.approved ? "Approved" : "Denied";
|
|
47
|
+
}
|
|
48
|
+
return [`Answerer · ${event.kind}`, answer ?? json(event.payload)];
|
|
49
|
+
}
|
|
50
|
+
if (event.type === "progress") {
|
|
51
|
+
const payload = event.payload ?? {};
|
|
52
|
+
const actor = event.agent?.role === "subagent" ? ` · Subagent ${event.agent.id}` : "";
|
|
53
|
+
if (event.kind === "agent-spawn") {
|
|
54
|
+
const spawned = payload.agent ?? {};
|
|
55
|
+
return [
|
|
56
|
+
`Subagent spawned · ${spawned.name ?? spawned.id ?? "unknown"}`,
|
|
57
|
+
[spawned.id, spawned.model, payload.prompt].filter(Boolean).join(" · "),
|
|
58
|
+
];
|
|
59
|
+
}
|
|
60
|
+
if (event.kind === "agent-completed") {
|
|
61
|
+
return [
|
|
62
|
+
`Subagent ${payload.status ?? "completed"} · ${payload.agentId ?? "unknown"}`,
|
|
63
|
+
"Agent lifecycle",
|
|
64
|
+
];
|
|
65
|
+
}
|
|
66
|
+
if (event.kind === "action") {
|
|
67
|
+
return [
|
|
68
|
+
`Action${actor} · ${payload.name ?? "unknown"}`,
|
|
69
|
+
payload.category ? `Category: ${payload.category}` : json(payload),
|
|
70
|
+
];
|
|
71
|
+
}
|
|
72
|
+
const name = event.kind === "tool-call" ? payload.name : event.kind;
|
|
73
|
+
return [`Progress${actor} · ${name ?? event.kind}`, json(payload)];
|
|
74
|
+
}
|
|
75
|
+
if (event.type === "error") return [`Error · ${event.exitReason}`, event.reason];
|
|
76
|
+
return ["Completed", event.summary ? json(event.summary) : "Driver finished"];
|
|
77
|
+
};
|
|
78
|
+
|
|
79
|
+
const eventTime = (event, startedAt) => {
|
|
80
|
+
if (typeof event.ts !== "number") return "setup";
|
|
81
|
+
if (event.ts > 1_000_000_000_000) return fullDate(event.ts);
|
|
82
|
+
const start = Date.parse(startedAt ?? "");
|
|
83
|
+
if (!Number.isFinite(start)) return duration(event.ts);
|
|
84
|
+
return `+${duration(Math.max(0, event.ts - start))}`;
|
|
85
|
+
};
|
|
86
|
+
|
|
87
|
+
const assertionMeta = (assertion) =>
|
|
88
|
+
[
|
|
89
|
+
assertion.type,
|
|
90
|
+
`threshold ${finite(assertion.threshold)}`,
|
|
91
|
+
`weight ${finite(assertion.weight)}`,
|
|
92
|
+
duration(assertion.durationMs),
|
|
93
|
+
assertion.usage ? `${compactNumber(totalTokens(assertion.usage))} judge tokens` : null,
|
|
94
|
+
]
|
|
95
|
+
.filter(Boolean)
|
|
96
|
+
.join(" · ");
|
|
97
|
+
|
|
98
|
+
const assertionDiagnostics = (assertion) => {
|
|
99
|
+
const rows = [
|
|
100
|
+
assertion.comment ? `<p class="assertion-comment">${escapeHtml(assertion.comment)}</p>` : "",
|
|
101
|
+
assertion.errorMessage
|
|
102
|
+
? `<p class="diagnostic-error">${escapeHtml(assertion.errorMessage)}</p>`
|
|
103
|
+
: "",
|
|
104
|
+
assertion.detail !== undefined
|
|
105
|
+
? `<details class="diagnostic"><summary>Structured detail</summary><pre>${escapeHtml(
|
|
106
|
+
json(assertion.detail),
|
|
107
|
+
)}</pre></details>`
|
|
108
|
+
: "",
|
|
109
|
+
];
|
|
110
|
+
return rows.join("");
|
|
111
|
+
};
|
|
112
|
+
|
|
113
|
+
const inferredMetricUnit = (name, unit) => {
|
|
114
|
+
if (unit) return unit;
|
|
115
|
+
if (name.endsWith("Ms")) return "milliseconds";
|
|
116
|
+
if (name.endsWith(".bytes")) return "bytes";
|
|
117
|
+
if (name.endsWith(".usd")) return "usd";
|
|
118
|
+
if (name.endsWith("Rate")) return "ratio";
|
|
119
|
+
if (
|
|
120
|
+
name.includes("tokens") ||
|
|
121
|
+
/(?:^|\.)(?:input|output|total|cacheRead|cacheCreation)$/.test(name)
|
|
122
|
+
) {
|
|
123
|
+
return "tokens";
|
|
124
|
+
}
|
|
125
|
+
return "number";
|
|
126
|
+
};
|
|
127
|
+
|
|
128
|
+
const driverMetricStats = (metrics) =>
|
|
129
|
+
Object.entries(metrics ?? {})
|
|
130
|
+
.map(
|
|
131
|
+
([name, metric]) =>
|
|
132
|
+
`<div><dt>${escapeHtml(metricName(name, metric.label))}</dt><dd>${escapeHtml(
|
|
133
|
+
metricValue(metric.value, metric.unit),
|
|
134
|
+
)}</dd><small>${escapeHtml(name)}</small></div>`,
|
|
135
|
+
)
|
|
136
|
+
.join("");
|
|
137
|
+
|
|
138
|
+
const agentActivityCards = (activities) =>
|
|
139
|
+
(activities ?? [])
|
|
140
|
+
.filter((activity) => activity.agent?.role === "subagent")
|
|
141
|
+
.map((activity, index) => {
|
|
142
|
+
const agent = activity.agent;
|
|
143
|
+
const actions = activity.actions ?? {};
|
|
144
|
+
const tools = activity.toolCalls ?? {};
|
|
145
|
+
const label = agent.name ?? `Subagent ${index + 1}`;
|
|
146
|
+
return `<article class="agent-card">
|
|
147
|
+
<header><div><strong>${escapeHtml(label)}</strong><small>${escapeHtml(agent.id)}</small></div><span>${escapeHtml(
|
|
148
|
+
`${activity.status ?? "unknown"} · ${duration(activity.durationMs)}`,
|
|
149
|
+
)}</span></header>
|
|
150
|
+
<dl>
|
|
151
|
+
<div><dt>Events</dt><dd>${compactNumber(activity.eventCount)}</dd></div>
|
|
152
|
+
<div><dt>Tokens</dt><dd>${compactNumber(activity.usage?.total)}</dd></div>
|
|
153
|
+
<div><dt>Tool calls</dt><dd>${compactNumber(tools.total)}</dd></div>
|
|
154
|
+
<div><dt>Succeeded</dt><dd>${compactNumber(tools.succeeded)}</dd></div>
|
|
155
|
+
<div><dt>Failed</dt><dd>${compactNumber(tools.failed)}</dd></div>
|
|
156
|
+
<div><dt>Actions</dt><dd>${compactNumber(actions.total)}</dd></div>
|
|
157
|
+
<div><dt>Mutations</dt><dd>${compactNumber(actions.mutations)}</dd></div>
|
|
158
|
+
<div><dt>First action</dt><dd>${escapeHtml(duration(actions.timeToFirstActionMs))}</dd></div>
|
|
159
|
+
<div><dt>First mutation</dt><dd>${escapeHtml(duration(actions.timeToFirstMutationMs))}</dd></div>
|
|
160
|
+
</dl>
|
|
161
|
+
<small class="agent-parent">Spawn ${escapeHtml(agent.spawnToolUseId ?? "unknown")} · parent ${escapeHtml(
|
|
162
|
+
agent.parentId ?? "unknown",
|
|
163
|
+
)}</small>
|
|
164
|
+
</article>`;
|
|
165
|
+
})
|
|
166
|
+
.join("");
|
|
167
|
+
|
|
168
|
+
export const renderRunDetail = (item, detail) => {
|
|
169
|
+
const metadata = detail.metadata;
|
|
170
|
+
const grading = detail.grading;
|
|
171
|
+
const assertions = grading?.assertions ?? [];
|
|
172
|
+
const events = detail.events ?? [];
|
|
173
|
+
const artifacts = detail.artifacts ?? [];
|
|
174
|
+
const agentTokens = totalTokens(metadata?.usage?.agent);
|
|
175
|
+
const answererTokens = totalTokens(metadata?.usage?.answerer);
|
|
176
|
+
const gradingTokens = totalTokens(metadata?.usage?.grading);
|
|
177
|
+
const harnessTokens = answererTokens + gradingTokens;
|
|
178
|
+
const phase = metadata?.phaseDurationMs;
|
|
179
|
+
const actionTiming = metadata?.actionTiming;
|
|
180
|
+
const prompt = detail.case?.prompt;
|
|
181
|
+
const driverMetrics = driverMetricStats(metadata?.driverMetrics);
|
|
182
|
+
const subagents = agentActivityCards(metadata?.agentActivity);
|
|
183
|
+
return `
|
|
184
|
+
<header class="detail-header">
|
|
185
|
+
<div class="detail-kicker">
|
|
186
|
+
${badge(item.status)}
|
|
187
|
+
<span class="detail-path">${escapeHtml(item.path)}</span>
|
|
188
|
+
</div>
|
|
189
|
+
<div class="detail-title-row">
|
|
190
|
+
<div>
|
|
191
|
+
<h2>${escapeHtml(detail.case?.id ?? item.caseId)}</h2>
|
|
192
|
+
<p class="detail-subtitle">${escapeHtml(
|
|
193
|
+
[metadata?.driver?.name, metadata?.driver?.model, fullDate(metadata?.startedAt)]
|
|
194
|
+
.filter(Boolean)
|
|
195
|
+
.join(" · "),
|
|
196
|
+
)}</p>
|
|
197
|
+
</div>
|
|
198
|
+
<div class="score-block">
|
|
199
|
+
<strong>${escapeHtml(percentage(grading?.caseScore))}</strong>
|
|
200
|
+
<small>case score</small>
|
|
201
|
+
</div>
|
|
202
|
+
</div>
|
|
203
|
+
</header>
|
|
204
|
+
<div class="detail-body">
|
|
205
|
+
${
|
|
206
|
+
grading?.incomplete
|
|
207
|
+
? `<div class="attention-banner">Incomplete ${escapeHtml(
|
|
208
|
+
grading.incomplete.phase ?? "run",
|
|
209
|
+
)}: ${escapeHtml(grading.incomplete.exitReason)}</div>`
|
|
210
|
+
: ""
|
|
211
|
+
}
|
|
212
|
+
<dl class="detail-stats">
|
|
213
|
+
<div><dt>Duration</dt><dd>${escapeHtml(duration(item.durationMs))}</dd></div>
|
|
214
|
+
<div><dt>Agent tokens</dt><dd>${escapeHtml(compactNumber(agentTokens))}</dd></div>
|
|
215
|
+
<div><dt>Harness tokens</dt><dd>${escapeHtml(compactNumber(harnessTokens))}</dd></div>
|
|
216
|
+
<div><dt>Artifacts</dt><dd>${artifacts.length}</dd></div>
|
|
217
|
+
<div><dt>First action</dt><dd>${escapeHtml(duration(actionTiming?.timeToFirstActionMs))}</dd></div>
|
|
218
|
+
<div><dt>First mutation</dt><dd>${escapeHtml(duration(actionTiming?.timeToFirstMutationMs))}</dd></div>
|
|
219
|
+
<div><dt>First durable mutation</dt><dd>${escapeHtml(
|
|
220
|
+
duration(actionTiming?.timeToFirstDurableMutationMs),
|
|
221
|
+
)}</dd></div>
|
|
222
|
+
</dl>
|
|
223
|
+
${
|
|
224
|
+
driverMetrics
|
|
225
|
+
? `<section class="detail-section"><h3>Driver metrics <span>reported by ${escapeHtml(
|
|
226
|
+
metadata?.driver?.name ?? "driver",
|
|
227
|
+
)}</span></h3><dl class="detail-stats driver-metrics">${driverMetrics}</dl></section>`
|
|
228
|
+
: ""
|
|
229
|
+
}
|
|
230
|
+
${
|
|
231
|
+
subagents
|
|
232
|
+
? `<section class="detail-section"><h3>Subagent activity <span>${
|
|
233
|
+
metadata.agentActivity.filter((activity) => activity.agent.role === "subagent").length
|
|
234
|
+
} spawned</span></h3><div class="agent-grid">${subagents}</div></section>`
|
|
235
|
+
: ""
|
|
236
|
+
}
|
|
237
|
+
<div class="evidence-visual-grid">
|
|
238
|
+
${compositionFigure(
|
|
239
|
+
"Judgment outcome",
|
|
240
|
+
[
|
|
241
|
+
{
|
|
242
|
+
key: "passed",
|
|
243
|
+
label: "Passed",
|
|
244
|
+
value: assertions.filter((assertion) => assertion.passed).length,
|
|
245
|
+
},
|
|
246
|
+
{
|
|
247
|
+
key: "regressed",
|
|
248
|
+
label: "Failed",
|
|
249
|
+
value: assertions.filter((assertion) => !assertion.passed).length,
|
|
250
|
+
},
|
|
251
|
+
],
|
|
252
|
+
`${assertions.filter((assertion) => assertion.passed).length} of ${
|
|
253
|
+
assertions.length
|
|
254
|
+
} assertions passed`,
|
|
255
|
+
)}
|
|
256
|
+
${
|
|
257
|
+
agentTokens + answererTokens + gradingTokens > 0
|
|
258
|
+
? compositionFigure(
|
|
259
|
+
"Token composition",
|
|
260
|
+
[
|
|
261
|
+
{ key: "agent", label: "Agent", value: agentTokens },
|
|
262
|
+
{ key: "answerer", label: "Answerer", value: answererTokens },
|
|
263
|
+
{ key: "grading", label: "Grading", value: gradingTokens },
|
|
264
|
+
],
|
|
265
|
+
`${agentTokens} agent, ${answererTokens} answerer, ${gradingTokens} grading tokens`,
|
|
266
|
+
)
|
|
267
|
+
: ""
|
|
268
|
+
}
|
|
269
|
+
${
|
|
270
|
+
phase
|
|
271
|
+
? compositionFigure(
|
|
272
|
+
"Runtime composition",
|
|
273
|
+
[
|
|
274
|
+
{ key: "agent", label: "Agent", value: phase.agent },
|
|
275
|
+
{ key: "collect", label: "Collect", value: phase.collect },
|
|
276
|
+
{ key: "grading", label: "Grading", value: phase.grading },
|
|
277
|
+
],
|
|
278
|
+
`${duration(phase.agent)} agent, ${duration(phase.collect)} collection, ${duration(
|
|
279
|
+
phase.grading,
|
|
280
|
+
)} grading`,
|
|
281
|
+
)
|
|
282
|
+
: ""
|
|
283
|
+
}
|
|
284
|
+
</div>
|
|
285
|
+
${
|
|
286
|
+
prompt
|
|
287
|
+
? `<section class="detail-section"><h3>Case prompt</h3><div class="prompt-block">${escapeHtml(
|
|
288
|
+
prompt,
|
|
289
|
+
)}</div></section>`
|
|
290
|
+
: ""
|
|
291
|
+
}
|
|
292
|
+
<section class="detail-section">
|
|
293
|
+
<h3>Judgment <span>${assertions.filter((entry) => entry.passed).length}/${assertions.length} passed</span></h3>
|
|
294
|
+
<div class="assertion-list">
|
|
295
|
+
${
|
|
296
|
+
assertions.length
|
|
297
|
+
? assertions
|
|
298
|
+
.map(
|
|
299
|
+
(assertion) => `
|
|
300
|
+
<article class="assertion${assertion.passed ? " passed" : ""}">
|
|
301
|
+
<span class="assertion-mark">${assertion.passed ? "✓" : "×"}</span>
|
|
302
|
+
<span class="assertion-content">
|
|
303
|
+
<strong>${escapeHtml(assertion.name)}</strong>
|
|
304
|
+
<small>${escapeHtml(assertionMeta(assertion))}</small>
|
|
305
|
+
${quantizedBar(
|
|
306
|
+
assertion.score,
|
|
307
|
+
assertion.passed ? "tone-positive" : "tone-negative",
|
|
308
|
+
`${assertion.name} scored ${percentage(
|
|
309
|
+
assertion.score,
|
|
310
|
+
)}; threshold ${percentage(assertion.threshold)}`,
|
|
311
|
+
assertion.threshold,
|
|
312
|
+
)}
|
|
313
|
+
${assertionDiagnostics(assertion)}
|
|
314
|
+
</span>
|
|
315
|
+
<span class="assertion-score">${escapeHtml(
|
|
316
|
+
percentage(assertion.score),
|
|
317
|
+
)}</span>
|
|
318
|
+
</article>`,
|
|
319
|
+
)
|
|
320
|
+
.join("")
|
|
321
|
+
: '<div class="artifact"><span>No grading assertions recorded</span><small>ungraded</small></div>'
|
|
322
|
+
}
|
|
323
|
+
</div>
|
|
324
|
+
</section>
|
|
325
|
+
<section class="detail-section">
|
|
326
|
+
<h3>Trace <span>${events.length} events</span></h3>
|
|
327
|
+
<div class="trace-list">
|
|
328
|
+
${
|
|
329
|
+
events.length
|
|
330
|
+
? events
|
|
331
|
+
.map((event) => {
|
|
332
|
+
const [title, content] = eventContent(event);
|
|
333
|
+
return `<article class="trace-item">
|
|
334
|
+
<time>${escapeHtml(eventTime(event, metadata?.startedAt))}</time>
|
|
335
|
+
<div><strong>${escapeHtml(title)}</strong><p>${escapeHtml(content)}</p></div>
|
|
336
|
+
</article>`;
|
|
337
|
+
})
|
|
338
|
+
.join("")
|
|
339
|
+
: '<div class="artifact"><span>No trace events recorded</span><small>empty</small></div>'
|
|
340
|
+
}
|
|
341
|
+
</div>
|
|
342
|
+
</section>
|
|
343
|
+
<section class="detail-section">
|
|
344
|
+
<h3>Artifacts <span>${artifacts.length} files</span></h3>
|
|
345
|
+
<div class="artifact-list">
|
|
346
|
+
${
|
|
347
|
+
artifacts.length
|
|
348
|
+
? artifacts
|
|
349
|
+
.map(
|
|
350
|
+
(artifact) => `<button class="artifact artifact-button" type="button"
|
|
351
|
+
data-artifact="${escapeHtml(artifact)}">
|
|
352
|
+
<span>${escapeHtml(artifact)}</span><small>preview →</small>
|
|
353
|
+
</button>`,
|
|
354
|
+
)
|
|
355
|
+
.join("")
|
|
356
|
+
: '<div class="artifact"><span>No captured artifacts</span><small>empty</small></div>'
|
|
357
|
+
}
|
|
358
|
+
</div>
|
|
359
|
+
</section>
|
|
360
|
+
</div>`;
|
|
361
|
+
};
|
|
362
|
+
|
|
363
|
+
const suiteItemStatus = (item) => {
|
|
364
|
+
if (item.status !== "completed") return item.status;
|
|
365
|
+
if (item.exitReason !== "done" || item.casePassed === false) return "failed";
|
|
366
|
+
if (item.casePassed === true) return "passed";
|
|
367
|
+
return "ungraded";
|
|
368
|
+
};
|
|
369
|
+
|
|
370
|
+
const metricSummaryRows = (metrics) =>
|
|
371
|
+
Object.entries(metrics ?? {})
|
|
372
|
+
.map(
|
|
373
|
+
([name, summary]) => `<tr>
|
|
374
|
+
<td>${escapeHtml(name)}</td>
|
|
375
|
+
<td>${escapeHtml(metricValue(summary.mean, inferredMetricUnit(name, summary.unit)))}</td>
|
|
376
|
+
<td>${escapeHtml(metricValue(summary.median, inferredMetricUnit(name, summary.unit)))}</td>
|
|
377
|
+
<td>${escapeHtml(metricValue(summary.p90, inferredMetricUnit(name, summary.unit)))}</td>
|
|
378
|
+
<td>${summary.n}</td>
|
|
379
|
+
</tr>`,
|
|
380
|
+
)
|
|
381
|
+
.join("");
|
|
382
|
+
|
|
383
|
+
export const renderSuiteDetail = (item, payload) => {
|
|
384
|
+
const summary = payload?.summary;
|
|
385
|
+
const totals = summary?.totals;
|
|
386
|
+
const metrics = metricSummaryRows(summary?.metrics);
|
|
387
|
+
return `
|
|
388
|
+
<header class="detail-header">
|
|
389
|
+
<div class="detail-kicker">${badge(item.status)}<span class="detail-path">${escapeHtml(
|
|
390
|
+
item.path,
|
|
391
|
+
)}</span></div>
|
|
392
|
+
<div class="detail-title-row">
|
|
393
|
+
<div><h2>${escapeHtml(item.suiteId)}</h2><p class="detail-subtitle">${escapeHtml(
|
|
394
|
+
`${item.profileId ?? "default profile"} · ${fullDate(item.startedAt)}`,
|
|
395
|
+
)}</p></div>
|
|
396
|
+
<div class="score-block"><strong>${escapeHtml(
|
|
397
|
+
percentage(totals?.passRate),
|
|
398
|
+
)}</strong><small>${totals?.passed ?? 0}/${totals?.attempts ?? item.items.length} cases passed</small></div>
|
|
399
|
+
</div>
|
|
400
|
+
</header>
|
|
401
|
+
<div class="detail-body">
|
|
402
|
+
<dl class="detail-stats">
|
|
403
|
+
<div><dt>Attempts</dt><dd>${totals?.attempts ?? item.items.length}</dd></div>
|
|
404
|
+
<div><dt>Concurrency</dt><dd>${item.concurrency}</dd></div>
|
|
405
|
+
<div><dt>Failed</dt><dd>${totals?.failed ?? 0}</dd></div>
|
|
406
|
+
<div><dt>Pending</dt><dd>${totals?.pending ?? 0}</dd></div>
|
|
407
|
+
</dl>
|
|
408
|
+
${compositionFigure(
|
|
409
|
+
"Suite outcomes",
|
|
410
|
+
[
|
|
411
|
+
{ key: "passed", label: "Passed", value: totals?.passed ?? 0 },
|
|
412
|
+
{ key: "regressed", label: "Failed", value: totals?.failed ?? 0 },
|
|
413
|
+
{ key: "ungraded", label: "Ungraded", value: totals?.ungraded ?? 0 },
|
|
414
|
+
{ key: "pending", label: "Pending", value: totals?.pending ?? 0 },
|
|
415
|
+
],
|
|
416
|
+
`${totals?.passed ?? 0} passed, ${totals?.failed ?? 0} failed, ${
|
|
417
|
+
totals?.ungraded ?? 0
|
|
418
|
+
} ungraded, ${totals?.pending ?? 0} pending`,
|
|
419
|
+
)}
|
|
420
|
+
${
|
|
421
|
+
metrics
|
|
422
|
+
? `<section class="detail-section">
|
|
423
|
+
<h3>Aggregate metrics <span>mean / median / p90</span></h3>
|
|
424
|
+
<div class="matrix-wrap"><table class="matrix">
|
|
425
|
+
<thead><tr><th>Metric</th><th>Mean</th><th>Median</th><th>P90</th><th>N</th></tr></thead>
|
|
426
|
+
<tbody>${metrics}</tbody>
|
|
427
|
+
</table></div>
|
|
428
|
+
</section>`
|
|
429
|
+
: ""
|
|
430
|
+
}
|
|
431
|
+
<section class="detail-section">
|
|
432
|
+
<h3>Case attempts <span>${item.items.length} total</span></h3>
|
|
433
|
+
<div class="suite-list">
|
|
434
|
+
${item.items
|
|
435
|
+
.map((entry) => {
|
|
436
|
+
const content = `<span>${escapeHtml(entry.caseId)}</span>
|
|
437
|
+
<small>${escapeHtml(statusLabel(suiteItemStatus(entry)))}</small>`;
|
|
438
|
+
return entry.runId
|
|
439
|
+
? `<button class="suite-item suite-button" type="button" data-open-run="${escapeHtml(
|
|
440
|
+
entry.runId,
|
|
441
|
+
)}">${content}</button>`
|
|
442
|
+
: `<div class="suite-item">${content}</div>`;
|
|
443
|
+
})
|
|
444
|
+
.join("")}
|
|
445
|
+
</div>
|
|
446
|
+
</section>
|
|
447
|
+
</div>`;
|
|
448
|
+
};
|
|
449
|
+
|
|
450
|
+
const candidateById = (payload, id) => payload.candidates.find((candidate) => candidate.id === id);
|
|
451
|
+
|
|
452
|
+
const baselineCandidate = (payload) =>
|
|
453
|
+
candidateById(payload, "baseline") ??
|
|
454
|
+
payload.candidates.find((candidate) => candidate.parentIds.length === 0) ??
|
|
455
|
+
payload.candidates[0] ??
|
|
456
|
+
null;
|
|
457
|
+
|
|
458
|
+
const evaluationKey = (evaluation) => `${evaluation.profileId}/${evaluation.split}`;
|
|
459
|
+
|
|
460
|
+
const caseDeltaRows = (baseline, target) => {
|
|
461
|
+
const cases = new Set([
|
|
462
|
+
...Object.keys(baseline?.casePassRates ?? {}),
|
|
463
|
+
...Object.keys(target?.casePassRates ?? {}),
|
|
464
|
+
]);
|
|
465
|
+
return [...cases]
|
|
466
|
+
.sort()
|
|
467
|
+
.map((caseId) => {
|
|
468
|
+
const before = baseline?.casePassRates?.[caseId];
|
|
469
|
+
const after = target?.casePassRates?.[caseId];
|
|
470
|
+
const delta = typeof before === "number" && typeof after === "number" ? after - before : null;
|
|
471
|
+
return `<tr>
|
|
472
|
+
<td>${escapeHtml(caseId)}</td>
|
|
473
|
+
<td class="bar-table-cell">${quantizedBar(
|
|
474
|
+
before,
|
|
475
|
+
"tone-before",
|
|
476
|
+
`${caseId} baseline pass rate ${percentage(before)}`,
|
|
477
|
+
)}<small>${escapeHtml(percentage(before))}</small></td>
|
|
478
|
+
<td class="bar-table-cell">${quantizedBar(
|
|
479
|
+
after,
|
|
480
|
+
"tone-after",
|
|
481
|
+
`${caseId} candidate pass rate ${percentage(after)}`,
|
|
482
|
+
)}<small>${escapeHtml(percentage(after))}</small></td>
|
|
483
|
+
<td class="${toneForDelta(delta)}">${escapeHtml(signedPercentage(delta))}</td>
|
|
484
|
+
</tr>`;
|
|
485
|
+
})
|
|
486
|
+
.join("");
|
|
487
|
+
};
|
|
488
|
+
|
|
489
|
+
const evaluationMetricComparisons = (before, after) => {
|
|
490
|
+
const definitions = { ...(before?.metricDefinitions ?? {}), ...(after?.metricDefinitions ?? {}) };
|
|
491
|
+
return Object.keys(definitions)
|
|
492
|
+
.sort()
|
|
493
|
+
.map((name) => {
|
|
494
|
+
const definition = definitions[name];
|
|
495
|
+
const a = before?.metrics?.[name];
|
|
496
|
+
const b = after?.metrics?.[name];
|
|
497
|
+
const delta = typeof a === "number" && typeof b === "number" ? b - a : null;
|
|
498
|
+
const tone =
|
|
499
|
+
definition.better === "neutral"
|
|
500
|
+
? "metric-neutral"
|
|
501
|
+
: toneForDelta(delta, definition.better === "lower");
|
|
502
|
+
return comparisonBars({
|
|
503
|
+
title: metricName(name, definition.label),
|
|
504
|
+
before: a,
|
|
505
|
+
after: b,
|
|
506
|
+
beforeText: metricValue(a, definition.unit),
|
|
507
|
+
afterText: metricValue(b, definition.unit),
|
|
508
|
+
deltaText: signedMetricValue(delta, definition.unit),
|
|
509
|
+
deltaTone: tone,
|
|
510
|
+
});
|
|
511
|
+
})
|
|
512
|
+
.join("");
|
|
513
|
+
};
|
|
514
|
+
|
|
515
|
+
const experimentScopes = (baselineEvaluations, targetEvaluations) => {
|
|
516
|
+
const baselineByKey = new Map(baselineEvaluations.map((entry) => [evaluationKey(entry), entry]));
|
|
517
|
+
const targetByKey = new Map(targetEvaluations.map((entry) => [evaluationKey(entry), entry]));
|
|
518
|
+
const keys = new Set([...baselineByKey.keys(), ...targetByKey.keys()]);
|
|
519
|
+
return [...keys]
|
|
520
|
+
.sort()
|
|
521
|
+
.map((key) => {
|
|
522
|
+
const before = baselineByKey.get(key);
|
|
523
|
+
const after = targetByKey.get(key);
|
|
524
|
+
const delta =
|
|
525
|
+
typeof before?.passRate === "number" && typeof after?.passRate === "number"
|
|
526
|
+
? after.passRate - before.passRate
|
|
527
|
+
: null;
|
|
528
|
+
const metrics = evaluationMetricComparisons(before, after);
|
|
529
|
+
return `<article class="comparison-scope">
|
|
530
|
+
<header>
|
|
531
|
+
<strong>${escapeHtml(key)}</strong>
|
|
532
|
+
<span class="${toneForDelta(delta)}">${escapeHtml(signedPercentage(delta))}</span>
|
|
533
|
+
</header>
|
|
534
|
+
<div class="matrix-wrap"><table class="matrix">
|
|
535
|
+
<thead><tr><th>Case</th><th>Baseline</th><th>Candidate</th><th>Δ</th></tr></thead>
|
|
536
|
+
<tbody>${caseDeltaRows(before, after) || '<tr><td colspan="4">No per-case evidence.</td></tr>'}</tbody>
|
|
537
|
+
</table></div>
|
|
538
|
+
${metrics ? `<div class="comparison-plot-grid experiment-metrics">${metrics}</div>` : ""}
|
|
539
|
+
</article>`;
|
|
540
|
+
})
|
|
541
|
+
.join("");
|
|
542
|
+
};
|
|
543
|
+
|
|
544
|
+
const candidateLandscape = (evaluations, baselineId, targetId) => {
|
|
545
|
+
const rows = evaluations
|
|
546
|
+
.slice()
|
|
547
|
+
.sort((left, right) =>
|
|
548
|
+
`${left.profileId}/${left.split}/${left.candidateId}`.localeCompare(
|
|
549
|
+
`${right.profileId}/${right.split}/${right.candidateId}`,
|
|
550
|
+
),
|
|
551
|
+
)
|
|
552
|
+
.map((evaluation) => {
|
|
553
|
+
let role = "peer";
|
|
554
|
+
let tone = "tone-neutral";
|
|
555
|
+
if (evaluation.candidateId === baselineId) {
|
|
556
|
+
role = "baseline";
|
|
557
|
+
tone = "tone-before";
|
|
558
|
+
} else if (evaluation.candidateId === targetId) {
|
|
559
|
+
role = "target";
|
|
560
|
+
tone = "tone-after";
|
|
561
|
+
}
|
|
562
|
+
return `<div class="landscape-row ${role}">
|
|
563
|
+
<span><strong>${escapeHtml(evaluation.candidateId)}</strong><small>${escapeHtml(
|
|
564
|
+
`${evaluation.profileId} / ${evaluation.split}`,
|
|
565
|
+
)}</small></span>
|
|
566
|
+
${quantizedBar(
|
|
567
|
+
evaluation.passRate,
|
|
568
|
+
tone,
|
|
569
|
+
`${evaluation.candidateId} ${evaluation.profileId} ${evaluation.split} pass rate ${percentage(
|
|
570
|
+
evaluation.passRate,
|
|
571
|
+
)}`,
|
|
572
|
+
)}
|
|
573
|
+
<b>${escapeHtml(percentage(evaluation.passRate))}</b>
|
|
574
|
+
</div>`;
|
|
575
|
+
})
|
|
576
|
+
.join("");
|
|
577
|
+
return `<figure class="candidate-landscape">
|
|
578
|
+
<figcaption><span>Candidate pass-rate landscape</span><small>shared 0–100% scale</small></figcaption>
|
|
579
|
+
<div class="landscape-axis"><span>0</span><span>50</span><span>100%</span></div>
|
|
580
|
+
${rows || '<div class="artifact"><span>No completed candidate evaluations</span><small>empty</small></div>'}
|
|
581
|
+
</figure>`;
|
|
582
|
+
};
|
|
583
|
+
|
|
584
|
+
const budgetUtilization = (decision) => {
|
|
585
|
+
if (!decision) return "";
|
|
586
|
+
const entries = [
|
|
587
|
+
["Candidates", decision.spend.candidates, decision.budget.maxCandidates, "count"],
|
|
588
|
+
["Metric calls", decision.spend.metricCalls, decision.budget.maxMetricCalls, "count"],
|
|
589
|
+
["Judge calls", decision.spend.judgeCalls, decision.budget.maxJudgeCalls, "count"],
|
|
590
|
+
["Proposal calls", decision.spend.proposalCalls, decision.budget.maxProposalCalls, "count"],
|
|
591
|
+
["Tokens", decision.spend.totalTokens, decision.budget.maxTokens, "compact"],
|
|
592
|
+
["Cost", decision.spend.costUsd, decision.budget.maxCostUsd, "currency"],
|
|
593
|
+
["Active time", decision.spend.activeTimeMs, decision.budget.maxWallTimeMs, "duration"],
|
|
594
|
+
];
|
|
595
|
+
const formatValue = (value, kind) => {
|
|
596
|
+
if (kind === "compact") return compactNumber(value);
|
|
597
|
+
if (kind === "currency") return `$${finite(value)}`;
|
|
598
|
+
if (kind === "duration") return duration(value);
|
|
599
|
+
return finite(value, 0);
|
|
600
|
+
};
|
|
601
|
+
return `<figure class="budget-visual">
|
|
602
|
+
<figcaption><span>Budget utilization</span><small>observed / ceiling</small></figcaption>
|
|
603
|
+
${entries
|
|
604
|
+
.map(([label, value, limit, kind]) => {
|
|
605
|
+
const ratio = limit > 0 ? value / limit : 0;
|
|
606
|
+
return `<div class="budget-row">
|
|
607
|
+
<span>${escapeHtml(label)}</span>
|
|
608
|
+
${quantizedBar(
|
|
609
|
+
ratio,
|
|
610
|
+
ratio > 0.9 ? "tone-negative" : "tone-neutral",
|
|
611
|
+
`${label} ${formatValue(value, kind)} of ${formatValue(limit, kind)}`,
|
|
612
|
+
)}
|
|
613
|
+
<b>${escapeHtml(`${formatValue(value, kind)} / ${formatValue(limit, kind)}`)}</b>
|
|
614
|
+
</div>`;
|
|
615
|
+
})
|
|
616
|
+
.join("")}
|
|
617
|
+
</figure>`;
|
|
618
|
+
};
|
|
619
|
+
|
|
620
|
+
const failedEvidence = (evaluations) =>
|
|
621
|
+
evaluations
|
|
622
|
+
.flatMap((evaluation) =>
|
|
623
|
+
evaluation.failedCases.map((failure) => ({
|
|
624
|
+
...failure,
|
|
625
|
+
scope: evaluationKey(evaluation),
|
|
626
|
+
})),
|
|
627
|
+
)
|
|
628
|
+
.map(
|
|
629
|
+
(failure) => `<article class="failure-card">
|
|
630
|
+
<header><strong>${escapeHtml(failure.caseId)}</strong><small>${escapeHtml(
|
|
631
|
+
failure.scope,
|
|
632
|
+
)} · ${escapeHtml(percentage(failure.score))}</small></header>
|
|
633
|
+
${
|
|
634
|
+
failure.feedback.length
|
|
635
|
+
? `<ul>${failure.feedback.map((entry) => `<li>${escapeHtml(entry)}</li>`).join("")}</ul>`
|
|
636
|
+
: "<p>No evaluator feedback recorded.</p>"
|
|
637
|
+
}
|
|
638
|
+
</article>`,
|
|
639
|
+
)
|
|
640
|
+
.join("");
|
|
641
|
+
|
|
642
|
+
export const experimentTargetId = (payload, requested) => {
|
|
643
|
+
const baseline = baselineCandidate(payload);
|
|
644
|
+
const ids = new Set(payload.experiment.evaluations.map((entry) => entry.candidateId));
|
|
645
|
+
if (requested && ids.has(requested)) return requested;
|
|
646
|
+
const champion = payload.experiment.championCandidateId;
|
|
647
|
+
if (champion && ids.has(champion)) return champion;
|
|
648
|
+
return [...ids].find((id) => id !== baseline?.id) ?? [...ids][0] ?? null;
|
|
649
|
+
};
|
|
650
|
+
|
|
651
|
+
export const renderExperimentDetail = (item, payload, requestedTargetId) => {
|
|
652
|
+
const experiment = payload.experiment;
|
|
653
|
+
const decision = payload.decision;
|
|
654
|
+
const baseline = baselineCandidate(payload);
|
|
655
|
+
const targetId = experimentTargetId(payload, requestedTargetId);
|
|
656
|
+
const target = targetId ? candidateById(payload, targetId) : null;
|
|
657
|
+
const baselineEvaluations = experiment.evaluations.filter(
|
|
658
|
+
(evaluation) => evaluation.candidateId === baseline?.id,
|
|
659
|
+
);
|
|
660
|
+
const targetEvaluations = experiment.evaluations.filter(
|
|
661
|
+
(evaluation) => evaluation.candidateId === targetId,
|
|
662
|
+
);
|
|
663
|
+
const options = [...new Set(experiment.evaluations.map((entry) => entry.candidateId))]
|
|
664
|
+
.filter((id) => id !== baseline?.id || id === targetId)
|
|
665
|
+
.map(
|
|
666
|
+
(id) =>
|
|
667
|
+
`<option value="${escapeHtml(id)}"${id === targetId ? " selected" : ""}>${escapeHtml(
|
|
668
|
+
id,
|
|
669
|
+
)}</option>`,
|
|
670
|
+
)
|
|
671
|
+
.join("");
|
|
672
|
+
const gates = (decision?.gates ?? [])
|
|
673
|
+
.map(
|
|
674
|
+
(gate) => `<article class="gate ${gate.passed ? "passed" : "failed"}">
|
|
675
|
+
<span>${gate.passed ? "✓" : "×"}</span>
|
|
676
|
+
<div><strong>${escapeHtml(gate.metric)}</strong><small>${escapeHtml(
|
|
677
|
+
[gate.profile, gate.split, gate.interval, gate.note].filter(Boolean).join(" · "),
|
|
678
|
+
)}</small></div>
|
|
679
|
+
<b>${gate.values.map((value) => finite(value)).join(", ")}</b>
|
|
680
|
+
</article>`,
|
|
681
|
+
)
|
|
682
|
+
.join("");
|
|
683
|
+
const lineage = target
|
|
684
|
+
? `<article class="lineage-card">
|
|
685
|
+
<span>${escapeHtml(target.id)}</span>
|
|
686
|
+
<small>from ${escapeHtml(target.parentIds.join(" + ") || "root")} · ${escapeHtml(
|
|
687
|
+
target.proposer,
|
|
688
|
+
)} · ${compactNumber(target.sourceBytes)} bytes</small>
|
|
689
|
+
${target.rationale ? `<p>${escapeHtml(target.rationale)}</p>` : ""}
|
|
690
|
+
<div>${target.components
|
|
691
|
+
.map((component) => `<code>${escapeHtml(component)}</code>`)
|
|
692
|
+
.join("")}</div>
|
|
693
|
+
</article>`
|
|
694
|
+
: "";
|
|
695
|
+
return `
|
|
696
|
+
<header class="detail-header">
|
|
697
|
+
<div class="detail-kicker">${badge(item.status)}<span class="detail-path">${escapeHtml(
|
|
698
|
+
item.path,
|
|
699
|
+
)}</span></div>
|
|
700
|
+
<div class="detail-title-row">
|
|
701
|
+
<div><h2>${escapeHtml(item.campaignId)}</h2><p class="detail-subtitle">${escapeHtml(
|
|
702
|
+
`${item.mode} · ${fullDate(item.startedAt)} · ${item.experimentId}`,
|
|
703
|
+
)}</p></div>
|
|
704
|
+
<div class="score-block"><strong>${escapeHtml(
|
|
705
|
+
decision?.outcome ? statusLabel(decision.outcome) : item.candidateCount,
|
|
706
|
+
)}</strong><small>${decision?.outcome ? "verified decision" : "candidates"}</small></div>
|
|
707
|
+
</div>
|
|
708
|
+
</header>
|
|
709
|
+
<div class="detail-body">
|
|
710
|
+
${
|
|
711
|
+
payload.integrity.verified
|
|
712
|
+
? '<div class="integrity-banner">✓ Candidate ledger and accounting verified</div>'
|
|
713
|
+
: `<div class="attention-banner">Integrity unavailable: ${escapeHtml(
|
|
714
|
+
payload.integrity.errors.join(" · "),
|
|
715
|
+
)}</div>`
|
|
716
|
+
}
|
|
717
|
+
<dl class="detail-stats">
|
|
718
|
+
<div><dt>Champion</dt><dd>${escapeHtml(experiment.championCandidateId ?? "—")}</dd></div>
|
|
719
|
+
<div><dt>Active time</dt><dd>${escapeHtml(duration(experiment.activeTimeMs))}</dd></div>
|
|
720
|
+
<div><dt>Judge calls</dt><dd>${experiment.budgetUsage.judgeCalls}</dd></div>
|
|
721
|
+
<div><dt>Spend</dt><dd>$${experiment.budgetUsage.costUsd.toFixed(2)}</dd></div>
|
|
722
|
+
</dl>
|
|
723
|
+
<div class="experiment-visuals">
|
|
724
|
+
${candidateLandscape(experiment.evaluations, baseline?.id, targetId)}
|
|
725
|
+
${budgetUtilization(decision)}
|
|
726
|
+
</div>
|
|
727
|
+
${
|
|
728
|
+
decision
|
|
729
|
+
? `<section class="detail-section">
|
|
730
|
+
<h3>Promotion gates <span>${decision.gates.filter((gate) => gate.passed).length}/${decision.gates.length} passed</span></h3>
|
|
731
|
+
<div class="gate-list">${gates || '<div class="artifact"><span>No completed gates</span><small>incomplete</small></div>'}</div>
|
|
732
|
+
<div class="decision-meta">
|
|
733
|
+
<span>Ranking</span><strong>${escapeHtml(decision.ranking.join(" → ") || "—")}</strong>
|
|
734
|
+
<span>Eligible</span><strong>${escapeHtml(
|
|
735
|
+
decision.eligibleCandidateIds.join(", ") || "none",
|
|
736
|
+
)}</strong>
|
|
737
|
+
</div>
|
|
738
|
+
</section>`
|
|
739
|
+
: ""
|
|
740
|
+
}
|
|
741
|
+
<section class="detail-section">
|
|
742
|
+
<div class="section-heading">
|
|
743
|
+
<h3>Candidate comparison <span>baseline → target</span></h3>
|
|
744
|
+
${
|
|
745
|
+
options
|
|
746
|
+
? `<label class="candidate-picker">Target <select data-candidate-target>${options}</select></label>`
|
|
747
|
+
: ""
|
|
748
|
+
}
|
|
749
|
+
</div>
|
|
750
|
+
${lineage}
|
|
751
|
+
<div class="comparison-scopes">
|
|
752
|
+
${experimentScopes(baselineEvaluations, targetEvaluations) || '<div class="artifact"><span>No comparable evaluations</span><small>empty</small></div>'}
|
|
753
|
+
</div>
|
|
754
|
+
</section>
|
|
755
|
+
${
|
|
756
|
+
failedEvidence(targetEvaluations)
|
|
757
|
+
? `<section class="detail-section"><h3>Failed evidence <span>judge feedback</span></h3><div class="failure-list">${failedEvidence(
|
|
758
|
+
targetEvaluations,
|
|
759
|
+
)}</div></section>`
|
|
760
|
+
: ""
|
|
761
|
+
}
|
|
762
|
+
</div>`;
|
|
763
|
+
};
|
|
764
|
+
|
|
765
|
+
const detailAssertionMap = (detail) =>
|
|
766
|
+
new Map(
|
|
767
|
+
(detail.grading?.assertions ?? []).map((assertion) => [
|
|
768
|
+
assertion.id ?? assertion.name,
|
|
769
|
+
assertion,
|
|
770
|
+
]),
|
|
771
|
+
);
|
|
772
|
+
|
|
773
|
+
const assertionDeltaRows = (payload) => {
|
|
774
|
+
const before = detailAssertionMap(payload.before);
|
|
775
|
+
const after = detailAssertionMap(payload.after);
|
|
776
|
+
return payload.comparison.grading.deltas
|
|
777
|
+
.map((delta) => {
|
|
778
|
+
const beforeAssertion = before.get(delta.id);
|
|
779
|
+
const afterAssertion = after.get(delta.id);
|
|
780
|
+
return `<tr>
|
|
781
|
+
<td><strong>${escapeHtml(delta.name)}</strong><small>${escapeHtml(delta.type)}</small></td>
|
|
782
|
+
<td>${escapeHtml(percentage(delta.before?.score))}</td>
|
|
783
|
+
<td>${escapeHtml(percentage(delta.after?.score))}</td>
|
|
784
|
+
<td><span class="delta-badge ${escapeHtml(delta.status)}">${escapeHtml(
|
|
785
|
+
delta.status,
|
|
786
|
+
)}</span></td>
|
|
787
|
+
<td class="rationale-cell">
|
|
788
|
+
<span>${escapeHtml(beforeAssertion?.comment ?? "—")}</span>
|
|
789
|
+
<b>→</b>
|
|
790
|
+
<span>${escapeHtml(afterAssertion?.comment ?? "—")}</span>
|
|
791
|
+
</td>
|
|
792
|
+
</tr>`;
|
|
793
|
+
})
|
|
794
|
+
.join("");
|
|
795
|
+
};
|
|
796
|
+
|
|
797
|
+
const artifactDelta = (payload) => {
|
|
798
|
+
const groups = [
|
|
799
|
+
["changed", payload.comparison.artifactsDelta.changed],
|
|
800
|
+
["added", payload.comparison.artifactsDelta.added],
|
|
801
|
+
["removed", payload.comparison.artifactsDelta.removed],
|
|
802
|
+
];
|
|
803
|
+
return groups
|
|
804
|
+
.flatMap(([status, paths]) =>
|
|
805
|
+
paths.map((path) => {
|
|
806
|
+
const clickable = status === "changed" || status === "added" || status === "removed";
|
|
807
|
+
const content = `<span>${escapeHtml(path)}</span><small>${escapeHtml(status)}${
|
|
808
|
+
clickable ? " →" : ""
|
|
809
|
+
}</small>`;
|
|
810
|
+
return clickable
|
|
811
|
+
? `<button class="artifact artifact-button ${escapeHtml(
|
|
812
|
+
status,
|
|
813
|
+
)}" type="button" data-compare-artifact="${escapeHtml(path)}">${content}</button>`
|
|
814
|
+
: `<div class="artifact">${content}</div>`;
|
|
815
|
+
}),
|
|
816
|
+
)
|
|
817
|
+
.join("");
|
|
818
|
+
};
|
|
819
|
+
|
|
820
|
+
const signedMetricValue = (value, unit) => {
|
|
821
|
+
if (typeof value !== "number") return "—";
|
|
822
|
+
if (unit === "milliseconds") return signedDuration(value);
|
|
823
|
+
if (unit === "ratio" || unit === "score") return signedPercentage(value);
|
|
824
|
+
return `${value > 0 ? "+" : ""}${metricValue(value, unit)}`;
|
|
825
|
+
};
|
|
826
|
+
|
|
827
|
+
const driverMetricComparisons = (runDelta) => {
|
|
828
|
+
const before = runDelta?.driverMetrics?.a ?? {};
|
|
829
|
+
const after = runDelta?.driverMetrics?.b ?? {};
|
|
830
|
+
const names = [...new Set([...Object.keys(before), ...Object.keys(after)])].sort();
|
|
831
|
+
return names
|
|
832
|
+
.map((name) => {
|
|
833
|
+
const a = before[name];
|
|
834
|
+
const b = after[name];
|
|
835
|
+
const unit = a?.unit ?? b?.unit ?? "number";
|
|
836
|
+
const comparable = a && b && a.unit === b.unit;
|
|
837
|
+
const delta = comparable ? b.value - a.value : null;
|
|
838
|
+
const better = b?.better ?? a?.better ?? "neutral";
|
|
839
|
+
const tone =
|
|
840
|
+
better === "neutral" ? "metric-neutral" : toneForDelta(delta, better === "lower");
|
|
841
|
+
return comparisonBars({
|
|
842
|
+
title: metricName(name, b?.label ?? a?.label),
|
|
843
|
+
before: a?.value,
|
|
844
|
+
after: b?.value,
|
|
845
|
+
beforeText: metricValue(a?.value, unit),
|
|
846
|
+
afterText: metricValue(b?.value, unit),
|
|
847
|
+
deltaText: comparable ? signedMetricValue(delta, unit) : "not paired",
|
|
848
|
+
deltaTone: tone,
|
|
849
|
+
});
|
|
850
|
+
})
|
|
851
|
+
.join("");
|
|
852
|
+
};
|
|
853
|
+
|
|
854
|
+
const subagentTotals = (activities) => {
|
|
855
|
+
const subagents = (activities ?? []).filter((activity) => activity.agent?.role === "subagent");
|
|
856
|
+
return {
|
|
857
|
+
count: subagents.length,
|
|
858
|
+
toolCalls: subagents.reduce((sum, activity) => sum + (activity.toolCalls?.total ?? 0), 0),
|
|
859
|
+
tokens: subagents.reduce((sum, activity) => sum + (activity.usage?.total ?? 0), 0),
|
|
860
|
+
durationMs: subagents.reduce((sum, activity) => sum + (activity.durationMs ?? 0), 0),
|
|
861
|
+
};
|
|
862
|
+
};
|
|
863
|
+
|
|
864
|
+
export const renderRunComparison = (payload, labels) => {
|
|
865
|
+
const comparison = payload.comparison;
|
|
866
|
+
const runDelta = comparison.runDelta;
|
|
867
|
+
const durationDelta =
|
|
868
|
+
typeof runDelta?.durationMsA === "number" && typeof runDelta?.durationMsB === "number"
|
|
869
|
+
? runDelta.durationMsB - runDelta.durationMsA
|
|
870
|
+
: null;
|
|
871
|
+
const mutationA = runDelta?.actionTiming?.a?.timeToFirstMutationMs;
|
|
872
|
+
const mutationB = runDelta?.actionTiming?.b?.timeToFirstMutationMs;
|
|
873
|
+
const mutationDelta =
|
|
874
|
+
typeof mutationA === "number" && typeof mutationB === "number" ? mutationB - mutationA : null;
|
|
875
|
+
const tokenA = totalTokens(runDelta?.usage?.a?.agent);
|
|
876
|
+
const tokenB = totalTokens(runDelta?.usage?.b?.agent);
|
|
877
|
+
const tokenDelta = tokenB - tokenA;
|
|
878
|
+
const scoreDelta =
|
|
879
|
+
typeof comparison.grading.caseScoreA === "number" &&
|
|
880
|
+
typeof comparison.grading.caseScoreB === "number"
|
|
881
|
+
? comparison.grading.caseScoreB - comparison.grading.caseScoreA
|
|
882
|
+
: null;
|
|
883
|
+
const driverMetrics = driverMetricComparisons(runDelta);
|
|
884
|
+
const subagentsA = subagentTotals(runDelta?.agentActivity?.a);
|
|
885
|
+
const subagentsB = subagentTotals(runDelta?.agentActivity?.b);
|
|
886
|
+
return `
|
|
887
|
+
<header class="detail-header comparison-header">
|
|
888
|
+
<div class="detail-kicker">${badge(
|
|
889
|
+
comparison.grading.regressions ? "failed" : "passed",
|
|
890
|
+
)}<span>run comparison</span></div>
|
|
891
|
+
<div class="detail-title-row">
|
|
892
|
+
<div><h2>${escapeHtml(labels.before)} → ${escapeHtml(
|
|
893
|
+
labels.after,
|
|
894
|
+
)}</h2><p class="detail-subtitle">Baseline A compared with candidate B</p></div>
|
|
895
|
+
<button class="secondary-button" type="button" data-close-compare>Done</button>
|
|
896
|
+
</div>
|
|
897
|
+
</header>
|
|
898
|
+
<div class="detail-body">
|
|
899
|
+
${
|
|
900
|
+
comparison.caseMismatch.length
|
|
901
|
+
? `<div class="attention-banner">Evidence identity differs: ${escapeHtml(
|
|
902
|
+
comparison.caseMismatch.map((entry) => entry.field).join(", "),
|
|
903
|
+
)}. Treat deltas as potentially incomparable.</div>`
|
|
904
|
+
: '<div class="integrity-banner">✓ Case, evaluator, runtime, and driver identities are comparable</div>'
|
|
905
|
+
}
|
|
906
|
+
<dl class="comparison-stats">
|
|
907
|
+
<div><dt>Case score</dt><dd>${escapeHtml(
|
|
908
|
+
`${percentage(comparison.grading.caseScoreA)} → ${percentage(
|
|
909
|
+
comparison.grading.caseScoreB,
|
|
910
|
+
)}`,
|
|
911
|
+
)}</dd><small class="${toneForDelta(scoreDelta)}">${escapeHtml(
|
|
912
|
+
signedPercentage(scoreDelta),
|
|
913
|
+
)}</small></div>
|
|
914
|
+
<div><dt>Duration</dt><dd>${escapeHtml(
|
|
915
|
+
`${duration(runDelta?.durationMsA)} → ${duration(runDelta?.durationMsB)}`,
|
|
916
|
+
)}</dd><small class="${toneForDelta(durationDelta, true)}">${escapeHtml(
|
|
917
|
+
signedDuration(durationDelta),
|
|
918
|
+
)}</small></div>
|
|
919
|
+
<div><dt>First mutation</dt><dd>${escapeHtml(
|
|
920
|
+
`${duration(mutationA)} → ${duration(mutationB)}`,
|
|
921
|
+
)}</dd><small class="${toneForDelta(mutationDelta, true)}">${escapeHtml(
|
|
922
|
+
signedDuration(mutationDelta),
|
|
923
|
+
)}</small></div>
|
|
924
|
+
<div><dt>Agent tokens</dt><dd>${escapeHtml(
|
|
925
|
+
`${compactNumber(tokenA)} → ${compactNumber(tokenB)}`,
|
|
926
|
+
)}</dd><small class="${toneForDelta(tokenDelta, true)}">${tokenDelta > 0 ? "+" : ""}${escapeHtml(
|
|
927
|
+
compactNumber(tokenDelta),
|
|
928
|
+
)}</small></div>
|
|
929
|
+
<div><dt>Assertions</dt><dd>${comparison.grading.improvements} ↑ / ${comparison.grading.regressions} ↓</dd><small>${comparison.grading.deltas.length} compared</small></div>
|
|
930
|
+
</dl>
|
|
931
|
+
<section class="detail-section">
|
|
932
|
+
<h3>Comparison profile <span>shared scales within each measure</span></h3>
|
|
933
|
+
<div class="comparison-plot-grid">
|
|
934
|
+
${comparisonBars({
|
|
935
|
+
title: "Case score",
|
|
936
|
+
before: comparison.grading.caseScoreA,
|
|
937
|
+
after: comparison.grading.caseScoreB,
|
|
938
|
+
maximum: 1,
|
|
939
|
+
beforeText: percentage(comparison.grading.caseScoreA),
|
|
940
|
+
afterText: percentage(comparison.grading.caseScoreB),
|
|
941
|
+
deltaText: signedPercentage(scoreDelta),
|
|
942
|
+
deltaTone: toneForDelta(scoreDelta),
|
|
943
|
+
})}
|
|
944
|
+
${comparisonBars({
|
|
945
|
+
title: "Duration",
|
|
946
|
+
before: runDelta?.durationMsA,
|
|
947
|
+
after: runDelta?.durationMsB,
|
|
948
|
+
beforeText: duration(runDelta?.durationMsA),
|
|
949
|
+
afterText: duration(runDelta?.durationMsB),
|
|
950
|
+
deltaText: signedDuration(durationDelta),
|
|
951
|
+
deltaTone: toneForDelta(durationDelta, true),
|
|
952
|
+
})}
|
|
953
|
+
${comparisonBars({
|
|
954
|
+
title: "Time to first mutation",
|
|
955
|
+
before: mutationA,
|
|
956
|
+
after: mutationB,
|
|
957
|
+
beforeText: duration(mutationA),
|
|
958
|
+
afterText: duration(mutationB),
|
|
959
|
+
deltaText: signedDuration(mutationDelta),
|
|
960
|
+
deltaTone: toneForDelta(mutationDelta, true),
|
|
961
|
+
})}
|
|
962
|
+
${comparisonBars({
|
|
963
|
+
title: "Agent tokens",
|
|
964
|
+
before: tokenA,
|
|
965
|
+
after: tokenB,
|
|
966
|
+
beforeText: compactNumber(tokenA),
|
|
967
|
+
afterText: compactNumber(tokenB),
|
|
968
|
+
deltaText: `${tokenDelta > 0 ? "+" : ""}${compactNumber(tokenDelta)}`,
|
|
969
|
+
deltaTone: toneForDelta(tokenDelta, true),
|
|
970
|
+
})}
|
|
971
|
+
${comparisonBars({
|
|
972
|
+
title: "Subagents spawned",
|
|
973
|
+
before: subagentsA.count,
|
|
974
|
+
after: subagentsB.count,
|
|
975
|
+
beforeText: compactNumber(subagentsA.count),
|
|
976
|
+
afterText: compactNumber(subagentsB.count),
|
|
977
|
+
deltaText: `${subagentsB.count - subagentsA.count > 0 ? "+" : ""}${compactNumber(
|
|
978
|
+
subagentsB.count - subagentsA.count,
|
|
979
|
+
)}`,
|
|
980
|
+
deltaTone: "metric-neutral",
|
|
981
|
+
})}
|
|
982
|
+
${comparisonBars({
|
|
983
|
+
title: "Subagent tool calls",
|
|
984
|
+
before: subagentsA.toolCalls,
|
|
985
|
+
after: subagentsB.toolCalls,
|
|
986
|
+
beforeText: compactNumber(subagentsA.toolCalls),
|
|
987
|
+
afterText: compactNumber(subagentsB.toolCalls),
|
|
988
|
+
deltaText: `${subagentsB.toolCalls - subagentsA.toolCalls > 0 ? "+" : ""}${compactNumber(
|
|
989
|
+
subagentsB.toolCalls - subagentsA.toolCalls,
|
|
990
|
+
)}`,
|
|
991
|
+
deltaTone: "metric-neutral",
|
|
992
|
+
})}
|
|
993
|
+
${comparisonBars({
|
|
994
|
+
title: "Total subagent time",
|
|
995
|
+
before: subagentsA.durationMs,
|
|
996
|
+
after: subagentsB.durationMs,
|
|
997
|
+
beforeText: duration(subagentsA.durationMs),
|
|
998
|
+
afterText: duration(subagentsB.durationMs),
|
|
999
|
+
deltaText: signedDuration(subagentsB.durationMs - subagentsA.durationMs),
|
|
1000
|
+
deltaTone: "metric-neutral",
|
|
1001
|
+
})}
|
|
1002
|
+
${comparisonBars({
|
|
1003
|
+
title: "Subagent tokens",
|
|
1004
|
+
before: subagentsA.tokens,
|
|
1005
|
+
after: subagentsB.tokens,
|
|
1006
|
+
beforeText: compactNumber(subagentsA.tokens),
|
|
1007
|
+
afterText: compactNumber(subagentsB.tokens),
|
|
1008
|
+
deltaText: `${subagentsB.tokens - subagentsA.tokens > 0 ? "+" : ""}${compactNumber(
|
|
1009
|
+
subagentsB.tokens - subagentsA.tokens,
|
|
1010
|
+
)}`,
|
|
1011
|
+
deltaTone: toneForDelta(subagentsB.tokens - subagentsA.tokens, true),
|
|
1012
|
+
})}
|
|
1013
|
+
</div>
|
|
1014
|
+
<div class="comparison-distributions">
|
|
1015
|
+
${compositionFigure(
|
|
1016
|
+
"Assertion movement",
|
|
1017
|
+
[
|
|
1018
|
+
{ key: "improved", label: "Improved", value: comparison.grading.improvements },
|
|
1019
|
+
{ key: "regressed", label: "Regressed", value: comparison.grading.regressions },
|
|
1020
|
+
{
|
|
1021
|
+
key: "added",
|
|
1022
|
+
label: "Added",
|
|
1023
|
+
value: comparison.grading.deltas.filter((delta) => delta.status === "added").length,
|
|
1024
|
+
},
|
|
1025
|
+
{
|
|
1026
|
+
key: "removed",
|
|
1027
|
+
label: "Removed",
|
|
1028
|
+
value: comparison.grading.deltas.filter((delta) => delta.status === "removed")
|
|
1029
|
+
.length,
|
|
1030
|
+
},
|
|
1031
|
+
{
|
|
1032
|
+
key: "unchanged",
|
|
1033
|
+
label: "Unchanged",
|
|
1034
|
+
value: comparison.grading.deltas.filter((delta) => delta.status === "unchanged")
|
|
1035
|
+
.length,
|
|
1036
|
+
},
|
|
1037
|
+
],
|
|
1038
|
+
"Distribution of assertion changes",
|
|
1039
|
+
)}
|
|
1040
|
+
${compositionFigure(
|
|
1041
|
+
"Artifact movement",
|
|
1042
|
+
[
|
|
1043
|
+
{
|
|
1044
|
+
key: "improved",
|
|
1045
|
+
label: "Added",
|
|
1046
|
+
value: comparison.artifactsDelta.added.length,
|
|
1047
|
+
},
|
|
1048
|
+
{
|
|
1049
|
+
key: "regressed",
|
|
1050
|
+
label: "Removed",
|
|
1051
|
+
value: comparison.artifactsDelta.removed.length,
|
|
1052
|
+
},
|
|
1053
|
+
{
|
|
1054
|
+
key: "changed",
|
|
1055
|
+
label: "Changed",
|
|
1056
|
+
value: comparison.artifactsDelta.changed.length,
|
|
1057
|
+
},
|
|
1058
|
+
{
|
|
1059
|
+
key: "unchanged",
|
|
1060
|
+
label: "Unchanged",
|
|
1061
|
+
value: comparison.artifactsDelta.unchanged.length,
|
|
1062
|
+
},
|
|
1063
|
+
],
|
|
1064
|
+
"Distribution of artifact changes",
|
|
1065
|
+
)}
|
|
1066
|
+
</div>
|
|
1067
|
+
</section>
|
|
1068
|
+
${
|
|
1069
|
+
driverMetrics
|
|
1070
|
+
? `<section class="detail-section"><h3>Driver metric comparison <span>shared scale per metric</span></h3><div class="comparison-plot-grid">${driverMetrics}</div></section>`
|
|
1071
|
+
: ""
|
|
1072
|
+
}
|
|
1073
|
+
<section class="detail-section">
|
|
1074
|
+
<h3>Judgment delta <span>rationale side by side</span></h3>
|
|
1075
|
+
<div class="matrix-wrap"><table class="matrix comparison-matrix">
|
|
1076
|
+
<thead><tr><th>Assertion</th><th>A</th><th>B</th><th>Change</th><th>Judge rationale A → B</th></tr></thead>
|
|
1077
|
+
<tbody>${assertionDeltaRows(payload) || '<tr><td colspan="5">No assertions.</td></tr>'}</tbody>
|
|
1078
|
+
</table></div>
|
|
1079
|
+
</section>
|
|
1080
|
+
<section class="detail-section">
|
|
1081
|
+
<h3>Artifacts <span>${comparison.artifactsDelta.changed.length} changed · ${comparison.artifactsDelta.added.length} added · ${comparison.artifactsDelta.removed.length} removed</span></h3>
|
|
1082
|
+
<div class="artifact-list">${artifactDelta(payload) || '<div class="artifact"><span>No artifact changes</span><small>unchanged</small></div>'}</div>
|
|
1083
|
+
</section>
|
|
1084
|
+
</div>`;
|
|
1085
|
+
};
|
|
1086
|
+
|
|
1087
|
+
const metricDeltaSummary = (metrics) =>
|
|
1088
|
+
Object.entries(metrics)
|
|
1089
|
+
.map(
|
|
1090
|
+
([name, summary]) =>
|
|
1091
|
+
`<span><b>${escapeHtml(name)}</b> ${escapeHtml(
|
|
1092
|
+
metricValue(summary.mean, inferredMetricUnit(name, summary.unit)),
|
|
1093
|
+
)}</span>`,
|
|
1094
|
+
)
|
|
1095
|
+
.join("");
|
|
1096
|
+
|
|
1097
|
+
export const renderSuiteComparison = (payload, labels) => {
|
|
1098
|
+
const comparison = payload.comparison;
|
|
1099
|
+
const movement = { improved: 0, regressed: 0, unchanged: 0, missing: 0 };
|
|
1100
|
+
for (const entry of comparison.cases) {
|
|
1101
|
+
if (!entry.before || !entry.after) movement.missing++;
|
|
1102
|
+
else if (entry.regressed) movement.regressed++;
|
|
1103
|
+
else if ((entry.passRateDelta ?? 0) > 0) movement.improved++;
|
|
1104
|
+
else movement.unchanged++;
|
|
1105
|
+
}
|
|
1106
|
+
return `
|
|
1107
|
+
<header class="detail-header comparison-header">
|
|
1108
|
+
<div class="detail-kicker">${badge(
|
|
1109
|
+
comparison.regressions || comparison.identityMismatches.length ? "failed" : "passed",
|
|
1110
|
+
)}<span>suite comparison</span></div>
|
|
1111
|
+
<div class="detail-title-row">
|
|
1112
|
+
<div><h2>${escapeHtml(labels.before)} → ${escapeHtml(
|
|
1113
|
+
labels.after,
|
|
1114
|
+
)}</h2><p class="detail-subtitle">Repeated evidence with paired case metrics</p></div>
|
|
1115
|
+
<button class="secondary-button" type="button" data-close-compare>Done</button>
|
|
1116
|
+
</div>
|
|
1117
|
+
</header>
|
|
1118
|
+
<div class="detail-body">
|
|
1119
|
+
${
|
|
1120
|
+
comparison.identityMismatches.length
|
|
1121
|
+
? `<div class="attention-banner">${comparison.identityMismatches.length} repetition identities differ; those pairs are not directly comparable.</div>`
|
|
1122
|
+
: '<div class="integrity-banner">✓ All paired repetition identities match</div>'
|
|
1123
|
+
}
|
|
1124
|
+
<dl class="comparison-stats">
|
|
1125
|
+
<div><dt>Pass rate</dt><dd>${escapeHtml(
|
|
1126
|
+
`${percentage(comparison.before.passRate)} → ${percentage(comparison.after.passRate)}`,
|
|
1127
|
+
)}</dd><small class="${toneForDelta(comparison.passRateDelta)}">${escapeHtml(
|
|
1128
|
+
signedPercentage(comparison.passRateDelta),
|
|
1129
|
+
)}</small></div>
|
|
1130
|
+
<div><dt>Attempts</dt><dd>${comparison.before.attempts} → ${comparison.after.attempts}</dd><small>paired populations</small></div>
|
|
1131
|
+
<div><dt>Regressions</dt><dd>${comparison.regressions}</dd><small>case-level</small></div>
|
|
1132
|
+
<div><dt>Identity drift</dt><dd>${comparison.identityMismatches.length}</dd><small>repetitions</small></div>
|
|
1133
|
+
</dl>
|
|
1134
|
+
<section class="detail-section">
|
|
1135
|
+
<h3>Outcome movement <span>shared 0–100% scale</span></h3>
|
|
1136
|
+
<div class="suite-outcome-visual">
|
|
1137
|
+
${comparisonBars({
|
|
1138
|
+
title: "Suite pass rate",
|
|
1139
|
+
before: comparison.before.passRate,
|
|
1140
|
+
after: comparison.after.passRate,
|
|
1141
|
+
maximum: 1,
|
|
1142
|
+
beforeText: percentage(comparison.before.passRate),
|
|
1143
|
+
afterText: percentage(comparison.after.passRate),
|
|
1144
|
+
deltaText: signedPercentage(comparison.passRateDelta),
|
|
1145
|
+
deltaTone: toneForDelta(comparison.passRateDelta),
|
|
1146
|
+
})}
|
|
1147
|
+
${compositionFigure(
|
|
1148
|
+
"Case movement",
|
|
1149
|
+
[
|
|
1150
|
+
{
|
|
1151
|
+
key: "improved",
|
|
1152
|
+
label: "Improved",
|
|
1153
|
+
value: movement.improved,
|
|
1154
|
+
},
|
|
1155
|
+
{
|
|
1156
|
+
key: "regressed",
|
|
1157
|
+
label: "Regressed",
|
|
1158
|
+
value: movement.regressed,
|
|
1159
|
+
},
|
|
1160
|
+
{
|
|
1161
|
+
key: "unchanged",
|
|
1162
|
+
label: "Unchanged",
|
|
1163
|
+
value: movement.unchanged,
|
|
1164
|
+
},
|
|
1165
|
+
{
|
|
1166
|
+
key: "pending",
|
|
1167
|
+
label: "Missing side",
|
|
1168
|
+
value: movement.missing,
|
|
1169
|
+
},
|
|
1170
|
+
],
|
|
1171
|
+
"Distribution of per-case suite movement",
|
|
1172
|
+
)}
|
|
1173
|
+
</div>
|
|
1174
|
+
</section>
|
|
1175
|
+
<section class="detail-section">
|
|
1176
|
+
<h3>Per-case delta <span>after − before</span></h3>
|
|
1177
|
+
<div class="matrix-wrap"><table class="matrix comparison-matrix">
|
|
1178
|
+
<thead><tr><th>Case</th><th>Before</th><th>After</th><th>Δ</th><th>Paired metric means</th></tr></thead>
|
|
1179
|
+
<tbody>${comparison.cases
|
|
1180
|
+
.map(
|
|
1181
|
+
(entry) => `<tr class="${entry.regressed ? "row-regressed" : ""}">
|
|
1182
|
+
<td>${escapeHtml(entry.canonicalCaseId)}</td>
|
|
1183
|
+
<td class="bar-table-cell">${quantizedBar(
|
|
1184
|
+
entry.before?.passRate,
|
|
1185
|
+
"tone-before",
|
|
1186
|
+
`${entry.canonicalCaseId} baseline pass rate ${percentage(
|
|
1187
|
+
entry.before?.passRate,
|
|
1188
|
+
)}`,
|
|
1189
|
+
)}<small>${escapeHtml(percentage(entry.before?.passRate))} · n=${
|
|
1190
|
+
entry.before?.attempts ?? 0
|
|
1191
|
+
}</small></td>
|
|
1192
|
+
<td class="bar-table-cell">${quantizedBar(
|
|
1193
|
+
entry.after?.passRate,
|
|
1194
|
+
"tone-after",
|
|
1195
|
+
`${entry.canonicalCaseId} candidate pass rate ${percentage(
|
|
1196
|
+
entry.after?.passRate,
|
|
1197
|
+
)}`,
|
|
1198
|
+
)}<small>${escapeHtml(percentage(entry.after?.passRate))} · n=${
|
|
1199
|
+
entry.after?.attempts ?? 0
|
|
1200
|
+
}</small></td>
|
|
1201
|
+
<td class="${toneForDelta(entry.passRateDelta)}">${escapeHtml(
|
|
1202
|
+
signedPercentage(entry.passRateDelta),
|
|
1203
|
+
)}</td>
|
|
1204
|
+
<td class="metric-deltas">${metricDeltaSummary(entry.pairedMetricDeltas)}</td>
|
|
1205
|
+
</tr>`,
|
|
1206
|
+
)
|
|
1207
|
+
.join("")}</tbody>
|
|
1208
|
+
</table></div>
|
|
1209
|
+
</section>
|
|
1210
|
+
</div>`;
|
|
1211
|
+
};
|
|
1212
|
+
|
|
1213
|
+
export { renderArtifactComparison, renderArtifactPreview } from "./artifact-view.js";
|
|
1214
|
+
export { detailEmpty };
|