@prismatic-io/lux 0.0.2-preview.4 → 0.0.2-preview.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/assertions/authoring.d.ts +2 -2
- package/lib/assertions/authoring.d.ts.map +1 -1
- package/lib/assertions/authoring.js.map +1 -1
- package/lib/assertions/core/index.d.ts.map +1 -1
- package/lib/assertions/core/index.js +2 -1
- package/lib/assertions/core/index.js.map +1 -1
- package/lib/assertions/core/resource-checks.d.ts +19 -0
- package/lib/assertions/core/resource-checks.d.ts.map +1 -1
- package/lib/assertions/core/resource-checks.js +32 -0
- package/lib/assertions/core/resource-checks.js.map +1 -1
- package/lib/cli/program.d.ts.map +1 -1
- package/lib/cli/program.js +6 -65
- package/lib/cli/program.js.map +1 -1
- package/lib/cli/view.d.ts +13 -0
- package/lib/cli/view.d.ts.map +1 -0
- package/lib/cli/view.js +126 -0
- package/lib/cli/view.js.map +1 -0
- package/lib/core/action-events.d.ts +29 -0
- package/lib/core/action-events.d.ts.map +1 -0
- package/lib/core/action-events.js +51 -0
- package/lib/core/action-events.js.map +1 -0
- package/lib/core/agent-activity.d.ts +49 -0
- package/lib/core/agent-activity.d.ts.map +1 -0
- package/lib/core/agent-activity.js +206 -0
- package/lib/core/agent-activity.js.map +1 -0
- package/lib/core/agent-events.d.ts +26 -0
- package/lib/core/agent-events.d.ts.map +1 -0
- package/lib/core/agent-events.js +13 -0
- package/lib/core/agent-events.js.map +1 -0
- package/lib/core/driver-metrics.d.ts +98 -0
- package/lib/core/driver-metrics.d.ts.map +1 -0
- package/lib/core/driver-metrics.js +36 -0
- package/lib/core/driver-metrics.js.map +1 -0
- package/lib/core/events.d.ts +34 -0
- package/lib/core/events.d.ts.map +1 -1
- package/lib/core/events.js +11 -2
- package/lib/core/events.js.map +1 -1
- package/lib/core/experiment.d.ts +19 -0
- package/lib/core/experiment.d.ts.map +1 -1
- package/lib/core/experiment.js +2 -0
- package/lib/core/experiment.js.map +1 -1
- package/lib/core/index.d.ts +4 -0
- package/lib/core/index.d.ts.map +1 -1
- package/lib/core/index.js +4 -0
- package/lib/core/index.js.map +1 -1
- package/lib/core/run.d.ts +149 -0
- package/lib/core/run.d.ts.map +1 -1
- package/lib/core/run.js +9 -0
- package/lib/core/run.js.map +1 -1
- package/lib/core/wire.d.ts +11 -0
- package/lib/core/wire.d.ts.map +1 -1
- package/lib/drivers/claude-code/parse-events.d.ts.map +1 -1
- package/lib/drivers/claude-code/parse-events.js +73 -8
- package/lib/drivers/claude-code/parse-events.js.map +1 -1
- package/lib/drivers/codex/app-events.d.ts +9 -2
- package/lib/drivers/codex/app-events.d.ts.map +1 -1
- package/lib/drivers/codex/app-events.js +93 -12
- package/lib/drivers/codex/app-events.js.map +1 -1
- package/lib/drivers/codex/index.d.ts.map +1 -1
- package/lib/drivers/codex/index.js +10 -14
- package/lib/drivers/codex/index.js.map +1 -1
- package/lib/index.d.ts +6 -2
- package/lib/index.d.ts.map +1 -1
- package/lib/index.js +6 -2
- package/lib/index.js.map +1 -1
- package/lib/orchestrator/compare.d.ts +13 -1
- package/lib/orchestrator/compare.d.ts.map +1 -1
- package/lib/orchestrator/compare.js +12 -0
- package/lib/orchestrator/compare.js.map +1 -1
- package/lib/orchestrator/experiment-contracts.d.ts +19 -0
- package/lib/orchestrator/experiment-contracts.d.ts.map +1 -1
- package/lib/orchestrator/experiment-evaluation.d.ts.map +1 -1
- package/lib/orchestrator/experiment-evaluation.js +17 -0
- package/lib/orchestrator/experiment-evaluation.js.map +1 -1
- package/lib/orchestrator/experiment-report.d.ts +19 -0
- package/lib/orchestrator/experiment-report.d.ts.map +1 -1
- package/lib/orchestrator/experiment-runs.d.ts +38 -0
- package/lib/orchestrator/experiment-runs.d.ts.map +1 -1
- package/lib/orchestrator/index.d.ts +1 -1
- package/lib/orchestrator/index.d.ts.map +1 -1
- package/lib/orchestrator/index.js +1 -1
- package/lib/orchestrator/index.js.map +1 -1
- package/lib/orchestrator/list-runs.d.ts +7 -1
- package/lib/orchestrator/list-runs.d.ts.map +1 -1
- package/lib/orchestrator/list-runs.js +50 -6
- package/lib/orchestrator/list-runs.js.map +1 -1
- package/lib/orchestrator/run-execution.d.ts.map +1 -1
- package/lib/orchestrator/run-execution.js +17 -3
- package/lib/orchestrator/run-execution.js.map +1 -1
- package/lib/orchestrator/suite-compare.d.ts +34 -0
- package/lib/orchestrator/suite-compare.d.ts.map +1 -1
- package/lib/orchestrator/suite-summary.d.ts +68 -0
- package/lib/orchestrator/suite-summary.d.ts.map +1 -1
- package/lib/orchestrator/suite-summary.js +45 -7
- package/lib/orchestrator/suite-summary.js.map +1 -1
- package/lib/previewer/artifacts.d.ts +25 -0
- package/lib/previewer/artifacts.d.ts.map +1 -0
- package/lib/previewer/artifacts.js +180 -0
- package/lib/previewer/artifacts.js.map +1 -0
- package/lib/previewer/server.d.ts +105 -0
- package/lib/previewer/server.d.ts.map +1 -0
- package/lib/previewer/server.js +542 -0
- package/lib/previewer/server.js.map +1 -0
- package/package.json +2 -1
- package/skills/lux/references/eval-authoring.md +4 -2
- package/skills/lux-answerer/SKILL.md +1 -1
- package/src/assertions/authoring.ts +2 -0
- package/src/assertions/core/index.ts +2 -0
- package/src/assertions/core/resource-checks.ts +35 -0
- package/src/cli/program.ts +5 -91
- package/src/cli/view.ts +168 -0
- package/src/core/action-events.ts +59 -0
- package/src/core/agent-activity.ts +236 -0
- package/src/core/agent-events.ts +17 -0
- package/src/core/driver-metrics.ts +51 -0
- package/src/core/events.ts +13 -2
- package/src/core/experiment.ts +2 -0
- package/src/core/index.ts +4 -0
- package/src/core/run.ts +9 -0
- package/src/drivers/claude-code/parse-events.ts +98 -12
- package/src/drivers/codex/app-events.ts +116 -16
- package/src/drivers/codex/index.ts +16 -14
- package/src/index.ts +40 -0
- package/src/orchestrator/compare.ts +18 -0
- package/src/orchestrator/experiment-evaluation.ts +23 -0
- package/src/orchestrator/index.ts +1 -0
- package/src/orchestrator/list-runs.ts +66 -5
- package/src/orchestrator/run-execution.ts +19 -2
- package/src/orchestrator/suite-summary.ts +101 -8
- package/src/previewer/artifacts.ts +221 -0
- package/src/previewer/server.ts +765 -0
- package/viewer/app.css +2075 -0
- package/viewer/app.js +606 -0
- package/viewer/artifact-view.js +48 -0
- package/viewer/charts.js +86 -0
- package/viewer/detail.js +1214 -0
- package/viewer/format.js +129 -0
- package/viewer/index.html +151 -0
package/viewer/detail.js
ADDED
|
@@ -0,0 +1,1214 @@
|
|
|
1
|
+
import { comparisonBars, compositionFigure, quantizedBar } from "./charts.js";
|
|
2
|
+
import {
|
|
3
|
+
badge,
|
|
4
|
+
compactNumber,
|
|
5
|
+
detailEmpty,
|
|
6
|
+
duration,
|
|
7
|
+
escapeHtml,
|
|
8
|
+
finite,
|
|
9
|
+
fullDate,
|
|
10
|
+
metricName,
|
|
11
|
+
metricValue,
|
|
12
|
+
percentage,
|
|
13
|
+
signedDuration,
|
|
14
|
+
signedPercentage,
|
|
15
|
+
statusLabel,
|
|
16
|
+
toneForDelta,
|
|
17
|
+
totalTokens,
|
|
18
|
+
} from "./format.js";
|
|
19
|
+
|
|
20
|
+
const json = (value) => {
|
|
21
|
+
if (value === undefined) return "";
|
|
22
|
+
try {
|
|
23
|
+
return JSON.stringify(value, null, 2);
|
|
24
|
+
} catch {
|
|
25
|
+
return String(value);
|
|
26
|
+
}
|
|
27
|
+
};
|
|
28
|
+
|
|
29
|
+
const eventContent = (event) => {
|
|
30
|
+
if (event.type === "lifecycle") return [event.stage, "Run lifecycle"];
|
|
31
|
+
if (event.type === "ready") {
|
|
32
|
+
return [
|
|
33
|
+
`Agent ready · ${event.agentId}`,
|
|
34
|
+
event.model ? `Model ${event.model}` : "Driver ready",
|
|
35
|
+
];
|
|
36
|
+
}
|
|
37
|
+
if (event.type === "interrupt") {
|
|
38
|
+
return [
|
|
39
|
+
`Human-in-the-loop · ${event.kind}`,
|
|
40
|
+
event.question ?? event.request ?? json(event.payload),
|
|
41
|
+
];
|
|
42
|
+
}
|
|
43
|
+
if (event.type === "responded") {
|
|
44
|
+
let answer = event.text ?? null;
|
|
45
|
+
if (!answer && typeof event.approved === "boolean") {
|
|
46
|
+
answer = event.approved ? "Approved" : "Denied";
|
|
47
|
+
}
|
|
48
|
+
return [`Answerer · ${event.kind}`, answer ?? json(event.payload)];
|
|
49
|
+
}
|
|
50
|
+
if (event.type === "progress") {
|
|
51
|
+
const payload = event.payload ?? {};
|
|
52
|
+
const actor = event.agent?.role === "subagent" ? ` · Subagent ${event.agent.id}` : "";
|
|
53
|
+
if (event.kind === "agent-spawn") {
|
|
54
|
+
const spawned = payload.agent ?? {};
|
|
55
|
+
return [
|
|
56
|
+
`Subagent spawned · ${spawned.name ?? spawned.id ?? "unknown"}`,
|
|
57
|
+
[spawned.id, spawned.model, payload.prompt].filter(Boolean).join(" · "),
|
|
58
|
+
];
|
|
59
|
+
}
|
|
60
|
+
if (event.kind === "agent-completed") {
|
|
61
|
+
return [
|
|
62
|
+
`Subagent ${payload.status ?? "completed"} · ${payload.agentId ?? "unknown"}`,
|
|
63
|
+
"Agent lifecycle",
|
|
64
|
+
];
|
|
65
|
+
}
|
|
66
|
+
if (event.kind === "action") {
|
|
67
|
+
return [
|
|
68
|
+
`Action${actor} · ${payload.name ?? "unknown"}`,
|
|
69
|
+
payload.category ? `Category: ${payload.category}` : json(payload),
|
|
70
|
+
];
|
|
71
|
+
}
|
|
72
|
+
const name = event.kind === "tool-call" ? payload.name : event.kind;
|
|
73
|
+
return [`Progress${actor} · ${name ?? event.kind}`, json(payload)];
|
|
74
|
+
}
|
|
75
|
+
if (event.type === "error") return [`Error · ${event.exitReason}`, event.reason];
|
|
76
|
+
return ["Completed", event.summary ? json(event.summary) : "Driver finished"];
|
|
77
|
+
};
|
|
78
|
+
|
|
79
|
+
const eventTime = (event, startedAt) => {
|
|
80
|
+
if (typeof event.ts !== "number") return "setup";
|
|
81
|
+
if (event.ts > 1_000_000_000_000) return fullDate(event.ts);
|
|
82
|
+
const start = Date.parse(startedAt ?? "");
|
|
83
|
+
if (!Number.isFinite(start)) return duration(event.ts);
|
|
84
|
+
return `+${duration(Math.max(0, event.ts - start))}`;
|
|
85
|
+
};
|
|
86
|
+
|
|
87
|
+
const assertionMeta = (assertion) =>
|
|
88
|
+
[
|
|
89
|
+
assertion.type,
|
|
90
|
+
`threshold ${finite(assertion.threshold)}`,
|
|
91
|
+
`weight ${finite(assertion.weight)}`,
|
|
92
|
+
duration(assertion.durationMs),
|
|
93
|
+
assertion.usage ? `${compactNumber(totalTokens(assertion.usage))} judge tokens` : null,
|
|
94
|
+
]
|
|
95
|
+
.filter(Boolean)
|
|
96
|
+
.join(" · ");
|
|
97
|
+
|
|
98
|
+
const assertionDiagnostics = (assertion) => {
|
|
99
|
+
const rows = [
|
|
100
|
+
assertion.comment ? `<p class="assertion-comment">${escapeHtml(assertion.comment)}</p>` : "",
|
|
101
|
+
assertion.errorMessage
|
|
102
|
+
? `<p class="diagnostic-error">${escapeHtml(assertion.errorMessage)}</p>`
|
|
103
|
+
: "",
|
|
104
|
+
assertion.detail !== undefined
|
|
105
|
+
? `<details class="diagnostic"><summary>Structured detail</summary><pre>${escapeHtml(
|
|
106
|
+
json(assertion.detail),
|
|
107
|
+
)}</pre></details>`
|
|
108
|
+
: "",
|
|
109
|
+
];
|
|
110
|
+
return rows.join("");
|
|
111
|
+
};
|
|
112
|
+
|
|
113
|
+
const inferredMetricUnit = (name, unit) => {
|
|
114
|
+
if (unit) return unit;
|
|
115
|
+
if (name.endsWith("Ms")) return "milliseconds";
|
|
116
|
+
if (name.endsWith(".bytes")) return "bytes";
|
|
117
|
+
if (name.endsWith(".usd")) return "usd";
|
|
118
|
+
if (name.endsWith("Rate")) return "ratio";
|
|
119
|
+
if (
|
|
120
|
+
name.includes("tokens") ||
|
|
121
|
+
/(?:^|\.)(?:input|output|total|cacheRead|cacheCreation)$/.test(name)
|
|
122
|
+
) {
|
|
123
|
+
return "tokens";
|
|
124
|
+
}
|
|
125
|
+
return "number";
|
|
126
|
+
};
|
|
127
|
+
|
|
128
|
+
const driverMetricStats = (metrics) =>
|
|
129
|
+
Object.entries(metrics ?? {})
|
|
130
|
+
.map(
|
|
131
|
+
([name, metric]) =>
|
|
132
|
+
`<div><dt>${escapeHtml(metricName(name, metric.label))}</dt><dd>${escapeHtml(
|
|
133
|
+
metricValue(metric.value, metric.unit),
|
|
134
|
+
)}</dd><small>${escapeHtml(name)}</small></div>`,
|
|
135
|
+
)
|
|
136
|
+
.join("");
|
|
137
|
+
|
|
138
|
+
const agentActivityCards = (activities) =>
|
|
139
|
+
(activities ?? [])
|
|
140
|
+
.filter((activity) => activity.agent?.role === "subagent")
|
|
141
|
+
.map((activity, index) => {
|
|
142
|
+
const agent = activity.agent;
|
|
143
|
+
const actions = activity.actions ?? {};
|
|
144
|
+
const tools = activity.toolCalls ?? {};
|
|
145
|
+
const label = agent.name ?? `Subagent ${index + 1}`;
|
|
146
|
+
return `<article class="agent-card">
|
|
147
|
+
<header><div><strong>${escapeHtml(label)}</strong><small>${escapeHtml(agent.id)}</small></div><span>${escapeHtml(
|
|
148
|
+
`${activity.status ?? "unknown"} · ${duration(activity.durationMs)}`,
|
|
149
|
+
)}</span></header>
|
|
150
|
+
<dl>
|
|
151
|
+
<div><dt>Events</dt><dd>${compactNumber(activity.eventCount)}</dd></div>
|
|
152
|
+
<div><dt>Tokens</dt><dd>${compactNumber(activity.usage?.total)}</dd></div>
|
|
153
|
+
<div><dt>Tool calls</dt><dd>${compactNumber(tools.total)}</dd></div>
|
|
154
|
+
<div><dt>Succeeded</dt><dd>${compactNumber(tools.succeeded)}</dd></div>
|
|
155
|
+
<div><dt>Failed</dt><dd>${compactNumber(tools.failed)}</dd></div>
|
|
156
|
+
<div><dt>Actions</dt><dd>${compactNumber(actions.total)}</dd></div>
|
|
157
|
+
<div><dt>Mutations</dt><dd>${compactNumber(actions.mutations)}</dd></div>
|
|
158
|
+
<div><dt>First action</dt><dd>${escapeHtml(duration(actions.timeToFirstActionMs))}</dd></div>
|
|
159
|
+
<div><dt>First mutation</dt><dd>${escapeHtml(duration(actions.timeToFirstMutationMs))}</dd></div>
|
|
160
|
+
</dl>
|
|
161
|
+
<small class="agent-parent">Spawn ${escapeHtml(agent.spawnToolUseId ?? "unknown")} · parent ${escapeHtml(
|
|
162
|
+
agent.parentId ?? "unknown",
|
|
163
|
+
)}</small>
|
|
164
|
+
</article>`;
|
|
165
|
+
})
|
|
166
|
+
.join("");
|
|
167
|
+
|
|
168
|
+
export const renderRunDetail = (item, detail) => {
|
|
169
|
+
const metadata = detail.metadata;
|
|
170
|
+
const grading = detail.grading;
|
|
171
|
+
const assertions = grading?.assertions ?? [];
|
|
172
|
+
const events = detail.events ?? [];
|
|
173
|
+
const artifacts = detail.artifacts ?? [];
|
|
174
|
+
const agentTokens = totalTokens(metadata?.usage?.agent);
|
|
175
|
+
const answererTokens = totalTokens(metadata?.usage?.answerer);
|
|
176
|
+
const gradingTokens = totalTokens(metadata?.usage?.grading);
|
|
177
|
+
const harnessTokens = answererTokens + gradingTokens;
|
|
178
|
+
const phase = metadata?.phaseDurationMs;
|
|
179
|
+
const actionTiming = metadata?.actionTiming;
|
|
180
|
+
const prompt = detail.case?.prompt;
|
|
181
|
+
const driverMetrics = driverMetricStats(metadata?.driverMetrics);
|
|
182
|
+
const subagents = agentActivityCards(metadata?.agentActivity);
|
|
183
|
+
return `
|
|
184
|
+
<header class="detail-header">
|
|
185
|
+
<div class="detail-kicker">
|
|
186
|
+
${badge(item.status)}
|
|
187
|
+
<span class="detail-path">${escapeHtml(item.path)}</span>
|
|
188
|
+
</div>
|
|
189
|
+
<div class="detail-title-row">
|
|
190
|
+
<div>
|
|
191
|
+
<h2>${escapeHtml(detail.case?.id ?? item.caseId)}</h2>
|
|
192
|
+
<p class="detail-subtitle">${escapeHtml(
|
|
193
|
+
[metadata?.driver?.name, metadata?.driver?.model, fullDate(metadata?.startedAt)]
|
|
194
|
+
.filter(Boolean)
|
|
195
|
+
.join(" · "),
|
|
196
|
+
)}</p>
|
|
197
|
+
</div>
|
|
198
|
+
<div class="score-block">
|
|
199
|
+
<strong>${escapeHtml(percentage(grading?.caseScore))}</strong>
|
|
200
|
+
<small>case score</small>
|
|
201
|
+
</div>
|
|
202
|
+
</div>
|
|
203
|
+
</header>
|
|
204
|
+
<div class="detail-body">
|
|
205
|
+
${
|
|
206
|
+
grading?.incomplete
|
|
207
|
+
? `<div class="attention-banner">Incomplete ${escapeHtml(
|
|
208
|
+
grading.incomplete.phase ?? "run",
|
|
209
|
+
)}: ${escapeHtml(grading.incomplete.exitReason)}</div>`
|
|
210
|
+
: ""
|
|
211
|
+
}
|
|
212
|
+
<dl class="detail-stats">
|
|
213
|
+
<div><dt>Duration</dt><dd>${escapeHtml(duration(item.durationMs))}</dd></div>
|
|
214
|
+
<div><dt>Agent tokens</dt><dd>${escapeHtml(compactNumber(agentTokens))}</dd></div>
|
|
215
|
+
<div><dt>Harness tokens</dt><dd>${escapeHtml(compactNumber(harnessTokens))}</dd></div>
|
|
216
|
+
<div><dt>Artifacts</dt><dd>${artifacts.length}</dd></div>
|
|
217
|
+
<div><dt>First action</dt><dd>${escapeHtml(duration(actionTiming?.timeToFirstActionMs))}</dd></div>
|
|
218
|
+
<div><dt>First mutation</dt><dd>${escapeHtml(duration(actionTiming?.timeToFirstMutationMs))}</dd></div>
|
|
219
|
+
<div><dt>First durable mutation</dt><dd>${escapeHtml(
|
|
220
|
+
duration(actionTiming?.timeToFirstDurableMutationMs),
|
|
221
|
+
)}</dd></div>
|
|
222
|
+
</dl>
|
|
223
|
+
${
|
|
224
|
+
driverMetrics
|
|
225
|
+
? `<section class="detail-section"><h3>Driver metrics <span>reported by ${escapeHtml(
|
|
226
|
+
metadata?.driver?.name ?? "driver",
|
|
227
|
+
)}</span></h3><dl class="detail-stats driver-metrics">${driverMetrics}</dl></section>`
|
|
228
|
+
: ""
|
|
229
|
+
}
|
|
230
|
+
${
|
|
231
|
+
subagents
|
|
232
|
+
? `<section class="detail-section"><h3>Subagent activity <span>${
|
|
233
|
+
metadata.agentActivity.filter((activity) => activity.agent.role === "subagent").length
|
|
234
|
+
} spawned</span></h3><div class="agent-grid">${subagents}</div></section>`
|
|
235
|
+
: ""
|
|
236
|
+
}
|
|
237
|
+
<div class="evidence-visual-grid">
|
|
238
|
+
${compositionFigure(
|
|
239
|
+
"Judgment outcome",
|
|
240
|
+
[
|
|
241
|
+
{
|
|
242
|
+
key: "passed",
|
|
243
|
+
label: "Passed",
|
|
244
|
+
value: assertions.filter((assertion) => assertion.passed).length,
|
|
245
|
+
},
|
|
246
|
+
{
|
|
247
|
+
key: "regressed",
|
|
248
|
+
label: "Failed",
|
|
249
|
+
value: assertions.filter((assertion) => !assertion.passed).length,
|
|
250
|
+
},
|
|
251
|
+
],
|
|
252
|
+
`${assertions.filter((assertion) => assertion.passed).length} of ${
|
|
253
|
+
assertions.length
|
|
254
|
+
} assertions passed`,
|
|
255
|
+
)}
|
|
256
|
+
${
|
|
257
|
+
agentTokens + answererTokens + gradingTokens > 0
|
|
258
|
+
? compositionFigure(
|
|
259
|
+
"Token composition",
|
|
260
|
+
[
|
|
261
|
+
{ key: "agent", label: "Agent", value: agentTokens },
|
|
262
|
+
{ key: "answerer", label: "Answerer", value: answererTokens },
|
|
263
|
+
{ key: "grading", label: "Grading", value: gradingTokens },
|
|
264
|
+
],
|
|
265
|
+
`${agentTokens} agent, ${answererTokens} answerer, ${gradingTokens} grading tokens`,
|
|
266
|
+
)
|
|
267
|
+
: ""
|
|
268
|
+
}
|
|
269
|
+
${
|
|
270
|
+
phase
|
|
271
|
+
? compositionFigure(
|
|
272
|
+
"Runtime composition",
|
|
273
|
+
[
|
|
274
|
+
{ key: "agent", label: "Agent", value: phase.agent },
|
|
275
|
+
{ key: "collect", label: "Collect", value: phase.collect },
|
|
276
|
+
{ key: "grading", label: "Grading", value: phase.grading },
|
|
277
|
+
],
|
|
278
|
+
`${duration(phase.agent)} agent, ${duration(phase.collect)} collection, ${duration(
|
|
279
|
+
phase.grading,
|
|
280
|
+
)} grading`,
|
|
281
|
+
)
|
|
282
|
+
: ""
|
|
283
|
+
}
|
|
284
|
+
</div>
|
|
285
|
+
${
|
|
286
|
+
prompt
|
|
287
|
+
? `<section class="detail-section"><h3>Case prompt</h3><div class="prompt-block">${escapeHtml(
|
|
288
|
+
prompt,
|
|
289
|
+
)}</div></section>`
|
|
290
|
+
: ""
|
|
291
|
+
}
|
|
292
|
+
<section class="detail-section">
|
|
293
|
+
<h3>Judgment <span>${assertions.filter((entry) => entry.passed).length}/${assertions.length} passed</span></h3>
|
|
294
|
+
<div class="assertion-list">
|
|
295
|
+
${
|
|
296
|
+
assertions.length
|
|
297
|
+
? assertions
|
|
298
|
+
.map(
|
|
299
|
+
(assertion) => `
|
|
300
|
+
<article class="assertion${assertion.passed ? " passed" : ""}">
|
|
301
|
+
<span class="assertion-mark">${assertion.passed ? "✓" : "×"}</span>
|
|
302
|
+
<span class="assertion-content">
|
|
303
|
+
<strong>${escapeHtml(assertion.name)}</strong>
|
|
304
|
+
<small>${escapeHtml(assertionMeta(assertion))}</small>
|
|
305
|
+
${quantizedBar(
|
|
306
|
+
assertion.score,
|
|
307
|
+
assertion.passed ? "tone-positive" : "tone-negative",
|
|
308
|
+
`${assertion.name} scored ${percentage(
|
|
309
|
+
assertion.score,
|
|
310
|
+
)}; threshold ${percentage(assertion.threshold)}`,
|
|
311
|
+
assertion.threshold,
|
|
312
|
+
)}
|
|
313
|
+
${assertionDiagnostics(assertion)}
|
|
314
|
+
</span>
|
|
315
|
+
<span class="assertion-score">${escapeHtml(
|
|
316
|
+
percentage(assertion.score),
|
|
317
|
+
)}</span>
|
|
318
|
+
</article>`,
|
|
319
|
+
)
|
|
320
|
+
.join("")
|
|
321
|
+
: '<div class="artifact"><span>No grading assertions recorded</span><small>ungraded</small></div>'
|
|
322
|
+
}
|
|
323
|
+
</div>
|
|
324
|
+
</section>
|
|
325
|
+
<section class="detail-section">
|
|
326
|
+
<h3>Trace <span>${events.length} events</span></h3>
|
|
327
|
+
<div class="trace-list">
|
|
328
|
+
${
|
|
329
|
+
events.length
|
|
330
|
+
? events
|
|
331
|
+
.map((event) => {
|
|
332
|
+
const [title, content] = eventContent(event);
|
|
333
|
+
return `<article class="trace-item">
|
|
334
|
+
<time>${escapeHtml(eventTime(event, metadata?.startedAt))}</time>
|
|
335
|
+
<div><strong>${escapeHtml(title)}</strong><p>${escapeHtml(content)}</p></div>
|
|
336
|
+
</article>`;
|
|
337
|
+
})
|
|
338
|
+
.join("")
|
|
339
|
+
: '<div class="artifact"><span>No trace events recorded</span><small>empty</small></div>'
|
|
340
|
+
}
|
|
341
|
+
</div>
|
|
342
|
+
</section>
|
|
343
|
+
<section class="detail-section">
|
|
344
|
+
<h3>Artifacts <span>${artifacts.length} files</span></h3>
|
|
345
|
+
<div class="artifact-list">
|
|
346
|
+
${
|
|
347
|
+
artifacts.length
|
|
348
|
+
? artifacts
|
|
349
|
+
.map(
|
|
350
|
+
(artifact) => `<button class="artifact artifact-button" type="button"
|
|
351
|
+
data-artifact="${escapeHtml(artifact)}">
|
|
352
|
+
<span>${escapeHtml(artifact)}</span><small>preview →</small>
|
|
353
|
+
</button>`,
|
|
354
|
+
)
|
|
355
|
+
.join("")
|
|
356
|
+
: '<div class="artifact"><span>No captured artifacts</span><small>empty</small></div>'
|
|
357
|
+
}
|
|
358
|
+
</div>
|
|
359
|
+
</section>
|
|
360
|
+
</div>`;
|
|
361
|
+
};
|
|
362
|
+
|
|
363
|
+
const suiteItemStatus = (item) => {
|
|
364
|
+
if (item.status !== "completed") return item.status;
|
|
365
|
+
if (item.exitReason !== "done" || item.casePassed === false) return "failed";
|
|
366
|
+
if (item.casePassed === true) return "passed";
|
|
367
|
+
return "ungraded";
|
|
368
|
+
};
|
|
369
|
+
|
|
370
|
+
const metricSummaryRows = (metrics) =>
|
|
371
|
+
Object.entries(metrics ?? {})
|
|
372
|
+
.map(
|
|
373
|
+
([name, summary]) => `<tr>
|
|
374
|
+
<td>${escapeHtml(name)}</td>
|
|
375
|
+
<td>${escapeHtml(metricValue(summary.mean, inferredMetricUnit(name, summary.unit)))}</td>
|
|
376
|
+
<td>${escapeHtml(metricValue(summary.median, inferredMetricUnit(name, summary.unit)))}</td>
|
|
377
|
+
<td>${escapeHtml(metricValue(summary.p90, inferredMetricUnit(name, summary.unit)))}</td>
|
|
378
|
+
<td>${summary.n}</td>
|
|
379
|
+
</tr>`,
|
|
380
|
+
)
|
|
381
|
+
.join("");
|
|
382
|
+
|
|
383
|
+
export const renderSuiteDetail = (item, payload) => {
|
|
384
|
+
const summary = payload?.summary;
|
|
385
|
+
const totals = summary?.totals;
|
|
386
|
+
const metrics = metricSummaryRows(summary?.metrics);
|
|
387
|
+
return `
|
|
388
|
+
<header class="detail-header">
|
|
389
|
+
<div class="detail-kicker">${badge(item.status)}<span class="detail-path">${escapeHtml(
|
|
390
|
+
item.path,
|
|
391
|
+
)}</span></div>
|
|
392
|
+
<div class="detail-title-row">
|
|
393
|
+
<div><h2>${escapeHtml(item.suiteId)}</h2><p class="detail-subtitle">${escapeHtml(
|
|
394
|
+
`${item.profileId ?? "default profile"} · ${fullDate(item.startedAt)}`,
|
|
395
|
+
)}</p></div>
|
|
396
|
+
<div class="score-block"><strong>${escapeHtml(
|
|
397
|
+
percentage(totals?.passRate),
|
|
398
|
+
)}</strong><small>${totals?.passed ?? 0}/${totals?.attempts ?? item.items.length} cases passed</small></div>
|
|
399
|
+
</div>
|
|
400
|
+
</header>
|
|
401
|
+
<div class="detail-body">
|
|
402
|
+
<dl class="detail-stats">
|
|
403
|
+
<div><dt>Attempts</dt><dd>${totals?.attempts ?? item.items.length}</dd></div>
|
|
404
|
+
<div><dt>Concurrency</dt><dd>${item.concurrency}</dd></div>
|
|
405
|
+
<div><dt>Failed</dt><dd>${totals?.failed ?? 0}</dd></div>
|
|
406
|
+
<div><dt>Pending</dt><dd>${totals?.pending ?? 0}</dd></div>
|
|
407
|
+
</dl>
|
|
408
|
+
${compositionFigure(
|
|
409
|
+
"Suite outcomes",
|
|
410
|
+
[
|
|
411
|
+
{ key: "passed", label: "Passed", value: totals?.passed ?? 0 },
|
|
412
|
+
{ key: "regressed", label: "Failed", value: totals?.failed ?? 0 },
|
|
413
|
+
{ key: "ungraded", label: "Ungraded", value: totals?.ungraded ?? 0 },
|
|
414
|
+
{ key: "pending", label: "Pending", value: totals?.pending ?? 0 },
|
|
415
|
+
],
|
|
416
|
+
`${totals?.passed ?? 0} passed, ${totals?.failed ?? 0} failed, ${
|
|
417
|
+
totals?.ungraded ?? 0
|
|
418
|
+
} ungraded, ${totals?.pending ?? 0} pending`,
|
|
419
|
+
)}
|
|
420
|
+
${
|
|
421
|
+
metrics
|
|
422
|
+
? `<section class="detail-section">
|
|
423
|
+
<h3>Aggregate metrics <span>mean / median / p90</span></h3>
|
|
424
|
+
<div class="matrix-wrap"><table class="matrix">
|
|
425
|
+
<thead><tr><th>Metric</th><th>Mean</th><th>Median</th><th>P90</th><th>N</th></tr></thead>
|
|
426
|
+
<tbody>${metrics}</tbody>
|
|
427
|
+
</table></div>
|
|
428
|
+
</section>`
|
|
429
|
+
: ""
|
|
430
|
+
}
|
|
431
|
+
<section class="detail-section">
|
|
432
|
+
<h3>Case attempts <span>${item.items.length} total</span></h3>
|
|
433
|
+
<div class="suite-list">
|
|
434
|
+
${item.items
|
|
435
|
+
.map((entry) => {
|
|
436
|
+
const content = `<span>${escapeHtml(entry.caseId)}</span>
|
|
437
|
+
<small>${escapeHtml(statusLabel(suiteItemStatus(entry)))}</small>`;
|
|
438
|
+
return entry.runId
|
|
439
|
+
? `<button class="suite-item suite-button" type="button" data-open-run="${escapeHtml(
|
|
440
|
+
entry.runId,
|
|
441
|
+
)}">${content}</button>`
|
|
442
|
+
: `<div class="suite-item">${content}</div>`;
|
|
443
|
+
})
|
|
444
|
+
.join("")}
|
|
445
|
+
</div>
|
|
446
|
+
</section>
|
|
447
|
+
</div>`;
|
|
448
|
+
};
|
|
449
|
+
|
|
450
|
+
const candidateById = (payload, id) => payload.candidates.find((candidate) => candidate.id === id);
|
|
451
|
+
|
|
452
|
+
const baselineCandidate = (payload) =>
|
|
453
|
+
candidateById(payload, "baseline") ??
|
|
454
|
+
payload.candidates.find((candidate) => candidate.parentIds.length === 0) ??
|
|
455
|
+
payload.candidates[0] ??
|
|
456
|
+
null;
|
|
457
|
+
|
|
458
|
+
const evaluationKey = (evaluation) => `${evaluation.profileId}/${evaluation.split}`;
|
|
459
|
+
|
|
460
|
+
const caseDeltaRows = (baseline, target) => {
|
|
461
|
+
const cases = new Set([
|
|
462
|
+
...Object.keys(baseline?.casePassRates ?? {}),
|
|
463
|
+
...Object.keys(target?.casePassRates ?? {}),
|
|
464
|
+
]);
|
|
465
|
+
return [...cases]
|
|
466
|
+
.sort()
|
|
467
|
+
.map((caseId) => {
|
|
468
|
+
const before = baseline?.casePassRates?.[caseId];
|
|
469
|
+
const after = target?.casePassRates?.[caseId];
|
|
470
|
+
const delta = typeof before === "number" && typeof after === "number" ? after - before : null;
|
|
471
|
+
return `<tr>
|
|
472
|
+
<td>${escapeHtml(caseId)}</td>
|
|
473
|
+
<td class="bar-table-cell">${quantizedBar(
|
|
474
|
+
before,
|
|
475
|
+
"tone-before",
|
|
476
|
+
`${caseId} baseline pass rate ${percentage(before)}`,
|
|
477
|
+
)}<small>${escapeHtml(percentage(before))}</small></td>
|
|
478
|
+
<td class="bar-table-cell">${quantizedBar(
|
|
479
|
+
after,
|
|
480
|
+
"tone-after",
|
|
481
|
+
`${caseId} candidate pass rate ${percentage(after)}`,
|
|
482
|
+
)}<small>${escapeHtml(percentage(after))}</small></td>
|
|
483
|
+
<td class="${toneForDelta(delta)}">${escapeHtml(signedPercentage(delta))}</td>
|
|
484
|
+
</tr>`;
|
|
485
|
+
})
|
|
486
|
+
.join("");
|
|
487
|
+
};
|
|
488
|
+
|
|
489
|
+
const evaluationMetricComparisons = (before, after) => {
|
|
490
|
+
const definitions = { ...(before?.metricDefinitions ?? {}), ...(after?.metricDefinitions ?? {}) };
|
|
491
|
+
return Object.keys(definitions)
|
|
492
|
+
.sort()
|
|
493
|
+
.map((name) => {
|
|
494
|
+
const definition = definitions[name];
|
|
495
|
+
const a = before?.metrics?.[name];
|
|
496
|
+
const b = after?.metrics?.[name];
|
|
497
|
+
const delta = typeof a === "number" && typeof b === "number" ? b - a : null;
|
|
498
|
+
const tone =
|
|
499
|
+
definition.better === "neutral"
|
|
500
|
+
? "metric-neutral"
|
|
501
|
+
: toneForDelta(delta, definition.better === "lower");
|
|
502
|
+
return comparisonBars({
|
|
503
|
+
title: metricName(name, definition.label),
|
|
504
|
+
before: a,
|
|
505
|
+
after: b,
|
|
506
|
+
beforeText: metricValue(a, definition.unit),
|
|
507
|
+
afterText: metricValue(b, definition.unit),
|
|
508
|
+
deltaText: signedMetricValue(delta, definition.unit),
|
|
509
|
+
deltaTone: tone,
|
|
510
|
+
});
|
|
511
|
+
})
|
|
512
|
+
.join("");
|
|
513
|
+
};
|
|
514
|
+
|
|
515
|
+
const experimentScopes = (baselineEvaluations, targetEvaluations) => {
|
|
516
|
+
const baselineByKey = new Map(baselineEvaluations.map((entry) => [evaluationKey(entry), entry]));
|
|
517
|
+
const targetByKey = new Map(targetEvaluations.map((entry) => [evaluationKey(entry), entry]));
|
|
518
|
+
const keys = new Set([...baselineByKey.keys(), ...targetByKey.keys()]);
|
|
519
|
+
return [...keys]
|
|
520
|
+
.sort()
|
|
521
|
+
.map((key) => {
|
|
522
|
+
const before = baselineByKey.get(key);
|
|
523
|
+
const after = targetByKey.get(key);
|
|
524
|
+
const delta =
|
|
525
|
+
typeof before?.passRate === "number" && typeof after?.passRate === "number"
|
|
526
|
+
? after.passRate - before.passRate
|
|
527
|
+
: null;
|
|
528
|
+
const metrics = evaluationMetricComparisons(before, after);
|
|
529
|
+
return `<article class="comparison-scope">
|
|
530
|
+
<header>
|
|
531
|
+
<strong>${escapeHtml(key)}</strong>
|
|
532
|
+
<span class="${toneForDelta(delta)}">${escapeHtml(signedPercentage(delta))}</span>
|
|
533
|
+
</header>
|
|
534
|
+
<div class="matrix-wrap"><table class="matrix">
|
|
535
|
+
<thead><tr><th>Case</th><th>Baseline</th><th>Candidate</th><th>Δ</th></tr></thead>
|
|
536
|
+
<tbody>${caseDeltaRows(before, after) || '<tr><td colspan="4">No per-case evidence.</td></tr>'}</tbody>
|
|
537
|
+
</table></div>
|
|
538
|
+
${metrics ? `<div class="comparison-plot-grid experiment-metrics">${metrics}</div>` : ""}
|
|
539
|
+
</article>`;
|
|
540
|
+
})
|
|
541
|
+
.join("");
|
|
542
|
+
};
|
|
543
|
+
|
|
544
|
+
const candidateLandscape = (evaluations, baselineId, targetId) => {
|
|
545
|
+
const rows = evaluations
|
|
546
|
+
.slice()
|
|
547
|
+
.sort((left, right) =>
|
|
548
|
+
`${left.profileId}/${left.split}/${left.candidateId}`.localeCompare(
|
|
549
|
+
`${right.profileId}/${right.split}/${right.candidateId}`,
|
|
550
|
+
),
|
|
551
|
+
)
|
|
552
|
+
.map((evaluation) => {
|
|
553
|
+
let role = "peer";
|
|
554
|
+
let tone = "tone-neutral";
|
|
555
|
+
if (evaluation.candidateId === baselineId) {
|
|
556
|
+
role = "baseline";
|
|
557
|
+
tone = "tone-before";
|
|
558
|
+
} else if (evaluation.candidateId === targetId) {
|
|
559
|
+
role = "target";
|
|
560
|
+
tone = "tone-after";
|
|
561
|
+
}
|
|
562
|
+
return `<div class="landscape-row ${role}">
|
|
563
|
+
<span><strong>${escapeHtml(evaluation.candidateId)}</strong><small>${escapeHtml(
|
|
564
|
+
`${evaluation.profileId} / ${evaluation.split}`,
|
|
565
|
+
)}</small></span>
|
|
566
|
+
${quantizedBar(
|
|
567
|
+
evaluation.passRate,
|
|
568
|
+
tone,
|
|
569
|
+
`${evaluation.candidateId} ${evaluation.profileId} ${evaluation.split} pass rate ${percentage(
|
|
570
|
+
evaluation.passRate,
|
|
571
|
+
)}`,
|
|
572
|
+
)}
|
|
573
|
+
<b>${escapeHtml(percentage(evaluation.passRate))}</b>
|
|
574
|
+
</div>`;
|
|
575
|
+
})
|
|
576
|
+
.join("");
|
|
577
|
+
return `<figure class="candidate-landscape">
|
|
578
|
+
<figcaption><span>Candidate pass-rate landscape</span><small>shared 0–100% scale</small></figcaption>
|
|
579
|
+
<div class="landscape-axis"><span>0</span><span>50</span><span>100%</span></div>
|
|
580
|
+
${rows || '<div class="artifact"><span>No completed candidate evaluations</span><small>empty</small></div>'}
|
|
581
|
+
</figure>`;
|
|
582
|
+
};
|
|
583
|
+
|
|
584
|
+
const budgetUtilization = (decision) => {
|
|
585
|
+
if (!decision) return "";
|
|
586
|
+
const entries = [
|
|
587
|
+
["Candidates", decision.spend.candidates, decision.budget.maxCandidates, "count"],
|
|
588
|
+
["Metric calls", decision.spend.metricCalls, decision.budget.maxMetricCalls, "count"],
|
|
589
|
+
["Judge calls", decision.spend.judgeCalls, decision.budget.maxJudgeCalls, "count"],
|
|
590
|
+
["Proposal calls", decision.spend.proposalCalls, decision.budget.maxProposalCalls, "count"],
|
|
591
|
+
["Tokens", decision.spend.totalTokens, decision.budget.maxTokens, "compact"],
|
|
592
|
+
["Cost", decision.spend.costUsd, decision.budget.maxCostUsd, "currency"],
|
|
593
|
+
["Active time", decision.spend.activeTimeMs, decision.budget.maxWallTimeMs, "duration"],
|
|
594
|
+
];
|
|
595
|
+
const formatValue = (value, kind) => {
|
|
596
|
+
if (kind === "compact") return compactNumber(value);
|
|
597
|
+
if (kind === "currency") return `$${finite(value)}`;
|
|
598
|
+
if (kind === "duration") return duration(value);
|
|
599
|
+
return finite(value, 0);
|
|
600
|
+
};
|
|
601
|
+
return `<figure class="budget-visual">
|
|
602
|
+
<figcaption><span>Budget utilization</span><small>observed / ceiling</small></figcaption>
|
|
603
|
+
${entries
|
|
604
|
+
.map(([label, value, limit, kind]) => {
|
|
605
|
+
const ratio = limit > 0 ? value / limit : 0;
|
|
606
|
+
return `<div class="budget-row">
|
|
607
|
+
<span>${escapeHtml(label)}</span>
|
|
608
|
+
${quantizedBar(
|
|
609
|
+
ratio,
|
|
610
|
+
ratio > 0.9 ? "tone-negative" : "tone-neutral",
|
|
611
|
+
`${label} ${formatValue(value, kind)} of ${formatValue(limit, kind)}`,
|
|
612
|
+
)}
|
|
613
|
+
<b>${escapeHtml(`${formatValue(value, kind)} / ${formatValue(limit, kind)}`)}</b>
|
|
614
|
+
</div>`;
|
|
615
|
+
})
|
|
616
|
+
.join("")}
|
|
617
|
+
</figure>`;
|
|
618
|
+
};
|
|
619
|
+
|
|
620
|
+
const failedEvidence = (evaluations) =>
|
|
621
|
+
evaluations
|
|
622
|
+
.flatMap((evaluation) =>
|
|
623
|
+
evaluation.failedCases.map((failure) => ({
|
|
624
|
+
...failure,
|
|
625
|
+
scope: evaluationKey(evaluation),
|
|
626
|
+
})),
|
|
627
|
+
)
|
|
628
|
+
.map(
|
|
629
|
+
(failure) => `<article class="failure-card">
|
|
630
|
+
<header><strong>${escapeHtml(failure.caseId)}</strong><small>${escapeHtml(
|
|
631
|
+
failure.scope,
|
|
632
|
+
)} · ${escapeHtml(percentage(failure.score))}</small></header>
|
|
633
|
+
${
|
|
634
|
+
failure.feedback.length
|
|
635
|
+
? `<ul>${failure.feedback.map((entry) => `<li>${escapeHtml(entry)}</li>`).join("")}</ul>`
|
|
636
|
+
: "<p>No evaluator feedback recorded.</p>"
|
|
637
|
+
}
|
|
638
|
+
</article>`,
|
|
639
|
+
)
|
|
640
|
+
.join("");
|
|
641
|
+
|
|
642
|
+
export const experimentTargetId = (payload, requested) => {
|
|
643
|
+
const baseline = baselineCandidate(payload);
|
|
644
|
+
const ids = new Set(payload.experiment.evaluations.map((entry) => entry.candidateId));
|
|
645
|
+
if (requested && ids.has(requested)) return requested;
|
|
646
|
+
const champion = payload.experiment.championCandidateId;
|
|
647
|
+
if (champion && ids.has(champion)) return champion;
|
|
648
|
+
return [...ids].find((id) => id !== baseline?.id) ?? [...ids][0] ?? null;
|
|
649
|
+
};
|
|
650
|
+
|
|
651
|
+
export const renderExperimentDetail = (item, payload, requestedTargetId) => {
|
|
652
|
+
const experiment = payload.experiment;
|
|
653
|
+
const decision = payload.decision;
|
|
654
|
+
const baseline = baselineCandidate(payload);
|
|
655
|
+
const targetId = experimentTargetId(payload, requestedTargetId);
|
|
656
|
+
const target = targetId ? candidateById(payload, targetId) : null;
|
|
657
|
+
const baselineEvaluations = experiment.evaluations.filter(
|
|
658
|
+
(evaluation) => evaluation.candidateId === baseline?.id,
|
|
659
|
+
);
|
|
660
|
+
const targetEvaluations = experiment.evaluations.filter(
|
|
661
|
+
(evaluation) => evaluation.candidateId === targetId,
|
|
662
|
+
);
|
|
663
|
+
const options = [...new Set(experiment.evaluations.map((entry) => entry.candidateId))]
|
|
664
|
+
.filter((id) => id !== baseline?.id || id === targetId)
|
|
665
|
+
.map(
|
|
666
|
+
(id) =>
|
|
667
|
+
`<option value="${escapeHtml(id)}"${id === targetId ? " selected" : ""}>${escapeHtml(
|
|
668
|
+
id,
|
|
669
|
+
)}</option>`,
|
|
670
|
+
)
|
|
671
|
+
.join("");
|
|
672
|
+
const gates = (decision?.gates ?? [])
|
|
673
|
+
.map(
|
|
674
|
+
(gate) => `<article class="gate ${gate.passed ? "passed" : "failed"}">
|
|
675
|
+
<span>${gate.passed ? "✓" : "×"}</span>
|
|
676
|
+
<div><strong>${escapeHtml(gate.metric)}</strong><small>${escapeHtml(
|
|
677
|
+
[gate.profile, gate.split, gate.interval, gate.note].filter(Boolean).join(" · "),
|
|
678
|
+
)}</small></div>
|
|
679
|
+
<b>${gate.values.map((value) => finite(value)).join(", ")}</b>
|
|
680
|
+
</article>`,
|
|
681
|
+
)
|
|
682
|
+
.join("");
|
|
683
|
+
const lineage = target
|
|
684
|
+
? `<article class="lineage-card">
|
|
685
|
+
<span>${escapeHtml(target.id)}</span>
|
|
686
|
+
<small>from ${escapeHtml(target.parentIds.join(" + ") || "root")} · ${escapeHtml(
|
|
687
|
+
target.proposer,
|
|
688
|
+
)} · ${compactNumber(target.sourceBytes)} bytes</small>
|
|
689
|
+
${target.rationale ? `<p>${escapeHtml(target.rationale)}</p>` : ""}
|
|
690
|
+
<div>${target.components
|
|
691
|
+
.map((component) => `<code>${escapeHtml(component)}</code>`)
|
|
692
|
+
.join("")}</div>
|
|
693
|
+
</article>`
|
|
694
|
+
: "";
|
|
695
|
+
return `
|
|
696
|
+
<header class="detail-header">
|
|
697
|
+
<div class="detail-kicker">${badge(item.status)}<span class="detail-path">${escapeHtml(
|
|
698
|
+
item.path,
|
|
699
|
+
)}</span></div>
|
|
700
|
+
<div class="detail-title-row">
|
|
701
|
+
<div><h2>${escapeHtml(item.campaignId)}</h2><p class="detail-subtitle">${escapeHtml(
|
|
702
|
+
`${item.mode} · ${fullDate(item.startedAt)} · ${item.experimentId}`,
|
|
703
|
+
)}</p></div>
|
|
704
|
+
<div class="score-block"><strong>${escapeHtml(
|
|
705
|
+
decision?.outcome ? statusLabel(decision.outcome) : item.candidateCount,
|
|
706
|
+
)}</strong><small>${decision?.outcome ? "verified decision" : "candidates"}</small></div>
|
|
707
|
+
</div>
|
|
708
|
+
</header>
|
|
709
|
+
<div class="detail-body">
|
|
710
|
+
${
|
|
711
|
+
payload.integrity.verified
|
|
712
|
+
? '<div class="integrity-banner">✓ Candidate ledger and accounting verified</div>'
|
|
713
|
+
: `<div class="attention-banner">Integrity unavailable: ${escapeHtml(
|
|
714
|
+
payload.integrity.errors.join(" · "),
|
|
715
|
+
)}</div>`
|
|
716
|
+
}
|
|
717
|
+
<dl class="detail-stats">
|
|
718
|
+
<div><dt>Champion</dt><dd>${escapeHtml(experiment.championCandidateId ?? "—")}</dd></div>
|
|
719
|
+
<div><dt>Active time</dt><dd>${escapeHtml(duration(experiment.activeTimeMs))}</dd></div>
|
|
720
|
+
<div><dt>Judge calls</dt><dd>${experiment.budgetUsage.judgeCalls}</dd></div>
|
|
721
|
+
<div><dt>Spend</dt><dd>$${experiment.budgetUsage.costUsd.toFixed(2)}</dd></div>
|
|
722
|
+
</dl>
|
|
723
|
+
<div class="experiment-visuals">
|
|
724
|
+
${candidateLandscape(experiment.evaluations, baseline?.id, targetId)}
|
|
725
|
+
${budgetUtilization(decision)}
|
|
726
|
+
</div>
|
|
727
|
+
${
|
|
728
|
+
decision
|
|
729
|
+
? `<section class="detail-section">
|
|
730
|
+
<h3>Promotion gates <span>${decision.gates.filter((gate) => gate.passed).length}/${decision.gates.length} passed</span></h3>
|
|
731
|
+
<div class="gate-list">${gates || '<div class="artifact"><span>No completed gates</span><small>incomplete</small></div>'}</div>
|
|
732
|
+
<div class="decision-meta">
|
|
733
|
+
<span>Ranking</span><strong>${escapeHtml(decision.ranking.join(" → ") || "—")}</strong>
|
|
734
|
+
<span>Eligible</span><strong>${escapeHtml(
|
|
735
|
+
decision.eligibleCandidateIds.join(", ") || "none",
|
|
736
|
+
)}</strong>
|
|
737
|
+
</div>
|
|
738
|
+
</section>`
|
|
739
|
+
: ""
|
|
740
|
+
}
|
|
741
|
+
<section class="detail-section">
|
|
742
|
+
<div class="section-heading">
|
|
743
|
+
<h3>Candidate comparison <span>baseline → target</span></h3>
|
|
744
|
+
${
|
|
745
|
+
options
|
|
746
|
+
? `<label class="candidate-picker">Target <select data-candidate-target>${options}</select></label>`
|
|
747
|
+
: ""
|
|
748
|
+
}
|
|
749
|
+
</div>
|
|
750
|
+
${lineage}
|
|
751
|
+
<div class="comparison-scopes">
|
|
752
|
+
${experimentScopes(baselineEvaluations, targetEvaluations) || '<div class="artifact"><span>No comparable evaluations</span><small>empty</small></div>'}
|
|
753
|
+
</div>
|
|
754
|
+
</section>
|
|
755
|
+
${
|
|
756
|
+
failedEvidence(targetEvaluations)
|
|
757
|
+
? `<section class="detail-section"><h3>Failed evidence <span>judge feedback</span></h3><div class="failure-list">${failedEvidence(
|
|
758
|
+
targetEvaluations,
|
|
759
|
+
)}</div></section>`
|
|
760
|
+
: ""
|
|
761
|
+
}
|
|
762
|
+
</div>`;
|
|
763
|
+
};
|
|
764
|
+
|
|
765
|
+
const detailAssertionMap = (detail) =>
|
|
766
|
+
new Map(
|
|
767
|
+
(detail.grading?.assertions ?? []).map((assertion) => [
|
|
768
|
+
assertion.id ?? assertion.name,
|
|
769
|
+
assertion,
|
|
770
|
+
]),
|
|
771
|
+
);
|
|
772
|
+
|
|
773
|
+
const assertionDeltaRows = (payload) => {
|
|
774
|
+
const before = detailAssertionMap(payload.before);
|
|
775
|
+
const after = detailAssertionMap(payload.after);
|
|
776
|
+
return payload.comparison.grading.deltas
|
|
777
|
+
.map((delta) => {
|
|
778
|
+
const beforeAssertion = before.get(delta.id);
|
|
779
|
+
const afterAssertion = after.get(delta.id);
|
|
780
|
+
return `<tr>
|
|
781
|
+
<td><strong>${escapeHtml(delta.name)}</strong><small>${escapeHtml(delta.type)}</small></td>
|
|
782
|
+
<td>${escapeHtml(percentage(delta.before?.score))}</td>
|
|
783
|
+
<td>${escapeHtml(percentage(delta.after?.score))}</td>
|
|
784
|
+
<td><span class="delta-badge ${escapeHtml(delta.status)}">${escapeHtml(
|
|
785
|
+
delta.status,
|
|
786
|
+
)}</span></td>
|
|
787
|
+
<td class="rationale-cell">
|
|
788
|
+
<span>${escapeHtml(beforeAssertion?.comment ?? "—")}</span>
|
|
789
|
+
<b>→</b>
|
|
790
|
+
<span>${escapeHtml(afterAssertion?.comment ?? "—")}</span>
|
|
791
|
+
</td>
|
|
792
|
+
</tr>`;
|
|
793
|
+
})
|
|
794
|
+
.join("");
|
|
795
|
+
};
|
|
796
|
+
|
|
797
|
+
const artifactDelta = (payload) => {
|
|
798
|
+
const groups = [
|
|
799
|
+
["changed", payload.comparison.artifactsDelta.changed],
|
|
800
|
+
["added", payload.comparison.artifactsDelta.added],
|
|
801
|
+
["removed", payload.comparison.artifactsDelta.removed],
|
|
802
|
+
];
|
|
803
|
+
return groups
|
|
804
|
+
.flatMap(([status, paths]) =>
|
|
805
|
+
paths.map((path) => {
|
|
806
|
+
const clickable = status === "changed" || status === "added" || status === "removed";
|
|
807
|
+
const content = `<span>${escapeHtml(path)}</span><small>${escapeHtml(status)}${
|
|
808
|
+
clickable ? " →" : ""
|
|
809
|
+
}</small>`;
|
|
810
|
+
return clickable
|
|
811
|
+
? `<button class="artifact artifact-button ${escapeHtml(
|
|
812
|
+
status,
|
|
813
|
+
)}" type="button" data-compare-artifact="${escapeHtml(path)}">${content}</button>`
|
|
814
|
+
: `<div class="artifact">${content}</div>`;
|
|
815
|
+
}),
|
|
816
|
+
)
|
|
817
|
+
.join("");
|
|
818
|
+
};
|
|
819
|
+
|
|
820
|
+
const signedMetricValue = (value, unit) => {
|
|
821
|
+
if (typeof value !== "number") return "—";
|
|
822
|
+
if (unit === "milliseconds") return signedDuration(value);
|
|
823
|
+
if (unit === "ratio" || unit === "score") return signedPercentage(value);
|
|
824
|
+
return `${value > 0 ? "+" : ""}${metricValue(value, unit)}`;
|
|
825
|
+
};
|
|
826
|
+
|
|
827
|
+
const driverMetricComparisons = (runDelta) => {
|
|
828
|
+
const before = runDelta?.driverMetrics?.a ?? {};
|
|
829
|
+
const after = runDelta?.driverMetrics?.b ?? {};
|
|
830
|
+
const names = [...new Set([...Object.keys(before), ...Object.keys(after)])].sort();
|
|
831
|
+
return names
|
|
832
|
+
.map((name) => {
|
|
833
|
+
const a = before[name];
|
|
834
|
+
const b = after[name];
|
|
835
|
+
const unit = a?.unit ?? b?.unit ?? "number";
|
|
836
|
+
const comparable = a && b && a.unit === b.unit;
|
|
837
|
+
const delta = comparable ? b.value - a.value : null;
|
|
838
|
+
const better = b?.better ?? a?.better ?? "neutral";
|
|
839
|
+
const tone =
|
|
840
|
+
better === "neutral" ? "metric-neutral" : toneForDelta(delta, better === "lower");
|
|
841
|
+
return comparisonBars({
|
|
842
|
+
title: metricName(name, b?.label ?? a?.label),
|
|
843
|
+
before: a?.value,
|
|
844
|
+
after: b?.value,
|
|
845
|
+
beforeText: metricValue(a?.value, unit),
|
|
846
|
+
afterText: metricValue(b?.value, unit),
|
|
847
|
+
deltaText: comparable ? signedMetricValue(delta, unit) : "not paired",
|
|
848
|
+
deltaTone: tone,
|
|
849
|
+
});
|
|
850
|
+
})
|
|
851
|
+
.join("");
|
|
852
|
+
};
|
|
853
|
+
|
|
854
|
+
const subagentTotals = (activities) => {
|
|
855
|
+
const subagents = (activities ?? []).filter((activity) => activity.agent?.role === "subagent");
|
|
856
|
+
return {
|
|
857
|
+
count: subagents.length,
|
|
858
|
+
toolCalls: subagents.reduce((sum, activity) => sum + (activity.toolCalls?.total ?? 0), 0),
|
|
859
|
+
tokens: subagents.reduce((sum, activity) => sum + (activity.usage?.total ?? 0), 0),
|
|
860
|
+
durationMs: subagents.reduce((sum, activity) => sum + (activity.durationMs ?? 0), 0),
|
|
861
|
+
};
|
|
862
|
+
};
|
|
863
|
+
|
|
864
|
+
export const renderRunComparison = (payload, labels) => {
|
|
865
|
+
const comparison = payload.comparison;
|
|
866
|
+
const runDelta = comparison.runDelta;
|
|
867
|
+
const durationDelta =
|
|
868
|
+
typeof runDelta?.durationMsA === "number" && typeof runDelta?.durationMsB === "number"
|
|
869
|
+
? runDelta.durationMsB - runDelta.durationMsA
|
|
870
|
+
: null;
|
|
871
|
+
const mutationA = runDelta?.actionTiming?.a?.timeToFirstMutationMs;
|
|
872
|
+
const mutationB = runDelta?.actionTiming?.b?.timeToFirstMutationMs;
|
|
873
|
+
const mutationDelta =
|
|
874
|
+
typeof mutationA === "number" && typeof mutationB === "number" ? mutationB - mutationA : null;
|
|
875
|
+
const tokenA = totalTokens(runDelta?.usage?.a?.agent);
|
|
876
|
+
const tokenB = totalTokens(runDelta?.usage?.b?.agent);
|
|
877
|
+
const tokenDelta = tokenB - tokenA;
|
|
878
|
+
const scoreDelta =
|
|
879
|
+
typeof comparison.grading.caseScoreA === "number" &&
|
|
880
|
+
typeof comparison.grading.caseScoreB === "number"
|
|
881
|
+
? comparison.grading.caseScoreB - comparison.grading.caseScoreA
|
|
882
|
+
: null;
|
|
883
|
+
const driverMetrics = driverMetricComparisons(runDelta);
|
|
884
|
+
const subagentsA = subagentTotals(runDelta?.agentActivity?.a);
|
|
885
|
+
const subagentsB = subagentTotals(runDelta?.agentActivity?.b);
|
|
886
|
+
return `
|
|
887
|
+
<header class="detail-header comparison-header">
|
|
888
|
+
<div class="detail-kicker">${badge(
|
|
889
|
+
comparison.grading.regressions ? "failed" : "passed",
|
|
890
|
+
)}<span>run comparison</span></div>
|
|
891
|
+
<div class="detail-title-row">
|
|
892
|
+
<div><h2>${escapeHtml(labels.before)} → ${escapeHtml(
|
|
893
|
+
labels.after,
|
|
894
|
+
)}</h2><p class="detail-subtitle">Baseline A compared with candidate B</p></div>
|
|
895
|
+
<button class="secondary-button" type="button" data-close-compare>Done</button>
|
|
896
|
+
</div>
|
|
897
|
+
</header>
|
|
898
|
+
<div class="detail-body">
|
|
899
|
+
${
|
|
900
|
+
comparison.caseMismatch.length
|
|
901
|
+
? `<div class="attention-banner">Evidence identity differs: ${escapeHtml(
|
|
902
|
+
comparison.caseMismatch.map((entry) => entry.field).join(", "),
|
|
903
|
+
)}. Treat deltas as potentially incomparable.</div>`
|
|
904
|
+
: '<div class="integrity-banner">✓ Case, evaluator, runtime, and driver identities are comparable</div>'
|
|
905
|
+
}
|
|
906
|
+
<dl class="comparison-stats">
|
|
907
|
+
<div><dt>Case score</dt><dd>${escapeHtml(
|
|
908
|
+
`${percentage(comparison.grading.caseScoreA)} → ${percentage(
|
|
909
|
+
comparison.grading.caseScoreB,
|
|
910
|
+
)}`,
|
|
911
|
+
)}</dd><small class="${toneForDelta(scoreDelta)}">${escapeHtml(
|
|
912
|
+
signedPercentage(scoreDelta),
|
|
913
|
+
)}</small></div>
|
|
914
|
+
<div><dt>Duration</dt><dd>${escapeHtml(
|
|
915
|
+
`${duration(runDelta?.durationMsA)} → ${duration(runDelta?.durationMsB)}`,
|
|
916
|
+
)}</dd><small class="${toneForDelta(durationDelta, true)}">${escapeHtml(
|
|
917
|
+
signedDuration(durationDelta),
|
|
918
|
+
)}</small></div>
|
|
919
|
+
<div><dt>First mutation</dt><dd>${escapeHtml(
|
|
920
|
+
`${duration(mutationA)} → ${duration(mutationB)}`,
|
|
921
|
+
)}</dd><small class="${toneForDelta(mutationDelta, true)}">${escapeHtml(
|
|
922
|
+
signedDuration(mutationDelta),
|
|
923
|
+
)}</small></div>
|
|
924
|
+
<div><dt>Agent tokens</dt><dd>${escapeHtml(
|
|
925
|
+
`${compactNumber(tokenA)} → ${compactNumber(tokenB)}`,
|
|
926
|
+
)}</dd><small class="${toneForDelta(tokenDelta, true)}">${tokenDelta > 0 ? "+" : ""}${escapeHtml(
|
|
927
|
+
compactNumber(tokenDelta),
|
|
928
|
+
)}</small></div>
|
|
929
|
+
<div><dt>Assertions</dt><dd>${comparison.grading.improvements} ↑ / ${comparison.grading.regressions} ↓</dd><small>${comparison.grading.deltas.length} compared</small></div>
|
|
930
|
+
</dl>
|
|
931
|
+
<section class="detail-section">
|
|
932
|
+
<h3>Comparison profile <span>shared scales within each measure</span></h3>
|
|
933
|
+
<div class="comparison-plot-grid">
|
|
934
|
+
${comparisonBars({
|
|
935
|
+
title: "Case score",
|
|
936
|
+
before: comparison.grading.caseScoreA,
|
|
937
|
+
after: comparison.grading.caseScoreB,
|
|
938
|
+
maximum: 1,
|
|
939
|
+
beforeText: percentage(comparison.grading.caseScoreA),
|
|
940
|
+
afterText: percentage(comparison.grading.caseScoreB),
|
|
941
|
+
deltaText: signedPercentage(scoreDelta),
|
|
942
|
+
deltaTone: toneForDelta(scoreDelta),
|
|
943
|
+
})}
|
|
944
|
+
${comparisonBars({
|
|
945
|
+
title: "Duration",
|
|
946
|
+
before: runDelta?.durationMsA,
|
|
947
|
+
after: runDelta?.durationMsB,
|
|
948
|
+
beforeText: duration(runDelta?.durationMsA),
|
|
949
|
+
afterText: duration(runDelta?.durationMsB),
|
|
950
|
+
deltaText: signedDuration(durationDelta),
|
|
951
|
+
deltaTone: toneForDelta(durationDelta, true),
|
|
952
|
+
})}
|
|
953
|
+
${comparisonBars({
|
|
954
|
+
title: "Time to first mutation",
|
|
955
|
+
before: mutationA,
|
|
956
|
+
after: mutationB,
|
|
957
|
+
beforeText: duration(mutationA),
|
|
958
|
+
afterText: duration(mutationB),
|
|
959
|
+
deltaText: signedDuration(mutationDelta),
|
|
960
|
+
deltaTone: toneForDelta(mutationDelta, true),
|
|
961
|
+
})}
|
|
962
|
+
${comparisonBars({
|
|
963
|
+
title: "Agent tokens",
|
|
964
|
+
before: tokenA,
|
|
965
|
+
after: tokenB,
|
|
966
|
+
beforeText: compactNumber(tokenA),
|
|
967
|
+
afterText: compactNumber(tokenB),
|
|
968
|
+
deltaText: `${tokenDelta > 0 ? "+" : ""}${compactNumber(tokenDelta)}`,
|
|
969
|
+
deltaTone: toneForDelta(tokenDelta, true),
|
|
970
|
+
})}
|
|
971
|
+
${comparisonBars({
|
|
972
|
+
title: "Subagents spawned",
|
|
973
|
+
before: subagentsA.count,
|
|
974
|
+
after: subagentsB.count,
|
|
975
|
+
beforeText: compactNumber(subagentsA.count),
|
|
976
|
+
afterText: compactNumber(subagentsB.count),
|
|
977
|
+
deltaText: `${subagentsB.count - subagentsA.count > 0 ? "+" : ""}${compactNumber(
|
|
978
|
+
subagentsB.count - subagentsA.count,
|
|
979
|
+
)}`,
|
|
980
|
+
deltaTone: "metric-neutral",
|
|
981
|
+
})}
|
|
982
|
+
${comparisonBars({
|
|
983
|
+
title: "Subagent tool calls",
|
|
984
|
+
before: subagentsA.toolCalls,
|
|
985
|
+
after: subagentsB.toolCalls,
|
|
986
|
+
beforeText: compactNumber(subagentsA.toolCalls),
|
|
987
|
+
afterText: compactNumber(subagentsB.toolCalls),
|
|
988
|
+
deltaText: `${subagentsB.toolCalls - subagentsA.toolCalls > 0 ? "+" : ""}${compactNumber(
|
|
989
|
+
subagentsB.toolCalls - subagentsA.toolCalls,
|
|
990
|
+
)}`,
|
|
991
|
+
deltaTone: "metric-neutral",
|
|
992
|
+
})}
|
|
993
|
+
${comparisonBars({
|
|
994
|
+
title: "Total subagent time",
|
|
995
|
+
before: subagentsA.durationMs,
|
|
996
|
+
after: subagentsB.durationMs,
|
|
997
|
+
beforeText: duration(subagentsA.durationMs),
|
|
998
|
+
afterText: duration(subagentsB.durationMs),
|
|
999
|
+
deltaText: signedDuration(subagentsB.durationMs - subagentsA.durationMs),
|
|
1000
|
+
deltaTone: "metric-neutral",
|
|
1001
|
+
})}
|
|
1002
|
+
${comparisonBars({
|
|
1003
|
+
title: "Subagent tokens",
|
|
1004
|
+
before: subagentsA.tokens,
|
|
1005
|
+
after: subagentsB.tokens,
|
|
1006
|
+
beforeText: compactNumber(subagentsA.tokens),
|
|
1007
|
+
afterText: compactNumber(subagentsB.tokens),
|
|
1008
|
+
deltaText: `${subagentsB.tokens - subagentsA.tokens > 0 ? "+" : ""}${compactNumber(
|
|
1009
|
+
subagentsB.tokens - subagentsA.tokens,
|
|
1010
|
+
)}`,
|
|
1011
|
+
deltaTone: toneForDelta(subagentsB.tokens - subagentsA.tokens, true),
|
|
1012
|
+
})}
|
|
1013
|
+
</div>
|
|
1014
|
+
<div class="comparison-distributions">
|
|
1015
|
+
${compositionFigure(
|
|
1016
|
+
"Assertion movement",
|
|
1017
|
+
[
|
|
1018
|
+
{ key: "improved", label: "Improved", value: comparison.grading.improvements },
|
|
1019
|
+
{ key: "regressed", label: "Regressed", value: comparison.grading.regressions },
|
|
1020
|
+
{
|
|
1021
|
+
key: "added",
|
|
1022
|
+
label: "Added",
|
|
1023
|
+
value: comparison.grading.deltas.filter((delta) => delta.status === "added").length,
|
|
1024
|
+
},
|
|
1025
|
+
{
|
|
1026
|
+
key: "removed",
|
|
1027
|
+
label: "Removed",
|
|
1028
|
+
value: comparison.grading.deltas.filter((delta) => delta.status === "removed")
|
|
1029
|
+
.length,
|
|
1030
|
+
},
|
|
1031
|
+
{
|
|
1032
|
+
key: "unchanged",
|
|
1033
|
+
label: "Unchanged",
|
|
1034
|
+
value: comparison.grading.deltas.filter((delta) => delta.status === "unchanged")
|
|
1035
|
+
.length,
|
|
1036
|
+
},
|
|
1037
|
+
],
|
|
1038
|
+
"Distribution of assertion changes",
|
|
1039
|
+
)}
|
|
1040
|
+
${compositionFigure(
|
|
1041
|
+
"Artifact movement",
|
|
1042
|
+
[
|
|
1043
|
+
{
|
|
1044
|
+
key: "improved",
|
|
1045
|
+
label: "Added",
|
|
1046
|
+
value: comparison.artifactsDelta.added.length,
|
|
1047
|
+
},
|
|
1048
|
+
{
|
|
1049
|
+
key: "regressed",
|
|
1050
|
+
label: "Removed",
|
|
1051
|
+
value: comparison.artifactsDelta.removed.length,
|
|
1052
|
+
},
|
|
1053
|
+
{
|
|
1054
|
+
key: "changed",
|
|
1055
|
+
label: "Changed",
|
|
1056
|
+
value: comparison.artifactsDelta.changed.length,
|
|
1057
|
+
},
|
|
1058
|
+
{
|
|
1059
|
+
key: "unchanged",
|
|
1060
|
+
label: "Unchanged",
|
|
1061
|
+
value: comparison.artifactsDelta.unchanged.length,
|
|
1062
|
+
},
|
|
1063
|
+
],
|
|
1064
|
+
"Distribution of artifact changes",
|
|
1065
|
+
)}
|
|
1066
|
+
</div>
|
|
1067
|
+
</section>
|
|
1068
|
+
${
|
|
1069
|
+
driverMetrics
|
|
1070
|
+
? `<section class="detail-section"><h3>Driver metric comparison <span>shared scale per metric</span></h3><div class="comparison-plot-grid">${driverMetrics}</div></section>`
|
|
1071
|
+
: ""
|
|
1072
|
+
}
|
|
1073
|
+
<section class="detail-section">
|
|
1074
|
+
<h3>Judgment delta <span>rationale side by side</span></h3>
|
|
1075
|
+
<div class="matrix-wrap"><table class="matrix comparison-matrix">
|
|
1076
|
+
<thead><tr><th>Assertion</th><th>A</th><th>B</th><th>Change</th><th>Judge rationale A → B</th></tr></thead>
|
|
1077
|
+
<tbody>${assertionDeltaRows(payload) || '<tr><td colspan="5">No assertions.</td></tr>'}</tbody>
|
|
1078
|
+
</table></div>
|
|
1079
|
+
</section>
|
|
1080
|
+
<section class="detail-section">
|
|
1081
|
+
<h3>Artifacts <span>${comparison.artifactsDelta.changed.length} changed · ${comparison.artifactsDelta.added.length} added · ${comparison.artifactsDelta.removed.length} removed</span></h3>
|
|
1082
|
+
<div class="artifact-list">${artifactDelta(payload) || '<div class="artifact"><span>No artifact changes</span><small>unchanged</small></div>'}</div>
|
|
1083
|
+
</section>
|
|
1084
|
+
</div>`;
|
|
1085
|
+
};
|
|
1086
|
+
|
|
1087
|
+
const metricDeltaSummary = (metrics) =>
|
|
1088
|
+
Object.entries(metrics)
|
|
1089
|
+
.map(
|
|
1090
|
+
([name, summary]) =>
|
|
1091
|
+
`<span><b>${escapeHtml(name)}</b> ${escapeHtml(
|
|
1092
|
+
metricValue(summary.mean, inferredMetricUnit(name, summary.unit)),
|
|
1093
|
+
)}</span>`,
|
|
1094
|
+
)
|
|
1095
|
+
.join("");
|
|
1096
|
+
|
|
1097
|
+
export const renderSuiteComparison = (payload, labels) => {
|
|
1098
|
+
const comparison = payload.comparison;
|
|
1099
|
+
const movement = { improved: 0, regressed: 0, unchanged: 0, missing: 0 };
|
|
1100
|
+
for (const entry of comparison.cases) {
|
|
1101
|
+
if (!entry.before || !entry.after) movement.missing++;
|
|
1102
|
+
else if (entry.regressed) movement.regressed++;
|
|
1103
|
+
else if ((entry.passRateDelta ?? 0) > 0) movement.improved++;
|
|
1104
|
+
else movement.unchanged++;
|
|
1105
|
+
}
|
|
1106
|
+
return `
|
|
1107
|
+
<header class="detail-header comparison-header">
|
|
1108
|
+
<div class="detail-kicker">${badge(
|
|
1109
|
+
comparison.regressions || comparison.identityMismatches.length ? "failed" : "passed",
|
|
1110
|
+
)}<span>suite comparison</span></div>
|
|
1111
|
+
<div class="detail-title-row">
|
|
1112
|
+
<div><h2>${escapeHtml(labels.before)} → ${escapeHtml(
|
|
1113
|
+
labels.after,
|
|
1114
|
+
)}</h2><p class="detail-subtitle">Repeated evidence with paired case metrics</p></div>
|
|
1115
|
+
<button class="secondary-button" type="button" data-close-compare>Done</button>
|
|
1116
|
+
</div>
|
|
1117
|
+
</header>
|
|
1118
|
+
<div class="detail-body">
|
|
1119
|
+
${
|
|
1120
|
+
comparison.identityMismatches.length
|
|
1121
|
+
? `<div class="attention-banner">${comparison.identityMismatches.length} repetition identities differ; those pairs are not directly comparable.</div>`
|
|
1122
|
+
: '<div class="integrity-banner">✓ All paired repetition identities match</div>'
|
|
1123
|
+
}
|
|
1124
|
+
<dl class="comparison-stats">
|
|
1125
|
+
<div><dt>Pass rate</dt><dd>${escapeHtml(
|
|
1126
|
+
`${percentage(comparison.before.passRate)} → ${percentage(comparison.after.passRate)}`,
|
|
1127
|
+
)}</dd><small class="${toneForDelta(comparison.passRateDelta)}">${escapeHtml(
|
|
1128
|
+
signedPercentage(comparison.passRateDelta),
|
|
1129
|
+
)}</small></div>
|
|
1130
|
+
<div><dt>Attempts</dt><dd>${comparison.before.attempts} → ${comparison.after.attempts}</dd><small>paired populations</small></div>
|
|
1131
|
+
<div><dt>Regressions</dt><dd>${comparison.regressions}</dd><small>case-level</small></div>
|
|
1132
|
+
<div><dt>Identity drift</dt><dd>${comparison.identityMismatches.length}</dd><small>repetitions</small></div>
|
|
1133
|
+
</dl>
|
|
1134
|
+
<section class="detail-section">
|
|
1135
|
+
<h3>Outcome movement <span>shared 0–100% scale</span></h3>
|
|
1136
|
+
<div class="suite-outcome-visual">
|
|
1137
|
+
${comparisonBars({
|
|
1138
|
+
title: "Suite pass rate",
|
|
1139
|
+
before: comparison.before.passRate,
|
|
1140
|
+
after: comparison.after.passRate,
|
|
1141
|
+
maximum: 1,
|
|
1142
|
+
beforeText: percentage(comparison.before.passRate),
|
|
1143
|
+
afterText: percentage(comparison.after.passRate),
|
|
1144
|
+
deltaText: signedPercentage(comparison.passRateDelta),
|
|
1145
|
+
deltaTone: toneForDelta(comparison.passRateDelta),
|
|
1146
|
+
})}
|
|
1147
|
+
${compositionFigure(
|
|
1148
|
+
"Case movement",
|
|
1149
|
+
[
|
|
1150
|
+
{
|
|
1151
|
+
key: "improved",
|
|
1152
|
+
label: "Improved",
|
|
1153
|
+
value: movement.improved,
|
|
1154
|
+
},
|
|
1155
|
+
{
|
|
1156
|
+
key: "regressed",
|
|
1157
|
+
label: "Regressed",
|
|
1158
|
+
value: movement.regressed,
|
|
1159
|
+
},
|
|
1160
|
+
{
|
|
1161
|
+
key: "unchanged",
|
|
1162
|
+
label: "Unchanged",
|
|
1163
|
+
value: movement.unchanged,
|
|
1164
|
+
},
|
|
1165
|
+
{
|
|
1166
|
+
key: "pending",
|
|
1167
|
+
label: "Missing side",
|
|
1168
|
+
value: movement.missing,
|
|
1169
|
+
},
|
|
1170
|
+
],
|
|
1171
|
+
"Distribution of per-case suite movement",
|
|
1172
|
+
)}
|
|
1173
|
+
</div>
|
|
1174
|
+
</section>
|
|
1175
|
+
<section class="detail-section">
|
|
1176
|
+
<h3>Per-case delta <span>after − before</span></h3>
|
|
1177
|
+
<div class="matrix-wrap"><table class="matrix comparison-matrix">
|
|
1178
|
+
<thead><tr><th>Case</th><th>Before</th><th>After</th><th>Δ</th><th>Paired metric means</th></tr></thead>
|
|
1179
|
+
<tbody>${comparison.cases
|
|
1180
|
+
.map(
|
|
1181
|
+
(entry) => `<tr class="${entry.regressed ? "row-regressed" : ""}">
|
|
1182
|
+
<td>${escapeHtml(entry.canonicalCaseId)}</td>
|
|
1183
|
+
<td class="bar-table-cell">${quantizedBar(
|
|
1184
|
+
entry.before?.passRate,
|
|
1185
|
+
"tone-before",
|
|
1186
|
+
`${entry.canonicalCaseId} baseline pass rate ${percentage(
|
|
1187
|
+
entry.before?.passRate,
|
|
1188
|
+
)}`,
|
|
1189
|
+
)}<small>${escapeHtml(percentage(entry.before?.passRate))} · n=${
|
|
1190
|
+
entry.before?.attempts ?? 0
|
|
1191
|
+
}</small></td>
|
|
1192
|
+
<td class="bar-table-cell">${quantizedBar(
|
|
1193
|
+
entry.after?.passRate,
|
|
1194
|
+
"tone-after",
|
|
1195
|
+
`${entry.canonicalCaseId} candidate pass rate ${percentage(
|
|
1196
|
+
entry.after?.passRate,
|
|
1197
|
+
)}`,
|
|
1198
|
+
)}<small>${escapeHtml(percentage(entry.after?.passRate))} · n=${
|
|
1199
|
+
entry.after?.attempts ?? 0
|
|
1200
|
+
}</small></td>
|
|
1201
|
+
<td class="${toneForDelta(entry.passRateDelta)}">${escapeHtml(
|
|
1202
|
+
signedPercentage(entry.passRateDelta),
|
|
1203
|
+
)}</td>
|
|
1204
|
+
<td class="metric-deltas">${metricDeltaSummary(entry.pairedMetricDeltas)}</td>
|
|
1205
|
+
</tr>`,
|
|
1206
|
+
)
|
|
1207
|
+
.join("")}</tbody>
|
|
1208
|
+
</table></div>
|
|
1209
|
+
</section>
|
|
1210
|
+
</div>`;
|
|
1211
|
+
};
|
|
1212
|
+
|
|
1213
|
+
export { renderArtifactComparison, renderArtifactPreview } from "./artifact-view.js";
|
|
1214
|
+
export { detailEmpty };
|