@prismatic-io/lux 0.0.2-preview.3 → 0.0.2-preview.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (169) hide show
  1. package/lib/answerers/terminal/index.js +1 -1
  2. package/lib/answerers/terminal/index.js.map +1 -1
  3. package/lib/assertions/authoring.d.ts +2 -2
  4. package/lib/assertions/authoring.d.ts.map +1 -1
  5. package/lib/assertions/authoring.js.map +1 -1
  6. package/lib/assertions/core/helpers.d.ts +2 -7
  7. package/lib/assertions/core/helpers.d.ts.map +1 -1
  8. package/lib/assertions/core/helpers.js +2 -6
  9. package/lib/assertions/core/helpers.js.map +1 -1
  10. package/lib/assertions/core/index.d.ts.map +1 -1
  11. package/lib/assertions/core/index.js +2 -1
  12. package/lib/assertions/core/index.js.map +1 -1
  13. package/lib/assertions/core/resource-checks.d.ts +19 -0
  14. package/lib/assertions/core/resource-checks.d.ts.map +1 -1
  15. package/lib/assertions/core/resource-checks.js +32 -0
  16. package/lib/assertions/core/resource-checks.js.map +1 -1
  17. package/lib/cli/program.d.ts.map +1 -1
  18. package/lib/cli/program.js +12 -67
  19. package/lib/cli/program.js.map +1 -1
  20. package/lib/cli/render/reporter.d.ts.map +1 -1
  21. package/lib/cli/render/reporter.js +16 -3
  22. package/lib/cli/render/reporter.js.map +1 -1
  23. package/lib/cli/run-options.d.ts +1 -0
  24. package/lib/cli/run-options.d.ts.map +1 -1
  25. package/lib/cli/run-options.js.map +1 -1
  26. package/lib/cli/view.d.ts +13 -0
  27. package/lib/cli/view.d.ts.map +1 -0
  28. package/lib/cli/view.js +126 -0
  29. package/lib/cli/view.js.map +1 -0
  30. package/lib/core/action-events.d.ts +29 -0
  31. package/lib/core/action-events.d.ts.map +1 -0
  32. package/lib/core/action-events.js +51 -0
  33. package/lib/core/action-events.js.map +1 -0
  34. package/lib/core/artifact-evidence.d.ts +8 -0
  35. package/lib/core/artifact-evidence.d.ts.map +1 -1
  36. package/lib/core/artifact-evidence.js +36 -4
  37. package/lib/core/artifact-evidence.js.map +1 -1
  38. package/lib/core/assertion.d.ts +12 -0
  39. package/lib/core/assertion.d.ts.map +1 -1
  40. package/lib/core/assertion.js +13 -0
  41. package/lib/core/assertion.js.map +1 -1
  42. package/lib/core/driver-metrics.d.ts +98 -0
  43. package/lib/core/driver-metrics.d.ts.map +1 -0
  44. package/lib/core/driver-metrics.js +36 -0
  45. package/lib/core/driver-metrics.js.map +1 -0
  46. package/lib/core/events.d.ts +2 -0
  47. package/lib/core/events.d.ts.map +1 -1
  48. package/lib/core/events.js +3 -2
  49. package/lib/core/events.js.map +1 -1
  50. package/lib/core/experiment.d.ts +19 -0
  51. package/lib/core/experiment.d.ts.map +1 -1
  52. package/lib/core/experiment.js +2 -0
  53. package/lib/core/experiment.js.map +1 -1
  54. package/lib/core/index.d.ts +2 -0
  55. package/lib/core/index.d.ts.map +1 -1
  56. package/lib/core/index.js +2 -0
  57. package/lib/core/index.js.map +1 -1
  58. package/lib/core/run.d.ts +51 -0
  59. package/lib/core/run.d.ts.map +1 -1
  60. package/lib/core/run.js +6 -0
  61. package/lib/core/run.js.map +1 -1
  62. package/lib/core/wire.d.ts +1 -0
  63. package/lib/core/wire.d.ts.map +1 -1
  64. package/lib/core/wire.js +5 -1
  65. package/lib/core/wire.js.map +1 -1
  66. package/lib/index.d.ts +3 -2
  67. package/lib/index.d.ts.map +1 -1
  68. package/lib/index.js +3 -2
  69. package/lib/index.js.map +1 -1
  70. package/lib/orchestrator/compare.d.ts +9 -1
  71. package/lib/orchestrator/compare.d.ts.map +1 -1
  72. package/lib/orchestrator/compare.js +8 -0
  73. package/lib/orchestrator/compare.js.map +1 -1
  74. package/lib/orchestrator/config.d.ts.map +1 -1
  75. package/lib/orchestrator/config.js +0 -2
  76. package/lib/orchestrator/config.js.map +1 -1
  77. package/lib/orchestrator/experiment-contracts.d.ts +19 -0
  78. package/lib/orchestrator/experiment-contracts.d.ts.map +1 -1
  79. package/lib/orchestrator/experiment-evaluation.d.ts.map +1 -1
  80. package/lib/orchestrator/experiment-evaluation.js +17 -0
  81. package/lib/orchestrator/experiment-evaluation.js.map +1 -1
  82. package/lib/orchestrator/experiment-report.d.ts +19 -0
  83. package/lib/orchestrator/experiment-report.d.ts.map +1 -1
  84. package/lib/orchestrator/experiment-runs.d.ts +38 -0
  85. package/lib/orchestrator/experiment-runs.d.ts.map +1 -1
  86. package/lib/orchestrator/grader-experiment.d.ts +12 -0
  87. package/lib/orchestrator/grader-experiment.d.ts.map +1 -1
  88. package/lib/orchestrator/grading.d.ts +2 -1
  89. package/lib/orchestrator/grading.d.ts.map +1 -1
  90. package/lib/orchestrator/grading.js +16 -4
  91. package/lib/orchestrator/grading.js.map +1 -1
  92. package/lib/orchestrator/index.d.ts +1 -1
  93. package/lib/orchestrator/index.d.ts.map +1 -1
  94. package/lib/orchestrator/index.js +1 -1
  95. package/lib/orchestrator/index.js.map +1 -1
  96. package/lib/orchestrator/list-runs.d.ts +6 -1
  97. package/lib/orchestrator/list-runs.d.ts.map +1 -1
  98. package/lib/orchestrator/list-runs.js +15 -1
  99. package/lib/orchestrator/list-runs.js.map +1 -1
  100. package/lib/orchestrator/orchestrator.d.ts +3 -0
  101. package/lib/orchestrator/orchestrator.d.ts.map +1 -1
  102. package/lib/orchestrator/orchestrator.js +3 -0
  103. package/lib/orchestrator/orchestrator.js.map +1 -1
  104. package/lib/orchestrator/regrade.d.ts +6 -0
  105. package/lib/orchestrator/regrade.d.ts.map +1 -1
  106. package/lib/orchestrator/run-execution.d.ts.map +1 -1
  107. package/lib/orchestrator/run-execution.js +13 -3
  108. package/lib/orchestrator/run-execution.js.map +1 -1
  109. package/lib/orchestrator/suite-compare.d.ts +34 -0
  110. package/lib/orchestrator/suite-compare.d.ts.map +1 -1
  111. package/lib/orchestrator/suite-summary.d.ts +68 -0
  112. package/lib/orchestrator/suite-summary.d.ts.map +1 -1
  113. package/lib/orchestrator/suite-summary.js +22 -7
  114. package/lib/orchestrator/suite-summary.js.map +1 -1
  115. package/lib/previewer/artifacts.d.ts +25 -0
  116. package/lib/previewer/artifacts.d.ts.map +1 -0
  117. package/lib/previewer/artifacts.js +180 -0
  118. package/lib/previewer/artifacts.js.map +1 -0
  119. package/lib/previewer/server.d.ts +104 -0
  120. package/lib/previewer/server.d.ts.map +1 -0
  121. package/lib/previewer/server.js +541 -0
  122. package/lib/previewer/server.js.map +1 -0
  123. package/package.json +2 -2
  124. package/skills/lux/SKILL.md +7 -3
  125. package/skills/lux/references/cli.md +25 -2
  126. package/skills/lux/references/eval-authoring.md +158 -4
  127. package/skills/lux-answerer/SKILL.md +1 -1
  128. package/src/answerers/terminal/index.ts +1 -1
  129. package/src/assertions/authoring.ts +2 -0
  130. package/src/assertions/core/helpers.ts +3 -6
  131. package/src/assertions/core/index.ts +2 -0
  132. package/src/assertions/core/resource-checks.ts +35 -0
  133. package/src/cli/program.ts +14 -93
  134. package/src/cli/render/reporter.ts +18 -3
  135. package/src/cli/run-options.ts +1 -0
  136. package/src/cli/view.ts +168 -0
  137. package/src/core/action-events.ts +59 -0
  138. package/src/core/artifact-evidence.ts +47 -4
  139. package/src/core/assertion.ts +15 -0
  140. package/src/core/driver-metrics.ts +51 -0
  141. package/src/core/events.ts +3 -2
  142. package/src/core/experiment.ts +2 -0
  143. package/src/core/index.ts +2 -0
  144. package/src/core/run.ts +6 -0
  145. package/src/core/wire.ts +5 -1
  146. package/src/index.ts +25 -5
  147. package/src/orchestrator/compare.ts +12 -0
  148. package/src/orchestrator/config.ts +0 -2
  149. package/src/orchestrator/experiment-evaluation.ts +23 -0
  150. package/src/orchestrator/grading.ts +21 -2
  151. package/src/orchestrator/index.ts +1 -0
  152. package/src/orchestrator/list-runs.ts +25 -0
  153. package/src/orchestrator/orchestrator.ts +6 -0
  154. package/src/orchestrator/run-execution.ts +14 -1
  155. package/src/orchestrator/suite-summary.ts +36 -8
  156. package/src/previewer/artifacts.ts +221 -0
  157. package/src/previewer/server.ts +763 -0
  158. package/viewer/app.css +2022 -0
  159. package/viewer/app.js +601 -0
  160. package/viewer/artifact-view.js +48 -0
  161. package/viewer/charts.js +86 -0
  162. package/viewer/detail.js +1108 -0
  163. package/viewer/format.js +129 -0
  164. package/viewer/index.html +151 -0
  165. package/lib/answerers/scripted/index.d.ts +0 -8
  166. package/lib/answerers/scripted/index.d.ts.map +0 -1
  167. package/lib/answerers/scripted/index.js +0 -163
  168. package/lib/answerers/scripted/index.js.map +0 -1
  169. package/src/answerers/scripted/index.ts +0 -186
@@ -0,0 +1,763 @@
1
+ import { existsSync } from "node:fs";
2
+ import { readFile } from "node:fs/promises";
3
+ import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http";
4
+ import { basename, relative, resolve, sep } from "node:path";
5
+ import { fileURLToPath } from "node:url";
6
+ import { type DriverMetricDescriptor, type DriverMetrics, exec } from "../core/index.js";
7
+ import {
8
+ compareRuns,
9
+ compareSuiteRuns,
10
+ type ListedRun,
11
+ listExperimentRuns,
12
+ listRuns,
13
+ listSuiteRuns,
14
+ loadExperimentCandidates,
15
+ loadRunArtifacts,
16
+ loadRunDetail,
17
+ loadSuiteSummary,
18
+ loadVerifiedExperimentDecisionSnapshot,
19
+ } from "../orchestrator/index.js";
20
+ import { compareArtifactPreviews, loadArtifactPreview } from "./artifacts.js";
21
+
22
+ const LOOPBACK_HOST = "127.0.0.1";
23
+ const ASSET_ROOT = fileURLToPath(new URL("../../viewer/", import.meta.url));
24
+
25
+ type PreviewKind = "run" | "suite" | "experiment";
26
+ type PreviewStatus = "passed" | "failed" | "running" | "aborted" | "error" | "ungraded";
27
+
28
+ type PreviewRun = {
29
+ id: string;
30
+ kind: "run";
31
+ status: PreviewStatus;
32
+ caseId: string;
33
+ driver: string;
34
+ agentId: string | null;
35
+ startedAt: string | null;
36
+ durationMs: number | null;
37
+ timeToFirstActionMs: number | null;
38
+ timeToFirstMutationMs: number | null;
39
+ timeToFirstDurableMutationMs: number | null;
40
+ driverMetrics: DriverMetrics;
41
+ score: number | null;
42
+ assertions: { passed: number; total: number };
43
+ path: string;
44
+ };
45
+
46
+ type PreviewSuite = {
47
+ id: string;
48
+ kind: "suite";
49
+ status: PreviewStatus;
50
+ suiteId: string;
51
+ startedAt: string;
52
+ endedAt: string | null;
53
+ concurrency: number;
54
+ profileId?: string | undefined;
55
+ items: Array<{
56
+ caseId: string;
57
+ canonicalCaseId: string;
58
+ status: string;
59
+ exitReason: string | null;
60
+ casePassed: boolean | null;
61
+ runId: string | null;
62
+ errorMessage?: string | undefined;
63
+ }>;
64
+ path: string;
65
+ };
66
+
67
+ type PreviewExperiment = {
68
+ id: string;
69
+ kind: "experiment";
70
+ status: PreviewStatus;
71
+ experimentId: string;
72
+ campaignId: string;
73
+ mode: "experiment" | "optimize";
74
+ startedAt: string;
75
+ endedAt: string | null;
76
+ activeTimeMs: number;
77
+ candidateCount: number;
78
+ championCandidateId: string | null;
79
+ budget: {
80
+ metricCalls: number;
81
+ judgeCalls: number;
82
+ proposalCalls: number;
83
+ costUsd: number;
84
+ };
85
+ evaluations: Array<{
86
+ candidateId: string;
87
+ profileId: string;
88
+ split: "train" | "validation" | "test";
89
+ passRate: number;
90
+ metrics: Record<string, number>;
91
+ metricDefinitions: Record<string, DriverMetricDescriptor>;
92
+ failedCases: Array<{ caseId: string; score: number | null; feedback: string[] }>;
93
+ }>;
94
+ path: string;
95
+ };
96
+
97
+ export type PreviewSnapshot = {
98
+ generatedAt: string;
99
+ runsRoot: string;
100
+ initial: { kind: PreviewKind; id: string } | null;
101
+ runs: PreviewRun[];
102
+ suites: PreviewSuite[];
103
+ experiments: PreviewExperiment[];
104
+ };
105
+
106
+ type PreviewAssets = {
107
+ html: Buffer;
108
+ files: Map<string, { body: Buffer; contentType: string }>;
109
+ };
110
+
111
+ export type PreviewServer = {
112
+ url: string;
113
+ closed: Promise<void>;
114
+ close: () => Promise<void>;
115
+ };
116
+
117
+ export type StartPreviewServerOptions = {
118
+ runsRoot: string;
119
+ port?: number | undefined;
120
+ initialPath?: string | undefined;
121
+ initialKind?: PreviewKind | undefined;
122
+ caseFilter?: string | undefined;
123
+ signal?: AbortSignal | undefined;
124
+ };
125
+
126
+ const opaqueId = (runsRoot: string, path: string): string =>
127
+ Buffer.from(relative(resolve(runsRoot), resolve(path))).toString("base64url");
128
+
129
+ const displayPath = (runsRoot: string, path: string): string => {
130
+ const value = relative(resolve(runsRoot), resolve(path));
131
+ return value.split(sep).join("/");
132
+ };
133
+
134
+ const runStatus = (run: ListedRun): PreviewStatus => {
135
+ if (run.exitReason && run.exitReason !== "done") return "error";
136
+ if (run.casePassed === true) return "passed";
137
+ if (run.casePassed === false) return "failed";
138
+ return "ungraded";
139
+ };
140
+
141
+ const normalizeStatus = (status: string): PreviewStatus => {
142
+ if (status === "passed" || status === "failed" || status === "running" || status === "aborted") {
143
+ return status;
144
+ }
145
+ return "ungraded";
146
+ };
147
+
148
+ const buildSnapshot = async (
149
+ runsRoot: string,
150
+ options: Pick<StartPreviewServerOptions, "initialPath" | "initialKind" | "caseFilter">,
151
+ ): Promise<PreviewSnapshot> => {
152
+ const absoluteRunsRoot = resolve(runsRoot);
153
+ const [listedRuns, listedSuites, listedExperiments] = await Promise.all([
154
+ listRuns(absoluteRunsRoot),
155
+ listSuiteRuns(absoluteRunsRoot),
156
+ listExperimentRuns(absoluteRunsRoot),
157
+ ]);
158
+ const runs = listedRuns
159
+ .filter((run) => !options.caseFilter || run.caseSlug === options.caseFilter)
160
+ .map(
161
+ (run): PreviewRun => ({
162
+ id: opaqueId(absoluteRunsRoot, run.runDir),
163
+ kind: "run",
164
+ status: runStatus(run),
165
+ caseId: run.caseSlug,
166
+ driver: run.driverName,
167
+ agentId: run.driverAgentId,
168
+ startedAt: run.startedAt,
169
+ durationMs: run.durationMs,
170
+ timeToFirstActionMs: run.timeToFirstActionMs,
171
+ timeToFirstMutationMs: run.timeToFirstMutationMs,
172
+ timeToFirstDurableMutationMs: run.timeToFirstDurableMutationMs,
173
+ driverMetrics: run.driverMetrics,
174
+ score: run.caseScore,
175
+ assertions: { passed: run.assertionPassed, total: run.assertionTotal },
176
+ path: displayPath(absoluteRunsRoot, run.runDir),
177
+ }),
178
+ );
179
+ const runIdsByPath = new Map(
180
+ listedRuns.map((run) => [resolve(run.runDir), opaqueId(absoluteRunsRoot, run.runDir)]),
181
+ );
182
+ const suites = listedSuites.map(
183
+ (suite): PreviewSuite => ({
184
+ id: opaqueId(absoluteRunsRoot, suite.suiteDir),
185
+ kind: "suite",
186
+ status: normalizeStatus(suite.status),
187
+ suiteId: suite.suiteId,
188
+ startedAt: suite.startedAt,
189
+ endedAt: suite.endedAt,
190
+ concurrency: suite.concurrency,
191
+ ...(suite.profileId ? { profileId: suite.profileId } : {}),
192
+ items: suite.items.map((item) => ({
193
+ caseId: item.caseId,
194
+ canonicalCaseId: item.canonicalCaseId,
195
+ status: item.status,
196
+ exitReason: item.exitReason,
197
+ casePassed: item.casePassed,
198
+ runId: item.runDir
199
+ ? (runIdsByPath.get(resolve(suite.suiteDir, item.runDir)) ?? null)
200
+ : null,
201
+ ...(item.errorMessage ? { errorMessage: item.errorMessage } : {}),
202
+ })),
203
+ path: displayPath(absoluteRunsRoot, suite.suiteDir),
204
+ }),
205
+ );
206
+ const experiments = listedExperiments.map(
207
+ (experiment): PreviewExperiment => ({
208
+ id: opaqueId(absoluteRunsRoot, experiment.experimentDir),
209
+ kind: "experiment",
210
+ status: normalizeStatus(experiment.status),
211
+ experimentId: experiment.experimentId,
212
+ campaignId: experiment.campaignId,
213
+ mode: experiment.mode,
214
+ startedAt: experiment.startedAt,
215
+ endedAt: experiment.endedAt,
216
+ activeTimeMs: experiment.activeTimeMs,
217
+ candidateCount: experiment.candidates.length,
218
+ championCandidateId: experiment.championCandidateId ?? null,
219
+ budget: {
220
+ metricCalls: experiment.budgetUsage.metricCalls,
221
+ judgeCalls: experiment.budgetUsage.judgeCalls,
222
+ proposalCalls: experiment.budgetUsage.proposalCalls,
223
+ costUsd: experiment.budgetUsage.costUsd,
224
+ },
225
+ evaluations: experiment.evaluations.map((evaluation) => ({
226
+ candidateId: evaluation.candidateId,
227
+ profileId: evaluation.profileId,
228
+ split: evaluation.split,
229
+ passRate: evaluation.passRate,
230
+ metrics: evaluation.metrics,
231
+ metricDefinitions: evaluation.metricDefinitions ?? {},
232
+ failedCases: evaluation.failedCases,
233
+ })),
234
+ path: displayPath(absoluteRunsRoot, experiment.experimentDir),
235
+ }),
236
+ );
237
+ const initialPath = options.initialPath ? resolve(options.initialPath) : null;
238
+ const initialKind = options.initialKind;
239
+ const candidates: Array<PreviewRun | PreviewSuite | PreviewExperiment> = [
240
+ ...runs,
241
+ ...suites,
242
+ ...experiments,
243
+ ];
244
+ let initialEntry: (typeof candidates)[number] | undefined;
245
+ if (initialPath) {
246
+ initialEntry = candidates.find(
247
+ (entry) =>
248
+ (!initialKind || entry.kind === initialKind) &&
249
+ resolve(absoluteRunsRoot, entry.path) === initialPath,
250
+ );
251
+ } else if (initialKind) {
252
+ initialEntry = candidates.find((entry) => entry.kind === initialKind);
253
+ }
254
+ return {
255
+ generatedAt: new Date().toISOString(),
256
+ runsRoot: absoluteRunsRoot,
257
+ initial: initialEntry ? { kind: initialEntry.kind, id: initialEntry.id } : null,
258
+ runs,
259
+ suites,
260
+ experiments,
261
+ };
262
+ };
263
+
264
+ const securityHeaders = {
265
+ "Content-Security-Policy":
266
+ "default-src 'self'; base-uri 'none'; connect-src 'self'; form-action 'none'; frame-ancestors 'none'; img-src 'self' data:; object-src 'none'; script-src 'self'; style-src 'self'",
267
+ "Cross-Origin-Opener-Policy": "same-origin",
268
+ "Cross-Origin-Resource-Policy": "same-origin",
269
+ "Permissions-Policy": "camera=(), geolocation=(), microphone=()",
270
+ "Referrer-Policy": "no-referrer",
271
+ "X-Content-Type-Options": "nosniff",
272
+ "X-Frame-Options": "DENY",
273
+ };
274
+
275
+ const send = (
276
+ request: IncomingMessage,
277
+ response: ServerResponse,
278
+ status: number,
279
+ contentType: string,
280
+ body: Buffer | string,
281
+ cacheControl = "no-store",
282
+ ): void => {
283
+ const bytes = Buffer.isBuffer(body) ? body : Buffer.from(body);
284
+ response.writeHead(status, {
285
+ ...securityHeaders,
286
+ "Cache-Control": cacheControl,
287
+ "Content-Length": bytes.length,
288
+ "Content-Type": contentType,
289
+ });
290
+ response.end(request.method === "HEAD" ? undefined : bytes);
291
+ };
292
+
293
+ const sendJson = (
294
+ request: IncomingMessage,
295
+ response: ServerResponse,
296
+ status: number,
297
+ value: unknown,
298
+ ): void => {
299
+ send(request, response, status, "application/json; charset=utf-8", JSON.stringify(value));
300
+ };
301
+
302
+ const findRunById = async (runsRoot: string, id: string): Promise<ListedRun | null> => {
303
+ const runs = await listRuns(runsRoot);
304
+ return runs.find((run) => opaqueId(runsRoot, run.runDir) === id) ?? null;
305
+ };
306
+
307
+ const findRunsById = async (
308
+ runsRoot: string,
309
+ ids: readonly string[],
310
+ ): Promise<ListedRun[] | null> => {
311
+ const runs = await listRuns(runsRoot);
312
+ const found = ids.map((id) => runs.find((run) => opaqueId(runsRoot, run.runDir) === id) ?? null);
313
+ return found.every((run): run is ListedRun => run !== null) ? found : null;
314
+ };
315
+
316
+ const findSuiteById = async (
317
+ runsRoot: string,
318
+ id: string,
319
+ ): Promise<Awaited<ReturnType<typeof listSuiteRuns>>[number] | null> => {
320
+ const suites = await listSuiteRuns(runsRoot);
321
+ return suites.find((suite) => opaqueId(runsRoot, suite.suiteDir) === id) ?? null;
322
+ };
323
+
324
+ const findSuitesById = async (
325
+ runsRoot: string,
326
+ ids: readonly string[],
327
+ ): Promise<Awaited<ReturnType<typeof listSuiteRuns>> | null> => {
328
+ const suites = await listSuiteRuns(runsRoot);
329
+ const found = ids.map(
330
+ (id) => suites.find((suite) => opaqueId(runsRoot, suite.suiteDir) === id) ?? null,
331
+ );
332
+ return found.every(
333
+ (suite): suite is Awaited<ReturnType<typeof listSuiteRuns>>[number] => suite !== null,
334
+ )
335
+ ? found
336
+ : null;
337
+ };
338
+
339
+ const findExperimentById = async (
340
+ runsRoot: string,
341
+ id: string,
342
+ ): Promise<Awaited<ReturnType<typeof listExperimentRuns>>[number] | null> => {
343
+ const experiments = await listExperimentRuns(runsRoot);
344
+ return (
345
+ experiments.find((experiment) => opaqueId(runsRoot, experiment.experimentDir) === id) ?? null
346
+ );
347
+ };
348
+
349
+ const artifactPreview = async (run: ListedRun, path: string) => {
350
+ const [detail, indexed] = await Promise.all([
351
+ loadRunDetail(run.runDir, { maxArtifacts: 2_000 }),
352
+ loadRunArtifacts(run.runDir),
353
+ ]);
354
+ return loadArtifactPreview(run.runDir, indexed, detail.artifacts, path);
355
+ };
356
+
357
+ const publicRunDetail = async (runsRoot: string, runDir: string) => ({
358
+ ...(await loadRunDetail(runDir, { maxArtifacts: 2_000 })),
359
+ runDir: displayPath(runsRoot, runDir),
360
+ });
361
+
362
+ const requiredPair = (url: URL): [string, string] | null => {
363
+ const before = url.searchParams.get("before");
364
+ const after = url.searchParams.get("after");
365
+ return before && after && before !== after ? [before, after] : null;
366
+ };
367
+
368
+ const isLoopbackHost = (host: string | undefined): boolean =>
369
+ host === LOOPBACK_HOST || host?.startsWith(`${LOOPBACK_HOST}:`) === true;
370
+
371
+ type RouteContext = {
372
+ request: IncomingMessage;
373
+ response: ServerResponse;
374
+ url: URL;
375
+ runsRoot: string;
376
+ };
377
+
378
+ const handleRunComparison = async ({
379
+ request,
380
+ response,
381
+ url,
382
+ runsRoot,
383
+ }: RouteContext): Promise<void> => {
384
+ const pair = requiredPair(url);
385
+ if (!pair) {
386
+ sendJson(request, response, 400, { error: "two distinct run ids are required" });
387
+ return;
388
+ }
389
+ const runs = await findRunsById(runsRoot, pair);
390
+ if (!runs) {
391
+ sendJson(request, response, 404, { error: "run not found" });
392
+ return;
393
+ }
394
+ const [before, after] = runs;
395
+ if (!before || !after) throw new Error("run pair resolution failed");
396
+ const [comparison, beforeDetail, afterDetail] = await Promise.all([
397
+ compareRuns({ pathA: before.runDir, pathB: after.runDir }),
398
+ publicRunDetail(runsRoot, before.runDir),
399
+ publicRunDetail(runsRoot, after.runDir),
400
+ ]);
401
+ sendJson(request, response, 200, {
402
+ kind: "run",
403
+ comparison: {
404
+ ...comparison,
405
+ pathA: displayPath(runsRoot, before.runDir),
406
+ pathB: displayPath(runsRoot, after.runDir),
407
+ },
408
+ before: beforeDetail,
409
+ after: afterDetail,
410
+ });
411
+ };
412
+
413
+ const handleSuiteComparison = async ({
414
+ request,
415
+ response,
416
+ url,
417
+ runsRoot,
418
+ }: RouteContext): Promise<void> => {
419
+ const pair = requiredPair(url);
420
+ if (!pair) {
421
+ sendJson(request, response, 400, { error: "two distinct suite ids are required" });
422
+ return;
423
+ }
424
+ const suites = await findSuitesById(runsRoot, pair);
425
+ if (!suites) {
426
+ sendJson(request, response, 404, { error: "suite not found" });
427
+ return;
428
+ }
429
+ const [before, after] = suites;
430
+ if (!before || !after) throw new Error("suite pair resolution failed");
431
+ const [comparison, beforeSummary, afterSummary] = await Promise.all([
432
+ compareSuiteRuns(before.suiteDir, after.suiteDir),
433
+ loadSuiteSummary(before.suiteDir),
434
+ loadSuiteSummary(after.suiteDir),
435
+ ]);
436
+ sendJson(request, response, 200, {
437
+ kind: "suite",
438
+ comparison: {
439
+ ...comparison,
440
+ pathA: displayPath(runsRoot, before.suiteDir),
441
+ pathB: displayPath(runsRoot, after.suiteDir),
442
+ },
443
+ before: beforeSummary,
444
+ after: afterSummary,
445
+ });
446
+ };
447
+
448
+ const handleArtifactComparison = async ({
449
+ request,
450
+ response,
451
+ url,
452
+ runsRoot,
453
+ }: RouteContext): Promise<void> => {
454
+ const pair = requiredPair(url);
455
+ const path = url.searchParams.get("path");
456
+ if (!pair || !path) {
457
+ sendJson(request, response, 400, {
458
+ error: "two distinct run ids and an artifact path are required",
459
+ });
460
+ return;
461
+ }
462
+ const runs = await findRunsById(runsRoot, pair);
463
+ if (!runs) {
464
+ sendJson(request, response, 404, { error: "run not found" });
465
+ return;
466
+ }
467
+ const [before, after] = runs;
468
+ if (!before || !after) throw new Error("artifact run pair resolution failed");
469
+ const [beforePreview, afterPreview] = await Promise.all([
470
+ artifactPreview(before, path),
471
+ artifactPreview(after, path),
472
+ ]);
473
+ sendJson(request, response, 200, compareArtifactPreviews(path, beforePreview, afterPreview));
474
+ };
475
+
476
+ const handleComparisonRoute = async (context: RouteContext): Promise<boolean> => {
477
+ if (context.url.pathname === "/api/compare/runs") {
478
+ await handleRunComparison(context);
479
+ return true;
480
+ }
481
+ if (context.url.pathname === "/api/compare/suites") {
482
+ await handleSuiteComparison(context);
483
+ return true;
484
+ }
485
+ if (context.url.pathname === "/api/compare/artifacts") {
486
+ await handleArtifactComparison(context);
487
+ return true;
488
+ }
489
+ return false;
490
+ };
491
+
492
+ const handleArtifactRoute = async ({
493
+ request,
494
+ response,
495
+ url,
496
+ runsRoot,
497
+ }: RouteContext): Promise<boolean> => {
498
+ if (url.pathname !== "/api/artifact") return false;
499
+ const runId = url.searchParams.get("run");
500
+ const path = url.searchParams.get("path");
501
+ if (!runId || !path) {
502
+ sendJson(request, response, 400, { error: "a run id and artifact path are required" });
503
+ return true;
504
+ }
505
+ const run = await findRunById(runsRoot, runId);
506
+ if (!run) {
507
+ sendJson(request, response, 404, { error: "run not found" });
508
+ return true;
509
+ }
510
+ const preview = await artifactPreview(run, path);
511
+ sendJson(request, response, preview.exists ? 200 : 404, preview);
512
+ return true;
513
+ };
514
+
515
+ const experimentDetail = async (
516
+ runsRoot: string,
517
+ experiment: NonNullable<Awaited<ReturnType<typeof findExperimentById>>>,
518
+ ): Promise<Record<string, unknown>> => {
519
+ const [candidateResult, decisionResult] = await Promise.allSettled([
520
+ loadExperimentCandidates(experiment.experimentDir),
521
+ loadVerifiedExperimentDecisionSnapshot(experiment.experimentDir),
522
+ ]);
523
+ const integrityErrors = [candidateResult, decisionResult].flatMap((result) =>
524
+ result.status === "rejected"
525
+ ? [result.reason instanceof Error ? result.reason.message : String(result.reason)]
526
+ : [],
527
+ );
528
+ return {
529
+ experiment: {
530
+ experimentId: experiment.experimentId,
531
+ campaignId: experiment.campaignId,
532
+ mode: experiment.mode,
533
+ evaluationKind: experiment.evaluationKind,
534
+ status: experiment.status,
535
+ startedAt: experiment.startedAt,
536
+ endedAt: experiment.endedAt,
537
+ activeTimeMs: experiment.activeTimeMs,
538
+ championCandidateId: experiment.championCandidateId ?? null,
539
+ errorMessage: experiment.errorMessage ?? null,
540
+ selection: experiment.selection ?? null,
541
+ budgetUsage: experiment.budgetUsage,
542
+ evaluations: experiment.evaluations.map((evaluation) => ({
543
+ candidateId: evaluation.candidateId,
544
+ profileId: evaluation.profileId,
545
+ split: evaluation.split,
546
+ repetitions: evaluation.repetitions,
547
+ passRate: evaluation.passRate,
548
+ casePassRates: evaluation.casePassRates,
549
+ metrics: evaluation.metrics,
550
+ metricDefinitions: evaluation.metricDefinitions ?? {},
551
+ costUsd: evaluation.costUsd ?? null,
552
+ failedCases: evaluation.failedCases,
553
+ })),
554
+ },
555
+ candidates:
556
+ candidateResult.status === "fulfilled"
557
+ ? candidateResult.value.map((candidate) => ({
558
+ id: candidate.id,
559
+ parentIds: candidate.parentIds,
560
+ createdAt: candidate.createdAt,
561
+ proposer: candidate.proposer,
562
+ sourceBytes: candidate.sourceBytes,
563
+ rationale: candidate.rationale ?? null,
564
+ components: Object.keys(candidate.components).sort(),
565
+ }))
566
+ : [],
567
+ decision:
568
+ decisionResult.status === "fulfilled"
569
+ ? {
570
+ ...decisionResult.value.report,
571
+ experimentDir: displayPath(runsRoot, experiment.experimentDir),
572
+ }
573
+ : null,
574
+ integrity: {
575
+ verified: integrityErrors.length === 0,
576
+ errors: integrityErrors,
577
+ },
578
+ };
579
+ };
580
+
581
+ const handleDetailRoute = async ({
582
+ request,
583
+ response,
584
+ url,
585
+ runsRoot,
586
+ }: RouteContext): Promise<boolean> => {
587
+ if (url.pathname.startsWith("/api/suites/")) {
588
+ const suite = await findSuiteById(runsRoot, url.pathname.slice("/api/suites/".length));
589
+ if (!suite) sendJson(request, response, 404, { error: "suite not found" });
590
+ else
591
+ sendJson(request, response, 200, {
592
+ suite: {
593
+ ...suite,
594
+ suiteDir: displayPath(runsRoot, suite.suiteDir),
595
+ },
596
+ summary: await loadSuiteSummary(suite.suiteDir),
597
+ });
598
+ return true;
599
+ }
600
+ if (url.pathname.startsWith("/api/experiments/")) {
601
+ const experiment = await findExperimentById(
602
+ runsRoot,
603
+ url.pathname.slice("/api/experiments/".length),
604
+ );
605
+ if (!experiment) sendJson(request, response, 404, { error: "experiment not found" });
606
+ else sendJson(request, response, 200, await experimentDetail(runsRoot, experiment));
607
+ return true;
608
+ }
609
+ if (url.pathname.startsWith("/api/runs/")) {
610
+ const run = await findRunById(runsRoot, url.pathname.slice("/api/runs/".length));
611
+ if (!run) sendJson(request, response, 404, { error: "run not found" });
612
+ else sendJson(request, response, 200, await publicRunDetail(runsRoot, run.runDir));
613
+ return true;
614
+ }
615
+ return false;
616
+ };
617
+
618
+ const requestHandler =
619
+ (
620
+ assets: PreviewAssets,
621
+ runsRoot: string,
622
+ options: Pick<StartPreviewServerOptions, "initialPath" | "initialKind" | "caseFilter">,
623
+ ) =>
624
+ async (request: IncomingMessage, response: ServerResponse): Promise<void> => {
625
+ if (!isLoopbackHost(request.headers.host)) {
626
+ sendJson(request, response, 421, { error: "untrusted host" });
627
+ return;
628
+ }
629
+ if (request.method !== "GET" && request.method !== "HEAD") {
630
+ sendJson(request, response, 405, { error: "method not allowed" });
631
+ return;
632
+ }
633
+ const url = new URL(request.url ?? "/", `http://${LOOPBACK_HOST}`);
634
+ if (url.pathname === "/" || url.pathname === "/index.html") {
635
+ send(request, response, 200, "text/html; charset=utf-8", assets.html);
636
+ return;
637
+ }
638
+ const asset = assets.files.get(url.pathname);
639
+ if (asset) {
640
+ send(request, response, 200, asset.contentType, asset.body, "no-cache");
641
+ return;
642
+ }
643
+ if (url.pathname === "/api/snapshot") {
644
+ sendJson(request, response, 200, await buildSnapshot(runsRoot, options));
645
+ return;
646
+ }
647
+ const context = { request, response, url, runsRoot };
648
+ if (await handleComparisonRoute(context)) return;
649
+ if (await handleArtifactRoute(context)) return;
650
+ if (await handleDetailRoute(context)) return;
651
+ if (url.pathname === "/health") {
652
+ sendJson(request, response, 200, { ok: true });
653
+ return;
654
+ }
655
+ sendJson(request, response, 404, { error: "not found" });
656
+ };
657
+
658
+ const loadAssets = async (): Promise<PreviewAssets> => {
659
+ const htmlPath = resolve(ASSET_ROOT, "index.html");
660
+ const definitions = [
661
+ ["/assets/app.css", "app.css", "text/css; charset=utf-8"],
662
+ ["/assets/app.js", "app.js", "text/javascript; charset=utf-8"],
663
+ ["/assets/artifact-view.js", "artifact-view.js", "text/javascript; charset=utf-8"],
664
+ ["/assets/charts.js", "charts.js", "text/javascript; charset=utf-8"],
665
+ ["/assets/detail.js", "detail.js", "text/javascript; charset=utf-8"],
666
+ ["/assets/format.js", "format.js", "text/javascript; charset=utf-8"],
667
+ ] as const;
668
+ const paths = [htmlPath, ...definitions.map((definition) => resolve(ASSET_ROOT, definition[1]))];
669
+ for (const path of paths) {
670
+ if (!existsSync(path)) throw new Error(`Lux preview asset is missing: ${basename(path)}`);
671
+ }
672
+ const [html, ...bodies] = await Promise.all(paths.map((path) => readFile(path)));
673
+ if (!html) throw new Error("Lux preview index is empty");
674
+ const files = new Map(
675
+ definitions.map(([url, , contentType], index) => {
676
+ const body = bodies[index];
677
+ if (!body) throw new Error(`Lux preview asset is empty: ${url}`);
678
+ return [url, { body, contentType }];
679
+ }),
680
+ );
681
+ return { html, files };
682
+ };
683
+
684
+ const listen = (server: Server, port: number): Promise<void> =>
685
+ new Promise((resolveListen, reject) => {
686
+ const onError = (error: Error): void => {
687
+ server.off("listening", onListening);
688
+ reject(error);
689
+ };
690
+ const onListening = (): void => {
691
+ server.off("error", onError);
692
+ resolveListen();
693
+ };
694
+ server.once("error", onError);
695
+ server.once("listening", onListening);
696
+ server.listen(port, LOOPBACK_HOST);
697
+ });
698
+
699
+ export const startPreviewServer = async (
700
+ options: StartPreviewServerOptions,
701
+ ): Promise<PreviewServer> => {
702
+ const runsRoot = resolve(options.runsRoot);
703
+ const assets = await loadAssets();
704
+ const server = createServer((request, response) => {
705
+ void requestHandler(
706
+ assets,
707
+ runsRoot,
708
+ options,
709
+ )(request, response).catch((error: unknown) => {
710
+ const message = error instanceof Error ? error.message : String(error);
711
+ if (!response.headersSent) sendJson(request, response, 500, { error: message });
712
+ else response.destroy();
713
+ });
714
+ });
715
+ await listen(server, options.port ?? 0);
716
+ const address = server.address();
717
+ if (!address || typeof address === "string") {
718
+ server.close();
719
+ throw new Error("Lux preview server did not receive a TCP address");
720
+ }
721
+ let closePromise: Promise<void> | null = null;
722
+ const close = (): Promise<void> => {
723
+ closePromise ??= new Promise((resolveClose, reject) => {
724
+ server.close((error) => {
725
+ options.signal?.removeEventListener("abort", onAbort);
726
+ if (error) reject(error);
727
+ else resolveClose();
728
+ });
729
+ });
730
+ return closePromise;
731
+ };
732
+ const onAbort = (): void => {
733
+ void close();
734
+ };
735
+ if (options.signal?.aborted) onAbort();
736
+ else options.signal?.addEventListener("abort", onAbort, { once: true });
737
+ const closed = new Promise<void>((resolveClosed) => {
738
+ server.once("close", resolveClosed);
739
+ });
740
+ return {
741
+ url: `http://${LOOPBACK_HOST}:${address.port}`,
742
+ closed,
743
+ close,
744
+ };
745
+ };
746
+
747
+ export const openPreviewUrl = async (url: string): Promise<void> => {
748
+ let launcher = { command: "xdg-open", args: [url] };
749
+ if (process.platform === "darwin") {
750
+ launcher = { command: "open", args: [url] };
751
+ } else if (process.platform === "win32") {
752
+ launcher = { command: "cmd.exe", args: ["/d", "/s", "/c", "start", "", url] };
753
+ }
754
+ const result = await exec(launcher.command, launcher.args, {
755
+ timeoutMs: 5_000,
756
+ maxBuffer: 2_000,
757
+ });
758
+ if (result.exitCode !== 0 || result.timedOut) {
759
+ throw new Error(
760
+ result.stderr.trim() || `browser launcher exited ${result.exitCode ?? "early"}`,
761
+ );
762
+ }
763
+ };