@prismatic-io/lux 0.0.1 → 0.0.2-preview.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (256) hide show
  1. package/lib/answerers/persona/index.d.ts +2 -1
  2. package/lib/answerers/persona/index.d.ts.map +1 -1
  3. package/lib/answerers/persona/index.js +19 -5
  4. package/lib/answerers/persona/index.js.map +1 -1
  5. package/lib/answerers/terminal/index.js +1 -1
  6. package/lib/answerers/terminal/index.js.map +1 -1
  7. package/lib/assertions/authoring.d.ts +14 -0
  8. package/lib/assertions/authoring.d.ts.map +1 -0
  9. package/lib/assertions/authoring.js +3 -0
  10. package/lib/assertions/authoring.js.map +1 -0
  11. package/lib/assertions/core/helpers.d.ts +2 -7
  12. package/lib/assertions/core/helpers.d.ts.map +1 -1
  13. package/lib/assertions/core/helpers.js +2 -6
  14. package/lib/assertions/core/helpers.js.map +1 -1
  15. package/lib/assertions/core/index.d.ts.map +1 -1
  16. package/lib/assertions/core/index.js +2 -1
  17. package/lib/assertions/core/index.js.map +1 -1
  18. package/lib/assertions/core/resource-checks.d.ts +19 -0
  19. package/lib/assertions/core/resource-checks.d.ts.map +1 -1
  20. package/lib/assertions/core/resource-checks.js +32 -0
  21. package/lib/assertions/core/resource-checks.js.map +1 -1
  22. package/lib/authoring.d.ts +1 -0
  23. package/lib/authoring.d.ts.map +1 -1
  24. package/lib/authoring.js +1 -0
  25. package/lib/authoring.js.map +1 -1
  26. package/lib/cli/program.d.ts.map +1 -1
  27. package/lib/cli/program.js +12 -67
  28. package/lib/cli/program.js.map +1 -1
  29. package/lib/cli/render/reporter.d.ts.map +1 -1
  30. package/lib/cli/render/reporter.js +16 -3
  31. package/lib/cli/render/reporter.js.map +1 -1
  32. package/lib/cli/run-options.d.ts +1 -0
  33. package/lib/cli/run-options.d.ts.map +1 -1
  34. package/lib/cli/run-options.js.map +1 -1
  35. package/lib/cli/view.d.ts +13 -0
  36. package/lib/cli/view.d.ts.map +1 -0
  37. package/lib/cli/view.js +126 -0
  38. package/lib/cli/view.js.map +1 -0
  39. package/lib/core/action-events.d.ts +29 -0
  40. package/lib/core/action-events.d.ts.map +1 -0
  41. package/lib/core/action-events.js +51 -0
  42. package/lib/core/action-events.js.map +1 -0
  43. package/lib/core/agent-activity.d.ts +49 -0
  44. package/lib/core/agent-activity.d.ts.map +1 -0
  45. package/lib/core/agent-activity.js +206 -0
  46. package/lib/core/agent-activity.js.map +1 -0
  47. package/lib/core/agent-events.d.ts +26 -0
  48. package/lib/core/agent-events.d.ts.map +1 -0
  49. package/lib/core/agent-events.js +13 -0
  50. package/lib/core/agent-events.js.map +1 -0
  51. package/lib/core/artifact-evidence.d.ts +8 -0
  52. package/lib/core/artifact-evidence.d.ts.map +1 -1
  53. package/lib/core/artifact-evidence.js +36 -4
  54. package/lib/core/artifact-evidence.js.map +1 -1
  55. package/lib/core/assertion.d.ts +19 -9
  56. package/lib/core/assertion.d.ts.map +1 -1
  57. package/lib/core/assertion.js +26 -0
  58. package/lib/core/assertion.js.map +1 -1
  59. package/lib/core/case-authoring-state.d.ts +12 -0
  60. package/lib/core/case-authoring-state.d.ts.map +1 -0
  61. package/lib/core/case-authoring-state.js +43 -0
  62. package/lib/core/case-authoring-state.js.map +1 -0
  63. package/lib/core/case.d.ts +29 -1
  64. package/lib/core/case.d.ts.map +1 -1
  65. package/lib/core/case.js +14 -6
  66. package/lib/core/case.js.map +1 -1
  67. package/lib/core/driver-metrics.d.ts +98 -0
  68. package/lib/core/driver-metrics.d.ts.map +1 -0
  69. package/lib/core/driver-metrics.js +36 -0
  70. package/lib/core/driver-metrics.js.map +1 -0
  71. package/lib/core/driver.d.ts +3 -0
  72. package/lib/core/driver.d.ts.map +1 -1
  73. package/lib/core/driver.js.map +1 -1
  74. package/lib/core/events.d.ts +36 -0
  75. package/lib/core/events.d.ts.map +1 -1
  76. package/lib/core/events.js +12 -2
  77. package/lib/core/events.js.map +1 -1
  78. package/lib/core/experiment.d.ts +19 -0
  79. package/lib/core/experiment.d.ts.map +1 -1
  80. package/lib/core/experiment.js +2 -0
  81. package/lib/core/experiment.js.map +1 -1
  82. package/lib/core/index.d.ts +4 -0
  83. package/lib/core/index.d.ts.map +1 -1
  84. package/lib/core/index.js +4 -0
  85. package/lib/core/index.js.map +1 -1
  86. package/lib/core/lifecycle-fixtures.d.ts +3 -1
  87. package/lib/core/lifecycle-fixtures.d.ts.map +1 -1
  88. package/lib/core/lifecycle-fixtures.js +4 -2
  89. package/lib/core/lifecycle-fixtures.js.map +1 -1
  90. package/lib/core/lifecycle-hooks.d.ts +9 -0
  91. package/lib/core/lifecycle-hooks.d.ts.map +1 -1
  92. package/lib/core/lifecycle-hooks.js +8 -2
  93. package/lib/core/lifecycle-hooks.js.map +1 -1
  94. package/lib/core/run.d.ts +150 -0
  95. package/lib/core/run.d.ts.map +1 -1
  96. package/lib/core/run.js +9 -0
  97. package/lib/core/run.js.map +1 -1
  98. package/lib/core/wire.d.ts +12 -0
  99. package/lib/core/wire.d.ts.map +1 -1
  100. package/lib/core/wire.js +5 -1
  101. package/lib/core/wire.js.map +1 -1
  102. package/lib/drivers/claude-code/parse-events.d.ts.map +1 -1
  103. package/lib/drivers/claude-code/parse-events.js +73 -8
  104. package/lib/drivers/claude-code/parse-events.js.map +1 -1
  105. package/lib/drivers/codex/app-events.d.ts +9 -2
  106. package/lib/drivers/codex/app-events.d.ts.map +1 -1
  107. package/lib/drivers/codex/app-events.js +93 -12
  108. package/lib/drivers/codex/app-events.js.map +1 -1
  109. package/lib/drivers/codex/index.d.ts.map +1 -1
  110. package/lib/drivers/codex/index.js +10 -14
  111. package/lib/drivers/codex/index.js.map +1 -1
  112. package/lib/drivers/subprocess/index.d.ts.map +1 -1
  113. package/lib/drivers/subprocess/index.js +6 -0
  114. package/lib/drivers/subprocess/index.js.map +1 -1
  115. package/lib/index.d.ts +10 -6
  116. package/lib/index.d.ts.map +1 -1
  117. package/lib/index.js +10 -6
  118. package/lib/index.js.map +1 -1
  119. package/lib/orchestrator/annotation-store.d.ts.map +1 -1
  120. package/lib/orchestrator/annotation-store.js +6 -2
  121. package/lib/orchestrator/annotation-store.js.map +1 -1
  122. package/lib/orchestrator/case-extensions.d.ts +6 -0
  123. package/lib/orchestrator/case-extensions.d.ts.map +1 -0
  124. package/lib/orchestrator/case-extensions.js +28 -0
  125. package/lib/orchestrator/case-extensions.js.map +1 -0
  126. package/lib/orchestrator/compare.d.ts +13 -1
  127. package/lib/orchestrator/compare.d.ts.map +1 -1
  128. package/lib/orchestrator/compare.js +12 -0
  129. package/lib/orchestrator/compare.js.map +1 -1
  130. package/lib/orchestrator/config.d.ts.map +1 -1
  131. package/lib/orchestrator/config.js +0 -2
  132. package/lib/orchestrator/config.js.map +1 -1
  133. package/lib/orchestrator/doctor.d.ts.map +1 -1
  134. package/lib/orchestrator/doctor.js +5 -3
  135. package/lib/orchestrator/doctor.js.map +1 -1
  136. package/lib/orchestrator/experiment-contracts.d.ts +19 -0
  137. package/lib/orchestrator/experiment-contracts.d.ts.map +1 -1
  138. package/lib/orchestrator/experiment-evaluation.d.ts.map +1 -1
  139. package/lib/orchestrator/experiment-evaluation.js +17 -0
  140. package/lib/orchestrator/experiment-evaluation.js.map +1 -1
  141. package/lib/orchestrator/experiment-report.d.ts +19 -0
  142. package/lib/orchestrator/experiment-report.d.ts.map +1 -1
  143. package/lib/orchestrator/experiment-runs.d.ts +38 -0
  144. package/lib/orchestrator/experiment-runs.d.ts.map +1 -1
  145. package/lib/orchestrator/grader-experiment.d.ts +12 -0
  146. package/lib/orchestrator/grader-experiment.d.ts.map +1 -1
  147. package/lib/orchestrator/grading.d.ts +2 -1
  148. package/lib/orchestrator/grading.d.ts.map +1 -1
  149. package/lib/orchestrator/grading.js +16 -4
  150. package/lib/orchestrator/grading.js.map +1 -1
  151. package/lib/orchestrator/index.d.ts +1 -1
  152. package/lib/orchestrator/index.d.ts.map +1 -1
  153. package/lib/orchestrator/index.js +1 -1
  154. package/lib/orchestrator/index.js.map +1 -1
  155. package/lib/orchestrator/list-runs.d.ts +7 -1
  156. package/lib/orchestrator/list-runs.d.ts.map +1 -1
  157. package/lib/orchestrator/list-runs.js +50 -6
  158. package/lib/orchestrator/list-runs.js.map +1 -1
  159. package/lib/orchestrator/loader.d.ts.map +1 -1
  160. package/lib/orchestrator/loader.js +2 -0
  161. package/lib/orchestrator/loader.js.map +1 -1
  162. package/lib/orchestrator/orchestrator.d.ts +3 -0
  163. package/lib/orchestrator/orchestrator.d.ts.map +1 -1
  164. package/lib/orchestrator/orchestrator.js +42 -14
  165. package/lib/orchestrator/orchestrator.js.map +1 -1
  166. package/lib/orchestrator/owned-lock.d.ts.map +1 -1
  167. package/lib/orchestrator/owned-lock.js +52 -38
  168. package/lib/orchestrator/owned-lock.js.map +1 -1
  169. package/lib/orchestrator/regrade.d.ts +6 -0
  170. package/lib/orchestrator/regrade.d.ts.map +1 -1
  171. package/lib/orchestrator/regrade.js +8 -4
  172. package/lib/orchestrator/regrade.js.map +1 -1
  173. package/lib/orchestrator/run-execution.d.ts +2 -1
  174. package/lib/orchestrator/run-execution.d.ts.map +1 -1
  175. package/lib/orchestrator/run-execution.js +20 -5
  176. package/lib/orchestrator/run-execution.js.map +1 -1
  177. package/lib/orchestrator/suite-compare.d.ts +34 -0
  178. package/lib/orchestrator/suite-compare.d.ts.map +1 -1
  179. package/lib/orchestrator/suite-summary.d.ts +68 -0
  180. package/lib/orchestrator/suite-summary.d.ts.map +1 -1
  181. package/lib/orchestrator/suite-summary.js +45 -7
  182. package/lib/orchestrator/suite-summary.js.map +1 -1
  183. package/lib/previewer/artifacts.d.ts +25 -0
  184. package/lib/previewer/artifacts.d.ts.map +1 -0
  185. package/lib/previewer/artifacts.js +180 -0
  186. package/lib/previewer/artifacts.js.map +1 -0
  187. package/lib/previewer/server.d.ts +105 -0
  188. package/lib/previewer/server.d.ts.map +1 -0
  189. package/lib/previewer/server.js +542 -0
  190. package/lib/previewer/server.js.map +1 -0
  191. package/package.json +2 -2
  192. package/skills/lux/SKILL.md +7 -3
  193. package/skills/lux/references/cli.md +25 -2
  194. package/skills/lux/references/eval-authoring.md +158 -4
  195. package/skills/lux-answerer/SKILL.md +1 -1
  196. package/src/answerers/persona/index.ts +20 -4
  197. package/src/answerers/terminal/index.ts +1 -1
  198. package/src/assertions/authoring.ts +81 -0
  199. package/src/assertions/core/helpers.ts +3 -6
  200. package/src/assertions/core/index.ts +2 -0
  201. package/src/assertions/core/resource-checks.ts +35 -0
  202. package/src/authoring.ts +1 -0
  203. package/src/cli/program.ts +14 -93
  204. package/src/cli/render/reporter.ts +18 -3
  205. package/src/cli/run-options.ts +1 -0
  206. package/src/cli/view.ts +168 -0
  207. package/src/core/action-events.ts +59 -0
  208. package/src/core/agent-activity.ts +236 -0
  209. package/src/core/agent-events.ts +17 -0
  210. package/src/core/artifact-evidence.ts +47 -4
  211. package/src/core/assertion.ts +41 -22
  212. package/src/core/case-authoring-state.ts +65 -0
  213. package/src/core/case.ts +57 -6
  214. package/src/core/driver-metrics.ts +51 -0
  215. package/src/core/driver.ts +3 -0
  216. package/src/core/events.ts +14 -2
  217. package/src/core/experiment.ts +2 -0
  218. package/src/core/index.ts +4 -0
  219. package/src/core/lifecycle-fixtures.ts +7 -2
  220. package/src/core/lifecycle-hooks.ts +16 -2
  221. package/src/core/run.ts +9 -0
  222. package/src/core/wire.ts +5 -1
  223. package/src/drivers/claude-code/parse-events.ts +98 -12
  224. package/src/drivers/codex/app-events.ts +116 -16
  225. package/src/drivers/codex/index.ts +16 -14
  226. package/src/drivers/subprocess/index.ts +6 -0
  227. package/src/index.ts +50 -6
  228. package/src/orchestrator/annotation-store.ts +5 -1
  229. package/src/orchestrator/case-extensions.ts +44 -0
  230. package/src/orchestrator/compare.ts +18 -0
  231. package/src/orchestrator/config.ts +0 -2
  232. package/src/orchestrator/doctor.ts +5 -3
  233. package/src/orchestrator/experiment-evaluation.ts +23 -0
  234. package/src/orchestrator/grading.ts +21 -2
  235. package/src/orchestrator/index.ts +1 -0
  236. package/src/orchestrator/list-runs.ts +66 -5
  237. package/src/orchestrator/loader.ts +2 -0
  238. package/src/orchestrator/orchestrator.ts +86 -40
  239. package/src/orchestrator/owned-lock.ts +54 -34
  240. package/src/orchestrator/regrade.ts +8 -7
  241. package/src/orchestrator/run-execution.ts +24 -3
  242. package/src/orchestrator/suite-summary.ts +101 -8
  243. package/src/previewer/artifacts.ts +221 -0
  244. package/src/previewer/server.ts +765 -0
  245. package/viewer/app.css +2075 -0
  246. package/viewer/app.js +606 -0
  247. package/viewer/artifact-view.js +48 -0
  248. package/viewer/charts.js +86 -0
  249. package/viewer/detail.js +1214 -0
  250. package/viewer/format.js +129 -0
  251. package/viewer/index.html +151 -0
  252. package/lib/answerers/scripted/index.d.ts +0 -8
  253. package/lib/answerers/scripted/index.d.ts.map +0 -1
  254. package/lib/answerers/scripted/index.js +0 -163
  255. package/lib/answerers/scripted/index.js.map +0 -1
  256. package/src/answerers/scripted/index.ts +0 -186
@@ -5,6 +5,7 @@ import { glob } from "tinyglobby";
5
5
  import { z } from "zod";
6
6
  import { RubricConfigSchema } from "../assertions/rubric/index.js";
7
7
  import type { EvalCase, Registry } from "../core/index.js";
8
+ import { registryForEvalCase } from "./case-extensions.js";
8
9
  import { buildRegistryFromConfig, loadLuxConfig, type ResolvedLuxConfig } from "./config.js";
9
10
  import { type DiscoveredCase, type DiscoveryResult, discoverCases } from "./discover.js";
10
11
  import { cliIdentity, ExperimentCampaignPlanSchema, planExperimentCampaign } from "./experiment.js";
@@ -559,12 +560,13 @@ const inspectCase = async (
559
560
  ): Promise<CliTarget[]> => {
560
561
  try {
561
562
  const authored = await loadEvalCase(discovered.path);
563
+ const caseRegistry = registryForEvalCase(registry, authored);
562
564
  const drivers = caseDriverReferences(authored.driver, config);
563
565
  const answerer = authored.answerer ?? config.defaultAnswerer;
564
- validateCaseDrivers(discovered.id, drivers, registry, recorder);
565
- validateCaseAnswerer(discovered.id, answerer, registry, recorder);
566
+ validateCaseDrivers(discovered.id, drivers, caseRegistry, recorder);
567
+ validateCaseAnswerer(discovered.id, answerer, caseRegistry, recorder);
566
568
  return [
567
- ...(await validateCaseAssertions(discovered.id, authored, config, registry, recorder)),
569
+ ...(await validateCaseAssertions(discovered.id, authored, config, caseRegistry, recorder)),
568
570
  ...executionTargetsForCase(discovered.id, drivers, answerer),
569
571
  ];
570
572
  } catch (error) {
@@ -101,6 +101,23 @@ const evaluationMetrics = (
101
101
  ),
102
102
  });
103
103
 
104
+ const evaluationMetricDefinitions = (summary: SuiteSummary) =>
105
+ Object.fromEntries(
106
+ Object.entries(summary.metrics).flatMap(([name, metric]) => {
107
+ if (!metric.unit && !metric.label && !metric.better) return [];
108
+ return [
109
+ [
110
+ name,
111
+ {
112
+ unit: metric.unit ?? "number",
113
+ better: metric.better ?? "neutral",
114
+ ...(metric.label ? { label: metric.label } : {}),
115
+ },
116
+ ],
117
+ ];
118
+ }),
119
+ );
120
+
104
121
  const summaryUsage = (summary: SuiteSummary) =>
105
122
  summary.samples.reduce(
106
123
  (usage, sample) =>
@@ -277,6 +294,7 @@ const agentEvaluationFromSuite = (input: {
277
294
  summary.cases.map((entry) => [entry.canonicalCaseId, entry.passRate]),
278
295
  ),
279
296
  metrics: evaluationMetrics(summary, candidate),
297
+ metricDefinitions: evaluationMetricDefinitions(summary),
280
298
  samples: evaluationSamples(summary),
281
299
  metricCalls: summary.samples.length,
282
300
  judgeCalls: results.reduce(
@@ -352,6 +370,10 @@ const aggregateAgentUnits = (
352
370
  .map(([, values]) => average(values)),
353
371
  ]),
354
372
  );
373
+ const metricDefinitions = Object.assign(
374
+ {},
375
+ ...units.map(({ evaluation }) => evaluation.metricDefinitions ?? {}),
376
+ );
355
377
  return {
356
378
  candidateId: candidate.id,
357
379
  profileId,
@@ -367,6 +389,7 @@ const aggregateAgentUnits = (
367
389
  name === "subject.bytes" ? candidate.sourceBytes : average(values),
368
390
  ]),
369
391
  ),
392
+ metricDefinitions,
370
393
  samples,
371
394
  metricCalls: units.reduce((total, unit) => total + (unit.evaluation.metricCalls ?? 0), 0),
372
395
  judgeCalls: units.reduce((total, unit) => total + (unit.evaluation.judgeCalls ?? 0), 0),
@@ -76,6 +76,7 @@ export const graderConfigHash = (input: {
76
76
  assertions: readonly AssertionEvaluatorIdentity[];
77
77
  validatedAssertionsHash: string;
78
78
  graderContext?: unknown;
79
+ skipJudge?: boolean;
79
80
  }): string =>
80
81
  valueHash({
81
82
  luxVersion: input.luxVersion,
@@ -83,6 +84,7 @@ export const graderConfigHash = (input: {
83
84
  assertions: input.assertions,
84
85
  validatedAssertionsHash: input.validatedAssertionsHash,
85
86
  graderContext: input.graderContext ?? null,
87
+ skipJudge: input.skipJudge ?? false,
86
88
  });
87
89
 
88
90
  export const validateAssertionPlan = async (
@@ -133,6 +135,7 @@ export const gradeRun = async (
133
135
  complete: (attemptIds: readonly string[]) => void | Promise<void>;
134
136
  },
135
137
  validatedConfigs?: readonly unknown[],
138
+ skipJudge = false,
136
139
  ): Promise<GradingReport> => {
137
140
  const guardedObserver = guardRunObserver(observer);
138
141
  const assertions = run.case.assertions ?? [];
@@ -141,12 +144,27 @@ export const gradeRun = async (
141
144
  }
142
145
 
143
146
  const entries: GradingAssertionEntry[] = [];
147
+ const skippedAssertions: Array<{
148
+ id: string;
149
+ type: string;
150
+ name: string;
151
+ reason: "judge-disabled";
152
+ }> = [];
144
153
  let weightedScoreSum = 0;
145
154
  let weightSum = 0;
146
155
  let allPassed = true;
147
156
  let gradingIncomplete = false;
148
157
 
149
158
  for (const [index, a] of assertions.entries()) {
159
+ if (skipJudge && a.type === "rubric") {
160
+ skippedAssertions.push({
161
+ id: a.id ?? `${a.type}:${index + 1}`,
162
+ type: a.type,
163
+ name: describeAssertion(a),
164
+ reason: "judge-disabled",
165
+ });
166
+ continue;
167
+ }
150
168
  // Grading is the last cancellation point — an LLM judge can run for
151
169
  // minutes per assertion. An aborted grade can't claim everything passed.
152
170
  if (abortSignal?.aborted) {
@@ -247,12 +265,13 @@ export const gradeRun = async (
247
265
  }
248
266
 
249
267
  let caseScore: number | null = null;
250
- if (!gradingIncomplete) caseScore = weightSum > 0 ? weightedScoreSum / weightSum : 0;
268
+ if (!gradingIncomplete && weightSum > 0) caseScore = weightedScoreSum / weightSum;
251
269
  return {
252
270
  schemaVersion: 1,
253
- casePassed: gradingIncomplete ? null : allPassed,
271
+ casePassed: gradingIncomplete || weightSum === 0 ? null : allPassed,
254
272
  caseScore,
255
273
  assertions: entries,
274
+ ...(skippedAssertions.length > 0 ? { skippedAssertions } : {}),
256
275
  ...(gradingIncomplete
257
276
  ? { incomplete: { exitReason: "error" as const, phase: "grading" as const } }
258
277
  : {}),
@@ -122,6 +122,7 @@ export {
122
122
  formatRunsTable,
123
123
  type ListedRun,
124
124
  listRuns,
125
+ loadRunArtifacts,
125
126
  loadRunDetail,
126
127
  type RunDetail,
127
128
  } from "./list-runs.js";
@@ -2,6 +2,11 @@ import { existsSync } from "node:fs";
2
2
  import { constants, lstat, open, readdir } from "node:fs/promises";
3
3
  import { basename, join, relative, sep } from "node:path";
4
4
  import {
5
+ ActionProgressPayloadSchema,
6
+ AgentCompletedPayloadSchema,
7
+ AgentSpawnPayloadSchema,
8
+ type Artifact,
9
+ ArtifactSchema,
5
10
  type GradingReport,
6
11
  GradingReportSchema,
7
12
  harnessUsage,
@@ -31,6 +36,11 @@ export type ListedRun = {
31
36
  startedAt: string | null;
32
37
  endedAt: string | null;
33
38
  durationMs: number | null;
39
+ timeToFirstActionMs: number | null;
40
+ timeToFirstMutationMs: number | null;
41
+ timeToFirstDurableMutationMs: number | null;
42
+ driverMetrics: NonNullable<RunMetadata["driverMetrics"]>;
43
+ subagentCount: number;
34
44
  casePassed: boolean | null;
35
45
  caseScore: number | null;
36
46
  assertionPassed: number;
@@ -63,6 +73,12 @@ const summarizeRun = async (
63
73
  startedAt,
64
74
  endedAt,
65
75
  durationMs,
76
+ timeToFirstActionMs: meta?.actionTiming?.timeToFirstActionMs ?? null,
77
+ timeToFirstMutationMs: meta?.actionTiming?.timeToFirstMutationMs ?? null,
78
+ timeToFirstDurableMutationMs: meta?.actionTiming?.timeToFirstDurableMutationMs ?? null,
79
+ driverMetrics: meta?.driverMetrics ?? {},
80
+ subagentCount:
81
+ meta?.agentActivity?.filter((activity) => activity.agent.role === "subagent").length ?? 0,
66
82
  casePassed: grading?.casePassed ?? null,
67
83
  caseScore: grading?.caseScore ?? null,
68
84
  assertionPassed: passing,
@@ -127,7 +143,7 @@ export const listRuns = async (runsRoot: string): Promise<ListedRun[]> => {
127
143
  export const formatRunsTable = (runs: ListedRun[]): string => {
128
144
  if (runs.length === 0) return "(no runs found)";
129
145
  const rows: string[][] = [
130
- ["status", "case", "driver", "ts", "exit", "score", "duration"],
146
+ ["status", "case", "driver", "agents", "ts", "exit", "score", "duration"],
131
147
  ...runs.map((r) => {
132
148
  let status = "ungraded";
133
149
  if (r.casePassed === true) status = "PASS";
@@ -137,7 +153,8 @@ export const formatRunsTable = (runs: ListedRun[]): string => {
137
153
  ? `${(r.caseScore * 100).toFixed(0)}% (${r.assertionPassed}/${r.assertionTotal})`
138
154
  : "—";
139
155
  const duration = r.durationMs !== null ? `${(r.durationMs / 1000).toFixed(1)}s` : "—";
140
- return [status, r.caseSlug, r.driverName, r.ts, r.exitReason ?? "?", score, duration];
156
+ const agents = r.subagentCount > 0 ? `1+${r.subagentCount}` : "1";
157
+ return [status, r.caseSlug, r.driverName, agents, r.ts, r.exitReason ?? "?", score, duration];
141
158
  }),
142
159
  ];
143
160
  const header = rows[0] ?? [];
@@ -156,6 +173,9 @@ export type RunDetail = {
156
173
  artifacts: string[];
157
174
  };
158
175
 
176
+ export const loadRunArtifacts = async (runDir: string): Promise<Artifact[]> =>
177
+ (await safeReadJson(join(runDir, "artifacts.json"), ArtifactSchema.array())) ?? [];
178
+
159
179
  const readEventsJsonl = async (path: string): Promise<RunEvent[]> => {
160
180
  if (!existsSync(path)) return [];
161
181
  if ((await lstat(path)).isSymbolicLink()) {
@@ -291,6 +311,27 @@ const appendMetadataDetail = (lines: string[], detail: RunDetail): void => {
291
311
  `phases: agent=${secs(phases.agent)} collect=${secs(phases.collect)} grading=${secs(phases.grading)}`,
292
312
  );
293
313
  }
314
+ const actions = detail.metadata.actionTiming;
315
+ if (actions) {
316
+ const value = (ms: number | undefined): string => (ms === undefined ? "—" : `${ms}ms`);
317
+ lines.push(
318
+ `first action: any=${value(actions.timeToFirstActionMs)} mutation=${value(actions.timeToFirstMutationMs)} durable-mutation=${value(actions.timeToFirstDurableMutationMs)}`,
319
+ );
320
+ }
321
+ const activities = detail.metadata.agentActivity ?? [];
322
+ const subagents = activities.filter((activity) => activity.agent.role === "subagent");
323
+ if (subagents.length > 0) {
324
+ lines.push(`subagents: ${subagents.length}`);
325
+ for (const activity of subagents) {
326
+ const label = activity.agent.name
327
+ ? `${activity.agent.name} (${activity.agent.id})`
328
+ : activity.agent.id;
329
+ const agentActions = activity.actions;
330
+ lines.push(
331
+ ` ${label}: ${activity.status}, ${(activity.durationMs / 1000).toFixed(1)}s, events=${activity.eventCount}, tokens=${totalTokens(activity.usage)}, tools=${activity.toolCalls.total} (${activity.toolCalls.succeeded} ok/${activity.toolCalls.failed} failed/${activity.toolCalls.unresolved} unresolved), actions=${agentActions.total}, mutations=${agentActions.mutations}`,
332
+ );
333
+ }
334
+ }
294
335
  }
295
336
  };
296
337
 
@@ -361,17 +402,37 @@ const formatEventLine = (event: RunEvent): string => {
361
402
  }`;
362
403
  }
363
404
  return `responded(custom): ${JSON.stringify(event.payload).slice(0, 80)}`;
364
- case "progress":
405
+ case "progress": {
406
+ const agent = event.agent?.role === "subagent" ? ` [subagent ${event.agent.id}]` : "";
407
+ if (event.kind === "agent-spawn") {
408
+ const payload = AgentSpawnPayloadSchema.safeParse(event.payload).data;
409
+ return payload
410
+ ? `agent-spawn: ${payload.agent.name ?? payload.agent.id} (${payload.agent.id})`
411
+ : "agent-spawn: malformed";
412
+ }
413
+ if (event.kind === "agent-completed") {
414
+ const payload = AgentCompletedPayloadSchema.safeParse(event.payload).data;
415
+ return payload
416
+ ? `agent-completed: ${payload.agentId} (${payload.status})`
417
+ : "agent-completed: malformed";
418
+ }
419
+ if (event.kind === "action") {
420
+ const payload = ActionProgressPayloadSchema.safeParse(event.payload).data;
421
+ return payload
422
+ ? `action${agent}: ${payload.name} (${payload.category})`
423
+ : "action: malformed";
424
+ }
365
425
  if (event.kind === "tool-call") {
366
426
  const payload = ToolCallPayloadSchema.safeParse(event.payload).data;
367
- return payload ? `tool-call: ${payload.name}` : "tool-call: malformed";
427
+ return payload ? `tool-call${agent}: ${payload.name}` : "tool-call: malformed";
368
428
  }
369
429
  if (event.kind === "tool-result") {
370
430
  const payload = ToolResultPayloadSchema.safeParse(event.payload).data;
371
431
  if (!payload) return "tool-result: malformed";
372
- return `tool-result: ${payload.name} (${payload.isError ? "failed" : "succeeded"})`;
432
+ return `tool-result${agent}: ${payload.name} (${payload.isError ? "failed" : "succeeded"})`;
373
433
  }
374
434
  return `progress(${event.kind})`;
435
+ }
375
436
  case "lifecycle":
376
437
  return `lifecycle: ${event.stage}`;
377
438
  case "ready":
@@ -1,5 +1,6 @@
1
1
  import { resolve } from "node:path";
2
2
  import { pathToFileURL } from "node:url";
3
+ import { transferEvalCaseExtensions } from "../core/case-authoring-state.js";
3
4
  import { type EvalCase, EvalCaseSchema, formatZodIssues } from "../core/index.js";
4
5
 
5
6
  export const loadEvalCase = async (filePath: string): Promise<EvalCase> => {
@@ -15,5 +16,6 @@ export const loadEvalCase = async (filePath: string): Promise<EvalCase> => {
15
16
  if (!result.success) {
16
17
  throw new Error(`Invalid eval case at ${absolute}: ${formatZodIssues(result.error)}`);
17
18
  }
19
+ transferEvalCaseExtensions(mod.default as EvalCase, result.data);
18
20
  return result.data;
19
21
  };
@@ -18,6 +18,12 @@ import {
18
18
  type SuiteLifecycleHookContext,
19
19
  valueHash,
20
20
  } from "../core/index.js";
21
+ import {
22
+ fixturesForEvalCase,
23
+ mergeLifecycleFixtures,
24
+ registryForEvalCase,
25
+ valuesForFixtures,
26
+ } from "./case-extensions.js";
21
27
  import { evaluatorImplementationHash } from "./evaluator-identity.js";
22
28
  import {
23
29
  assertionEvaluatorIdentities,
@@ -122,6 +128,8 @@ export type RunRequest = {
122
128
  /** Root that relative `fixtures` dir paths resolve against. */
123
129
  fixturesRoot?: string | undefined;
124
130
  answererOverride?: string | undefined;
131
+ /** Skip built-in rubric judge assertions without changing subject execution. */
132
+ skipJudge?: boolean | undefined;
125
133
  startedAt?: Date | undefined;
126
134
  /** The `runSuite` invocation this case attempt belongs to. */
127
135
  suiteId?: string | undefined;
@@ -476,11 +484,6 @@ export class Orchestrator {
476
484
  }
477
485
 
478
486
  async run(request: RunRequest): Promise<RunResult> {
479
- if (this.lifecycleFixtures.length > 0 && !request.suiteId) {
480
- throw new Error(
481
- "Orchestrator.run() cannot execute configured lifecycle fixtures; use runSuite() for a single case",
482
- );
483
- }
484
487
  if (!request.case && !request.casePath) {
485
488
  throw new Error("RunRequest requires either 'case' or 'casePath'");
486
489
  }
@@ -489,11 +492,24 @@ export class Orchestrator {
489
492
  }
490
493
  const casePath = request.casePath ? resolve(request.casePath) : INLINE_CASE_PATH;
491
494
  const authoredCase = request.case ?? (await loadEvalCase(casePath));
495
+ const lifecycleFixtures = mergeLifecycleFixtures(
496
+ this.lifecycleFixtures,
497
+ fixturesForEvalCase(authoredCase),
498
+ );
499
+ if (
500
+ lifecycleFixtures.some(({ setupSuite, setupCase }) => setupSuite || setupCase) &&
501
+ !request.suiteId
502
+ ) {
503
+ throw new Error(
504
+ "Orchestrator.run() cannot execute lifecycle fixtures; use runSuite() for a single case",
505
+ );
506
+ }
507
+ const registry = registryForEvalCase(this.registry, authoredCase);
492
508
  const evalCase = bindSubjectRoot(authoredCase, request.subjectRoot);
493
509
  const caseId = resolveCaseId(casePath, evalCase, request.caseId);
494
510
  const validatedAssertionConfigs = await validateAssertionPlan(
495
511
  evalCase.assertions ?? [],
496
- this.registry,
512
+ registry,
497
513
  );
498
514
 
499
515
  const driverRef = resolveDriverReference(evalCase, this.defaultDriver);
@@ -506,11 +522,11 @@ export class Orchestrator {
506
522
  );
507
523
  const answererName = answererNameOf(answererRef);
508
524
 
509
- const driverPlugin = this.registry.getDriver(driverName);
525
+ const driverPlugin = registry.getDriver(driverName);
510
526
  if (!driverPlugin) {
511
527
  throw new Error(`no driver registered for '${driverName}'`);
512
528
  }
513
- const answererPlugin = this.registry.getAnswerer(answererName);
529
+ const answererPlugin = registry.getAnswerer(answererName);
514
530
  if (!answererPlugin) {
515
531
  throw new Error(`no answerer registered for '${answererName}'`);
516
532
  }
@@ -544,7 +560,7 @@ export class Orchestrator {
544
560
  try {
545
561
  const assertionImplementations = assertionEvaluatorIdentities(
546
562
  authoredCase.assertions ?? [],
547
- this.registry,
563
+ registry,
548
564
  );
549
565
  const validatedAssertionsHash = valueHash(validatedAssertionConfigs);
550
566
  const implementationHash = await this.evaluatorImplementationIdentity();
@@ -554,34 +570,40 @@ export class Orchestrator {
554
570
  assertions: assertionImplementations,
555
571
  validatedAssertionsHash,
556
572
  graderContext: this.harness,
573
+ skipJudge: request.skipJudge ?? false,
557
574
  });
558
- return await this.runWithPlugins({
559
- request,
560
- casePath,
561
- authoredCase,
562
- evalCase,
563
- validatedAssertionConfigs,
564
- caseId,
565
- driverName,
566
- answererName,
567
- answererConfig: answererCfg,
568
- driverConfigHash: valueHash(driverCfg),
569
- answererConfigHash: valueHash(answererCfg),
570
- validatedAssertionsHash,
571
- graderConfigHash: graderHash,
572
- evaluatorConfigHash: valueHash({
573
- luxVersion: LUX_VERSION,
574
- implementationHash,
575
- harness: this.harness,
576
- driver: { name: driverPlugin.name, create: driverPlugin.create },
577
- answerer: { name: answererPlugin.name, create: answererPlugin.create },
578
- assertions: assertionImplementations,
579
- lifecycleFixtures: this.lifecycleFixtures,
575
+ return await this.runWithPlugins(
576
+ {
577
+ request,
578
+ casePath,
579
+ authoredCase,
580
+ evalCase,
581
+ validatedAssertionConfigs,
582
+ caseId,
583
+ driverName,
584
+ answererName,
585
+ answererConfig: answererCfg,
586
+ driverConfigHash: valueHash(driverCfg),
587
+ answererConfigHash: valueHash(answererCfg),
580
588
  validatedAssertionsHash,
581
- }),
582
- driver,
583
- answerer,
584
- });
589
+ graderConfigHash: graderHash,
590
+ evaluatorConfigHash: valueHash({
591
+ luxVersion: LUX_VERSION,
592
+ implementationHash,
593
+ harness: this.harness,
594
+ driver: { name: driverPlugin.name, create: driverPlugin.create },
595
+ answerer: { name: answererPlugin.name, create: answererPlugin.create },
596
+ assertions: assertionImplementations,
597
+ lifecycleFixtures,
598
+ validatedAssertionsHash,
599
+ skipJudge: request.skipJudge ?? false,
600
+ }),
601
+ fixtureValues: valuesForFixtures(lifecycleFixtures),
602
+ driver,
603
+ answerer,
604
+ },
605
+ registry,
606
+ );
585
607
  } finally {
586
608
  await closeQuietly(() => driver.close(), `driver '${driverName}'`);
587
609
  await closeQuietly(() => answerer.close(), `answerer '${answererName}'`);
@@ -599,6 +621,7 @@ export class Orchestrator {
599
621
  runsRoot: string;
600
622
  fixturesRoot?: string | undefined;
601
623
  answererOverride?: string | undefined;
624
+ skipJudge?: boolean | undefined;
602
625
  observer?: SuiteObserver | undefined;
603
626
  /** Aborts the cases in flight and stops the pool pulling new ones. */
604
627
  abortSignal?: AbortSignal | undefined;
@@ -628,6 +651,20 @@ export class Orchestrator {
628
651
  const observer = opts.observer;
629
652
  const requestedConcurrency = validateSuiteConcurrency(opts.concurrency);
630
653
  const concurrency = Math.min(requestedConcurrency, Math.max(items.length, 1));
654
+ const authoredCases = await Promise.allSettled(items.map((item) => loadEvalCase(item.path)));
655
+ const localFixtures = authoredCases.map((result) =>
656
+ result.status === "fulfilled" ? fixturesForEvalCase(result.value) : [],
657
+ );
658
+ const lifecycleFixtures = mergeLifecycleFixtures(this.lifecycleFixtures, ...localFixtures);
659
+ const globalFixtureNames = new Set(this.lifecycleFixtures.map((fixture) => fixture.name));
660
+ const itemIndexesByFixture = new Map<string, number[]>();
661
+ for (const [index, fixtures] of localFixtures.entries()) {
662
+ for (const fixture of fixtures) {
663
+ const indexes = itemIndexesByFixture.get(fixture.name) ?? [];
664
+ indexes.push(index);
665
+ itemIndexesByFixture.set(fixture.name, indexes);
666
+ }
667
+ }
631
668
  const suite = await openSuiteRun({
632
669
  runsRoot: opts.runsRoot,
633
670
  concurrency,
@@ -639,7 +676,7 @@ export class Orchestrator {
639
676
  ...(opts.variantId ? { variantId: opts.variantId } : {}),
640
677
  ...(opts.candidateId ? { candidateId: opts.candidateId } : {}),
641
678
  ...(opts.subjectHash ? { subjectHash: opts.subjectHash } : {}),
642
- fixtures: this.lifecycleFixtures.map(({ name, pluginVersion }) => ({
679
+ fixtures: lifecycleFixtures.map(({ name, pluginVersion }) => ({
643
680
  name,
644
681
  pluginVersion,
645
682
  })),
@@ -677,9 +714,16 @@ export class Orchestrator {
677
714
  if (!opts.abortSignal?.aborted) {
678
715
  try {
679
716
  cleanupSuiteHooks = await setupLifecycleHooks(
680
- this.lifecycleFixtures,
717
+ lifecycleFixtures,
681
718
  "suite",
682
- (hook) => hook.setupSuite?.(hookContext),
719
+ (hook) => {
720
+ if (globalFixtureNames.has(hook.name)) return hook.setupSuite?.(hookContext);
721
+ const indexes = itemIndexesByFixture.get(hook.name) ?? [];
722
+ return hook.setupSuite?.({
723
+ ...hookContext,
724
+ items: indexes.flatMap((index) => hookContext.items[index] ?? []),
725
+ });
726
+ },
683
727
  opts.abortSignal,
684
728
  );
685
729
  } catch (error) {
@@ -718,7 +762,7 @@ export class Orchestrator {
718
762
  let cleanupCaseHooks: () => void | Promise<void>;
719
763
  try {
720
764
  cleanupCaseHooks = await setupLifecycleHooks(
721
- this.lifecycleFixtures,
765
+ mergeLifecycleFixtures(this.lifecycleFixtures, localFixtures[index] ?? []),
722
766
  "case",
723
767
  (hook) =>
724
768
  hook.setupCase?.({
@@ -749,6 +793,7 @@ export class Orchestrator {
749
793
  runsRoot: opts.runsRoot,
750
794
  fixturesRoot: opts.fixturesRoot,
751
795
  answererOverride: opts.answererOverride,
796
+ skipJudge: opts.skipJudge,
752
797
  observer: handle ?? observer,
753
798
  abortSignal: opts.abortSignal,
754
799
  suiteId: suite.manifest.suiteId,
@@ -906,11 +951,12 @@ export class Orchestrator {
906
951
 
907
952
  private runWithPlugins(
908
953
  args: Omit<RunWithPluginsArgs, "luxVersion" | "registry">,
954
+ registry: Registry = this.registry,
909
955
  ): Promise<RunResult> {
910
956
  return executeRunWithPlugins({
911
957
  ...args,
912
958
  luxVersion: LUX_VERSION,
913
- registry: this.registry,
959
+ registry,
914
960
  });
915
961
  }
916
962
  }
@@ -1,8 +1,6 @@
1
- import { randomInt, randomUUID } from "node:crypto";
2
- import { link, lstat, open, readdir, readFile, rm, unlink } from "node:fs/promises";
1
+ import { randomUUID } from "node:crypto";
2
+ import { link, lstat, open, readFile, rm, unlink } from "node:fs/promises";
3
3
  import { hostname } from "node:os";
4
- import { basename, dirname, join } from "node:path";
5
- import { setTimeout as delay } from "node:timers/promises";
6
4
 
7
5
  export type LockRecoveryClaim = {
8
6
  path: string;
@@ -60,36 +58,54 @@ const linkedClaim = (lockPath: string, claimPath: string): LockRecoveryClaim =>
60
58
  release: () => rm(claimPath, { force: true }),
61
59
  });
62
60
 
63
- const recoveryHost = Buffer.from(hostname()).toString("base64url");
61
+ type RecoveryMutexOwner = {
62
+ pid: number;
63
+ hostname: string;
64
+ lockToken: string;
65
+ };
64
66
 
65
- const localClaimProcessIsDead = (prefix: string, name: string): boolean => {
66
- const [claimHost, rawPid, id, extra] = name.slice(prefix.length).split(".");
67
- if (claimHost !== recoveryHost || !rawPid || !id || extra !== undefined) return false;
67
+ const recoveryOwnerIsDead = (owner: RecoveryMutexOwner): boolean => {
68
+ if (owner.hostname !== hostname()) return false;
68
69
  try {
69
- process.kill(Number(rawPid), 0);
70
+ process.kill(owner.pid, 0);
70
71
  return false;
71
72
  } catch (error) {
72
73
  return (error as NodeJS.ErrnoException).code === "ESRCH";
73
74
  }
74
75
  };
75
76
 
76
- const hasCompetingRecoveryClaim = async (lockPath: string, claimPath: string): Promise<boolean> => {
77
- const claim = await lstat(claimPath);
78
- const directory = dirname(lockPath);
79
- const claimName = basename(claimPath);
80
- const prefix = `${basename(lockPath)}.recovering-`;
81
- for (const name of await readdir(directory)) {
82
- if (name === claimName || !name.startsWith(prefix)) continue;
83
- const peerPath = join(directory, name);
84
- const peer = await missingAsNull(lstat(peerPath));
85
- if (!peer || peer.dev !== claim.dev || peer.ino !== claim.ino) continue;
86
- if (localClaimProcessIsDead(prefix, name)) {
87
- await unlinkIfExists(peerPath);
88
- continue;
77
+ const acquireRecoveryMutex = async (
78
+ lockPath: string,
79
+ ): Promise<{ path: string; lockToken: string } | null> => {
80
+ const path = `${lockPath}.recovery`;
81
+ while (true) {
82
+ try {
83
+ const owner = await publishOwnedLock(path, {
84
+ pid: process.pid,
85
+ hostname: hostname(),
86
+ });
87
+ return { path, lockToken: owner.lockToken };
88
+ } catch (error) {
89
+ if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
89
90
  }
90
- return true;
91
+ let owner: RecoveryMutexOwner;
92
+ try {
93
+ const value = JSON.parse(await readFile(path, "utf8")) as Partial<RecoveryMutexOwner>;
94
+ if (
95
+ !Number.isSafeInteger(value.pid) ||
96
+ typeof value.hostname !== "string" ||
97
+ typeof value.lockToken !== "string"
98
+ ) {
99
+ return null;
100
+ }
101
+ owner = value as RecoveryMutexOwner;
102
+ } catch (error) {
103
+ if ((error as NodeJS.ErrnoException).code === "ENOENT") continue;
104
+ return null;
105
+ }
106
+ if (!recoveryOwnerIsDead(owner)) return null;
107
+ await releaseOwnedLock(path, owner.lockToken);
91
108
  }
92
- return false;
93
109
  };
94
110
 
95
111
  /**
@@ -97,24 +113,28 @@ const hasCompetingRecoveryClaim = async (lockPath: string, claimPath: string): P
97
113
  * Foreign claims stay conservative because foreign lock owners do too.
98
114
  */
99
115
  export const claimLockForRecovery = async (lockPath: string): Promise<LockRecoveryClaim | null> => {
100
- const claimPath = `${lockPath}.recovering-${recoveryHost}.${process.pid}.${randomUUID()}`;
116
+ const mutex = await acquireRecoveryMutex(lockPath);
117
+ if (!mutex) return null;
118
+ const claimPath = `${lockPath}.recovering-${mutex.lockToken}`;
101
119
  try {
102
120
  await link(lockPath, claimPath);
103
121
  } catch (error) {
104
122
  const code = (error as NodeJS.ErrnoException).code;
123
+ await releaseOwnedLock(mutex.path, mutex.lockToken);
105
124
  if (code === "ENOENT") return null;
106
125
  throw error;
107
126
  }
108
127
  const claim = linkedClaim(lockPath, claimPath);
109
- try {
110
- if (!(await hasCompetingRecoveryClaim(lockPath, claimPath))) return claim;
111
- } catch (error) {
112
- await claim.release();
113
- throw error;
114
- }
115
- await claim.release();
116
- await delay(randomInt(1, 8));
117
- return null;
128
+ return {
129
+ ...claim,
130
+ release: async () => {
131
+ try {
132
+ await claim.release();
133
+ } finally {
134
+ await releaseOwnedLock(mutex.path, mutex.lockToken);
135
+ }
136
+ },
137
+ };
118
138
  };
119
139
 
120
140
  /** Remove only the canonical lock that still carries this acquisition token. */