@prismatic-io/lux 0.0.1 → 0.0.2-preview.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (256) hide show
  1. package/lib/answerers/persona/index.d.ts +2 -1
  2. package/lib/answerers/persona/index.d.ts.map +1 -1
  3. package/lib/answerers/persona/index.js +20 -5
  4. package/lib/answerers/persona/index.js.map +1 -1
  5. package/lib/answerers/terminal/index.js +1 -1
  6. package/lib/answerers/terminal/index.js.map +1 -1
  7. package/lib/assertions/authoring.d.ts +14 -0
  8. package/lib/assertions/authoring.d.ts.map +1 -0
  9. package/lib/assertions/authoring.js +3 -0
  10. package/lib/assertions/authoring.js.map +1 -0
  11. package/lib/assertions/core/helpers.d.ts +2 -7
  12. package/lib/assertions/core/helpers.d.ts.map +1 -1
  13. package/lib/assertions/core/helpers.js +2 -6
  14. package/lib/assertions/core/helpers.js.map +1 -1
  15. package/lib/assertions/core/index.d.ts.map +1 -1
  16. package/lib/assertions/core/index.js +2 -1
  17. package/lib/assertions/core/index.js.map +1 -1
  18. package/lib/assertions/core/resource-checks.d.ts +19 -0
  19. package/lib/assertions/core/resource-checks.d.ts.map +1 -1
  20. package/lib/assertions/core/resource-checks.js +32 -0
  21. package/lib/assertions/core/resource-checks.js.map +1 -1
  22. package/lib/authoring.d.ts +1 -0
  23. package/lib/authoring.d.ts.map +1 -1
  24. package/lib/authoring.js +1 -0
  25. package/lib/authoring.js.map +1 -1
  26. package/lib/cli/program.d.ts.map +1 -1
  27. package/lib/cli/program.js +12 -67
  28. package/lib/cli/program.js.map +1 -1
  29. package/lib/cli/render/reporter.d.ts.map +1 -1
  30. package/lib/cli/render/reporter.js +16 -3
  31. package/lib/cli/render/reporter.js.map +1 -1
  32. package/lib/cli/run-options.d.ts +1 -0
  33. package/lib/cli/run-options.d.ts.map +1 -1
  34. package/lib/cli/run-options.js.map +1 -1
  35. package/lib/cli/view.d.ts +13 -0
  36. package/lib/cli/view.d.ts.map +1 -0
  37. package/lib/cli/view.js +126 -0
  38. package/lib/cli/view.js.map +1 -0
  39. package/lib/core/action-events.d.ts +29 -0
  40. package/lib/core/action-events.d.ts.map +1 -0
  41. package/lib/core/action-events.js +51 -0
  42. package/lib/core/action-events.js.map +1 -0
  43. package/lib/core/agent-activity.d.ts +49 -0
  44. package/lib/core/agent-activity.d.ts.map +1 -0
  45. package/lib/core/agent-activity.js +206 -0
  46. package/lib/core/agent-activity.js.map +1 -0
  47. package/lib/core/agent-events.d.ts +26 -0
  48. package/lib/core/agent-events.d.ts.map +1 -0
  49. package/lib/core/agent-events.js +13 -0
  50. package/lib/core/agent-events.js.map +1 -0
  51. package/lib/core/artifact-evidence.d.ts +8 -0
  52. package/lib/core/artifact-evidence.d.ts.map +1 -1
  53. package/lib/core/artifact-evidence.js +36 -4
  54. package/lib/core/artifact-evidence.js.map +1 -1
  55. package/lib/core/assertion.d.ts +19 -9
  56. package/lib/core/assertion.d.ts.map +1 -1
  57. package/lib/core/assertion.js +26 -0
  58. package/lib/core/assertion.js.map +1 -1
  59. package/lib/core/case-authoring-state.d.ts +12 -0
  60. package/lib/core/case-authoring-state.d.ts.map +1 -0
  61. package/lib/core/case-authoring-state.js +43 -0
  62. package/lib/core/case-authoring-state.js.map +1 -0
  63. package/lib/core/case.d.ts +29 -1
  64. package/lib/core/case.d.ts.map +1 -1
  65. package/lib/core/case.js +14 -6
  66. package/lib/core/case.js.map +1 -1
  67. package/lib/core/driver-metrics.d.ts +98 -0
  68. package/lib/core/driver-metrics.d.ts.map +1 -0
  69. package/lib/core/driver-metrics.js +36 -0
  70. package/lib/core/driver-metrics.js.map +1 -0
  71. package/lib/core/driver.d.ts +3 -0
  72. package/lib/core/driver.d.ts.map +1 -1
  73. package/lib/core/driver.js.map +1 -1
  74. package/lib/core/events.d.ts +36 -0
  75. package/lib/core/events.d.ts.map +1 -1
  76. package/lib/core/events.js +12 -2
  77. package/lib/core/events.js.map +1 -1
  78. package/lib/core/experiment.d.ts +19 -0
  79. package/lib/core/experiment.d.ts.map +1 -1
  80. package/lib/core/experiment.js +2 -0
  81. package/lib/core/experiment.js.map +1 -1
  82. package/lib/core/index.d.ts +4 -0
  83. package/lib/core/index.d.ts.map +1 -1
  84. package/lib/core/index.js +4 -0
  85. package/lib/core/index.js.map +1 -1
  86. package/lib/core/lifecycle-fixtures.d.ts +3 -1
  87. package/lib/core/lifecycle-fixtures.d.ts.map +1 -1
  88. package/lib/core/lifecycle-fixtures.js +4 -2
  89. package/lib/core/lifecycle-fixtures.js.map +1 -1
  90. package/lib/core/lifecycle-hooks.d.ts +9 -0
  91. package/lib/core/lifecycle-hooks.d.ts.map +1 -1
  92. package/lib/core/lifecycle-hooks.js +8 -2
  93. package/lib/core/lifecycle-hooks.js.map +1 -1
  94. package/lib/core/run.d.ts +150 -0
  95. package/lib/core/run.d.ts.map +1 -1
  96. package/lib/core/run.js +9 -0
  97. package/lib/core/run.js.map +1 -1
  98. package/lib/core/wire.d.ts +12 -0
  99. package/lib/core/wire.d.ts.map +1 -1
  100. package/lib/core/wire.js +5 -1
  101. package/lib/core/wire.js.map +1 -1
  102. package/lib/drivers/claude-code/parse-events.d.ts.map +1 -1
  103. package/lib/drivers/claude-code/parse-events.js +73 -8
  104. package/lib/drivers/claude-code/parse-events.js.map +1 -1
  105. package/lib/drivers/codex/app-events.d.ts +9 -2
  106. package/lib/drivers/codex/app-events.d.ts.map +1 -1
  107. package/lib/drivers/codex/app-events.js +93 -12
  108. package/lib/drivers/codex/app-events.js.map +1 -1
  109. package/lib/drivers/codex/index.d.ts.map +1 -1
  110. package/lib/drivers/codex/index.js +10 -14
  111. package/lib/drivers/codex/index.js.map +1 -1
  112. package/lib/drivers/subprocess/index.d.ts.map +1 -1
  113. package/lib/drivers/subprocess/index.js +6 -0
  114. package/lib/drivers/subprocess/index.js.map +1 -1
  115. package/lib/index.d.ts +10 -6
  116. package/lib/index.d.ts.map +1 -1
  117. package/lib/index.js +10 -6
  118. package/lib/index.js.map +1 -1
  119. package/lib/orchestrator/annotation-store.d.ts.map +1 -1
  120. package/lib/orchestrator/annotation-store.js +6 -2
  121. package/lib/orchestrator/annotation-store.js.map +1 -1
  122. package/lib/orchestrator/case-extensions.d.ts +6 -0
  123. package/lib/orchestrator/case-extensions.d.ts.map +1 -0
  124. package/lib/orchestrator/case-extensions.js +28 -0
  125. package/lib/orchestrator/case-extensions.js.map +1 -0
  126. package/lib/orchestrator/compare.d.ts +13 -1
  127. package/lib/orchestrator/compare.d.ts.map +1 -1
  128. package/lib/orchestrator/compare.js +12 -0
  129. package/lib/orchestrator/compare.js.map +1 -1
  130. package/lib/orchestrator/config.d.ts.map +1 -1
  131. package/lib/orchestrator/config.js +0 -2
  132. package/lib/orchestrator/config.js.map +1 -1
  133. package/lib/orchestrator/doctor.d.ts.map +1 -1
  134. package/lib/orchestrator/doctor.js +5 -3
  135. package/lib/orchestrator/doctor.js.map +1 -1
  136. package/lib/orchestrator/experiment-contracts.d.ts +19 -0
  137. package/lib/orchestrator/experiment-contracts.d.ts.map +1 -1
  138. package/lib/orchestrator/experiment-evaluation.d.ts.map +1 -1
  139. package/lib/orchestrator/experiment-evaluation.js +17 -0
  140. package/lib/orchestrator/experiment-evaluation.js.map +1 -1
  141. package/lib/orchestrator/experiment-report.d.ts +19 -0
  142. package/lib/orchestrator/experiment-report.d.ts.map +1 -1
  143. package/lib/orchestrator/experiment-runs.d.ts +38 -0
  144. package/lib/orchestrator/experiment-runs.d.ts.map +1 -1
  145. package/lib/orchestrator/grader-experiment.d.ts +12 -0
  146. package/lib/orchestrator/grader-experiment.d.ts.map +1 -1
  147. package/lib/orchestrator/grading.d.ts +2 -1
  148. package/lib/orchestrator/grading.d.ts.map +1 -1
  149. package/lib/orchestrator/grading.js +16 -4
  150. package/lib/orchestrator/grading.js.map +1 -1
  151. package/lib/orchestrator/index.d.ts +1 -1
  152. package/lib/orchestrator/index.d.ts.map +1 -1
  153. package/lib/orchestrator/index.js +1 -1
  154. package/lib/orchestrator/index.js.map +1 -1
  155. package/lib/orchestrator/list-runs.d.ts +7 -1
  156. package/lib/orchestrator/list-runs.d.ts.map +1 -1
  157. package/lib/orchestrator/list-runs.js +50 -6
  158. package/lib/orchestrator/list-runs.js.map +1 -1
  159. package/lib/orchestrator/loader.d.ts.map +1 -1
  160. package/lib/orchestrator/loader.js +2 -0
  161. package/lib/orchestrator/loader.js.map +1 -1
  162. package/lib/orchestrator/orchestrator.d.ts +3 -0
  163. package/lib/orchestrator/orchestrator.d.ts.map +1 -1
  164. package/lib/orchestrator/orchestrator.js +42 -14
  165. package/lib/orchestrator/orchestrator.js.map +1 -1
  166. package/lib/orchestrator/owned-lock.d.ts.map +1 -1
  167. package/lib/orchestrator/owned-lock.js +52 -38
  168. package/lib/orchestrator/owned-lock.js.map +1 -1
  169. package/lib/orchestrator/regrade.d.ts +6 -0
  170. package/lib/orchestrator/regrade.d.ts.map +1 -1
  171. package/lib/orchestrator/regrade.js +8 -4
  172. package/lib/orchestrator/regrade.js.map +1 -1
  173. package/lib/orchestrator/run-execution.d.ts +2 -1
  174. package/lib/orchestrator/run-execution.d.ts.map +1 -1
  175. package/lib/orchestrator/run-execution.js +20 -5
  176. package/lib/orchestrator/run-execution.js.map +1 -1
  177. package/lib/orchestrator/suite-compare.d.ts +34 -0
  178. package/lib/orchestrator/suite-compare.d.ts.map +1 -1
  179. package/lib/orchestrator/suite-summary.d.ts +68 -0
  180. package/lib/orchestrator/suite-summary.d.ts.map +1 -1
  181. package/lib/orchestrator/suite-summary.js +45 -7
  182. package/lib/orchestrator/suite-summary.js.map +1 -1
  183. package/lib/previewer/artifacts.d.ts +25 -0
  184. package/lib/previewer/artifacts.d.ts.map +1 -0
  185. package/lib/previewer/artifacts.js +180 -0
  186. package/lib/previewer/artifacts.js.map +1 -0
  187. package/lib/previewer/server.d.ts +105 -0
  188. package/lib/previewer/server.d.ts.map +1 -0
  189. package/lib/previewer/server.js +542 -0
  190. package/lib/previewer/server.js.map +1 -0
  191. package/package.json +2 -2
  192. package/skills/lux/SKILL.md +7 -3
  193. package/skills/lux/references/cli.md +25 -2
  194. package/skills/lux/references/eval-authoring.md +158 -4
  195. package/skills/lux-answerer/SKILL.md +1 -1
  196. package/src/answerers/persona/index.ts +21 -4
  197. package/src/answerers/terminal/index.ts +1 -1
  198. package/src/assertions/authoring.ts +81 -0
  199. package/src/assertions/core/helpers.ts +3 -6
  200. package/src/assertions/core/index.ts +2 -0
  201. package/src/assertions/core/resource-checks.ts +35 -0
  202. package/src/authoring.ts +1 -0
  203. package/src/cli/program.ts +14 -93
  204. package/src/cli/render/reporter.ts +18 -3
  205. package/src/cli/run-options.ts +1 -0
  206. package/src/cli/view.ts +168 -0
  207. package/src/core/action-events.ts +59 -0
  208. package/src/core/agent-activity.ts +236 -0
  209. package/src/core/agent-events.ts +17 -0
  210. package/src/core/artifact-evidence.ts +47 -4
  211. package/src/core/assertion.ts +41 -22
  212. package/src/core/case-authoring-state.ts +65 -0
  213. package/src/core/case.ts +57 -6
  214. package/src/core/driver-metrics.ts +51 -0
  215. package/src/core/driver.ts +3 -0
  216. package/src/core/events.ts +14 -2
  217. package/src/core/experiment.ts +2 -0
  218. package/src/core/index.ts +4 -0
  219. package/src/core/lifecycle-fixtures.ts +7 -2
  220. package/src/core/lifecycle-hooks.ts +16 -2
  221. package/src/core/run.ts +9 -0
  222. package/src/core/wire.ts +5 -1
  223. package/src/drivers/claude-code/parse-events.ts +98 -12
  224. package/src/drivers/codex/app-events.ts +116 -16
  225. package/src/drivers/codex/index.ts +16 -14
  226. package/src/drivers/subprocess/index.ts +6 -0
  227. package/src/index.ts +50 -6
  228. package/src/orchestrator/annotation-store.ts +5 -1
  229. package/src/orchestrator/case-extensions.ts +44 -0
  230. package/src/orchestrator/compare.ts +18 -0
  231. package/src/orchestrator/config.ts +0 -2
  232. package/src/orchestrator/doctor.ts +5 -3
  233. package/src/orchestrator/experiment-evaluation.ts +23 -0
  234. package/src/orchestrator/grading.ts +21 -2
  235. package/src/orchestrator/index.ts +1 -0
  236. package/src/orchestrator/list-runs.ts +66 -5
  237. package/src/orchestrator/loader.ts +2 -0
  238. package/src/orchestrator/orchestrator.ts +86 -40
  239. package/src/orchestrator/owned-lock.ts +54 -34
  240. package/src/orchestrator/regrade.ts +8 -7
  241. package/src/orchestrator/run-execution.ts +24 -3
  242. package/src/orchestrator/suite-summary.ts +101 -8
  243. package/src/previewer/artifacts.ts +221 -0
  244. package/src/previewer/server.ts +765 -0
  245. package/viewer/app.css +2075 -0
  246. package/viewer/app.js +606 -0
  247. package/viewer/artifact-view.js +48 -0
  248. package/viewer/charts.js +86 -0
  249. package/viewer/detail.js +1214 -0
  250. package/viewer/format.js +129 -0
  251. package/viewer/index.html +151 -0
  252. package/lib/answerers/scripted/index.d.ts +0 -8
  253. package/lib/answerers/scripted/index.d.ts.map +0 -1
  254. package/lib/answerers/scripted/index.js +0 -163
  255. package/lib/answerers/scripted/index.js.map +0 -1
  256. package/src/answerers/scripted/index.ts +0 -186
@@ -57,11 +57,22 @@ export const AnswerSchema = z.discriminatedUnion("kind", [
57
57
  ]);
58
58
  export type Answer = z.infer<typeof AnswerSchema>;
59
59
 
60
+ export const AgentAttributionSchema = z.object({
61
+ id: z.string().min(1),
62
+ role: z.enum(["main", "subagent"]),
63
+ parentId: z.exactOptional(z.string().min(1)),
64
+ spawnToolUseId: z.exactOptional(z.string().min(1)),
65
+ name: z.exactOptional(z.string().min(1)),
66
+ model: z.exactOptional(z.string().min(1)),
67
+ });
68
+ export type AgentAttribution = z.infer<typeof AgentAttributionSchema>;
69
+
60
70
  export const ProgressSchema = z.object({
61
71
  id: z.string(),
62
72
  kind: z.string(),
63
73
  payload: z.unknown(),
64
74
  ts: z.number(),
75
+ agent: z.exactOptional(AgentAttributionSchema),
65
76
  });
66
77
  export type Progress = z.infer<typeof ProgressSchema>;
67
78
 
@@ -69,8 +80,8 @@ export const DoneSchema = z.object({
69
80
  exitReason: z.literal("done"),
70
81
  /**
71
82
  * Driver-defined summary. The orchestrator recognizes `usage`, `cost`,
72
- * `modelCalls`, and `unpricedModelCalls` when present while preserving room
73
- * for domain-specific measurements.
83
+ * `modelCalls`, `unpricedModelCalls`, and scalar `metrics` when present while
84
+ * preserving room for other domain-specific evidence.
74
85
  */
75
86
  summary: z.unknown().optional(),
76
87
  });
@@ -82,6 +93,7 @@ export type ErrorEventReason = z.infer<typeof ErrorEventReasonSchema>;
82
93
  export const ErrorEventSchema = z.object({
83
94
  exitReason: ErrorEventReasonSchema,
84
95
  reason: z.string(),
96
+ summary: z.unknown().optional(),
85
97
  });
86
98
  export type ErrorEvent = z.infer<typeof ErrorEventSchema>;
87
99
 
@@ -1,4 +1,5 @@
1
1
  import { z } from "zod";
2
+ import { DriverMetricDescriptorSchema } from "./driver-metrics.js";
2
3
  import { ConfidenceConfigSchema } from "./statistics.js";
3
4
  import { TokenUsageSchema } from "./usage.js";
4
5
  import { formatZodIssues } from "./zod.js";
@@ -415,6 +416,7 @@ export const CandidateEvaluationSchema = z.object({
415
416
  passRate: z.number().min(0).max(1),
416
417
  casePassRates: z.record(z.string(), z.number().min(0).max(1)),
417
418
  metrics: z.record(z.string(), z.number()),
419
+ metricDefinitions: z.exactOptional(z.record(z.string(), DriverMetricDescriptorSchema)),
418
420
  metricCalls: z.exactOptional(z.number().int().nonnegative()),
419
421
  judgeCalls: z.exactOptional(z.number().int().nonnegative()),
420
422
  unpricedModelCalls: z.exactOptional(z.number().int().nonnegative()),
package/src/core/index.ts CHANGED
@@ -1,3 +1,6 @@
1
+ export * from "./action-events.js";
2
+ export * from "./agent-activity.js";
3
+ export * from "./agent-events.js";
1
4
  export * from "./annotation.js";
2
5
  export * from "./answerer.js";
3
6
  export * from "./artifact-evidence.js";
@@ -6,6 +9,7 @@ export * from "./assertion.js";
6
9
  export * from "./case.js";
7
10
  export * from "./conversation-turn.js";
8
11
  export * from "./driver.js";
12
+ export * from "./driver-metrics.js";
9
13
  export * from "./emitter.js";
10
14
  export * from "./events.js";
11
15
  export * from "./exec.js";
@@ -2,6 +2,7 @@ import { valueHash } from "./hash.js";
2
2
  import {
3
3
  type CaseLifecycleHookContext,
4
4
  defineLifecycleHooks,
5
+ type FixtureValue,
5
6
  type LifecycleHookCleanup,
6
7
  type LifecycleHooksPlugin,
7
8
  type SuiteLifecycleHookContext,
@@ -22,6 +23,8 @@ export type LifecycleFixturesDefinition = {
22
23
  * to execution identity.
23
24
  */
24
25
  identity?: unknown;
26
+ /** Public, serializable data made available to the selected case's driver. */
27
+ value?: FixtureValue;
25
28
  suite?: LifecycleFixture<SuiteLifecycleHookContext>;
26
29
  case?: LifecycleFixture<CaseLifecycleHookContext>;
27
30
  };
@@ -109,8 +112,8 @@ const activateFixture = async <Context extends SuiteLifecycleHookContext>(
109
112
  * Setup belongs before `use`; teardown belongs after it.
110
113
  */
111
114
  export const defineFixtures = (definition: LifecycleFixturesDefinition): LifecycleHooksPlugin => {
112
- if (!definition.suite && !definition.case) {
113
- throw new TypeError("fixtures plugin must define suite or case");
115
+ if (!definition.suite && !definition.case && definition.value === undefined) {
116
+ throw new TypeError("fixtures plugin must define value, suite, or case");
114
117
  }
115
118
  if (definition.suite !== undefined && typeof definition.suite !== "function") {
116
119
  throw new TypeError("fixtures plugin suite must be a function");
@@ -123,8 +126,10 @@ export const defineFixtures = (definition: LifecycleFixturesDefinition): Lifecyc
123
126
  return defineLifecycleHooks({
124
127
  name: definition.name,
125
128
  pluginVersion: definition.pluginVersion,
129
+ ...(definition.value === undefined ? {} : { value: definition.value }),
126
130
  ...(definition.identity === undefined ? {} : { identity: definition.identity }),
127
131
  implementationHash: valueHash({
132
+ value: definition.value,
128
133
  identity: definition.identity,
129
134
  suite: definition.suite,
130
135
  case: definition.case,
@@ -1,7 +1,15 @@
1
+ import { z } from "zod";
1
2
  import { valueHash } from "./hash.js";
2
3
 
3
4
  export type LifecycleHookCleanup = () => void | Promise<void>;
4
5
  export type LifecycleHookSetupResult = LifecycleHookCleanup | undefined;
6
+ export const FixtureValueSchema = z.json();
7
+ export type FixtureValue = z.infer<typeof FixtureValueSchema>;
8
+
9
+ export type FixtureValueBinding = {
10
+ name: string;
11
+ value: FixtureValue;
12
+ };
5
13
 
6
14
  export type SuiteLifecycleHookItem = {
7
15
  id: string;
@@ -27,6 +35,8 @@ export type LifecycleHooksPlugin = {
27
35
  name: string;
28
36
  /** Author-maintained identity for setup and teardown behavior. */
29
37
  pluginVersion: string;
38
+ /** Public, serializable data made available to the selected case's driver. */
39
+ value?: FixtureValue;
30
40
  /**
31
41
  * Public, non-secret values captured by hook closures and relevant to
32
42
  * execution identity. Included in the implementation hash.
@@ -75,8 +85,11 @@ export const lifecycleHooksPluginIssue = (
75
85
  if (definition.setupCase !== undefined && typeof definition.setupCase !== "function") {
76
86
  return "lifecycle hooks plugin setupCase must be a function";
77
87
  }
78
- if (!definition.setupSuite && !definition.setupCase) {
79
- return "lifecycle hooks plugin must define setupSuite or setupCase";
88
+ if (definition.value !== undefined && !FixtureValueSchema.safeParse(definition.value).success) {
89
+ return "lifecycle hooks plugin value must be JSON-serializable";
90
+ }
91
+ if (!definition.setupSuite && !definition.setupCase && definition.value === undefined) {
92
+ return "lifecycle hooks plugin must define a value, setupSuite, or setupCase";
80
93
  }
81
94
  return undefined;
82
95
  };
@@ -93,6 +106,7 @@ export const defineLifecycleHooks = (definition: LifecycleHooksPlugin): Lifecycl
93
106
  implementationHash:
94
107
  definition.implementationHash ??
95
108
  valueHash({
109
+ value: definition.value,
96
110
  identity: definition.identity,
97
111
  setupSuite: definition.setupSuite,
98
112
  setupCase: definition.setupCase,
package/src/core/run.ts CHANGED
@@ -1,6 +1,9 @@
1
1
  import { z } from "zod";
2
+ import { ActionTimingSchema } from "./action-events.js";
3
+ import { AgentActivitySchema } from "./agent-activity.js";
2
4
  import { EvalCaseSchema } from "./case.js";
3
5
  import { ArtifactSchema } from "./driver.js";
6
+ import { DriverMetricsSchema } from "./driver-metrics.js";
4
7
  import { ErrorEventReasonSchema, ReadyStateSchema, RunEventSchema } from "./events.js";
5
8
  import { ToolOutcomeSummarySchema } from "./tool-events.js";
6
9
  import { RunUsageSchema } from "./usage.js";
@@ -112,6 +115,8 @@ export const RunMetadataSchema = z.object({
112
115
  }),
113
116
  ),
114
117
  cost: z.exactOptional(z.object({ usd: z.exactOptional(z.number()) })),
118
+ /** Driver-defined scalar measurements with display and optimization semantics. */
119
+ driverMetrics: z.exactOptional(DriverMetricsSchema),
115
120
  interruptCounts: z.object({ ask: z.number(), approve: z.number(), custom: z.number() }),
116
121
  /** Legacy raw count of progress events whose kind is `tool-call`. */
117
122
  toolCallCount: z.exactOptional(z.number()),
@@ -124,6 +129,10 @@ export const RunMetadataSchema = z.object({
124
129
  ),
125
130
  /** Wall-clock per phase, derived from the lifecycle event stream. */
126
131
  phaseDurationMs: z.exactOptional(PhaseDurationsSchema),
132
+ /** Driver-reported semantic action latency, measured from `start-driver`. */
133
+ actionTiming: z.exactOptional(ActionTimingSchema),
134
+ /** Per-agent lifecycle and activity derived from the attributed event stream. */
135
+ agentActivity: z.exactOptional(z.array(AgentActivitySchema)),
127
136
  /**
128
137
  * Harness/infrastructure failure that invalidates this run as candidate
129
138
  * evidence. A driver-reported subject failure intentionally leaves this
package/src/core/wire.ts CHANGED
@@ -47,6 +47,7 @@ const ErrorEventSchema = z.object({
47
47
  type: z.literal("error"),
48
48
  exitReason: ErrorEventReasonSchema.default("error"),
49
49
  reason: z.string(),
50
+ summary: z.unknown().optional(),
50
51
  });
51
52
 
52
53
  export const WireEventSchema = z.discriminatedUnion("type", [
@@ -85,7 +86,10 @@ const wireEventToParsed = (event: WireEvent): ParsedDriverEvent => {
85
86
  if (event.type === "done") {
86
87
  return { tag: "done", done: { exitReason: "done", summary: event.summary } };
87
88
  }
88
- return { tag: "error", error: { exitReason: event.exitReason, reason: event.reason } };
89
+ return {
90
+ tag: "error",
91
+ error: { exitReason: event.exitReason, reason: event.reason, summary: event.summary },
92
+ };
89
93
  };
90
94
 
91
95
  /**
@@ -1,5 +1,6 @@
1
1
  import { z } from "zod";
2
2
  import {
3
+ type AgentAttribution,
3
4
  type Interrupt,
4
5
  type ParsedDriverEvent,
5
6
  parseAgentUsage,
@@ -8,6 +9,39 @@ import {
8
9
  toolResultProgress,
9
10
  } from "../../core/index.js";
10
11
 
12
+ const claudeAgent = (parentToolUseId?: string): AgentAttribution =>
13
+ parentToolUseId
14
+ ? {
15
+ id: `claude-code:${parentToolUseId}`,
16
+ role: "subagent",
17
+ parentId: "claude-code",
18
+ spawnToolUseId: parentToolUseId,
19
+ }
20
+ : { id: "claude-code", role: "main" };
21
+
22
+ const isSubagentTool = (name: string): boolean => name === "Agent" || name === "Task";
23
+
24
+ const spawnedClaudeAgent = (
25
+ toolUseId: string,
26
+ input: unknown,
27
+ parentToolUseId?: string,
28
+ ): AgentAttribution => {
29
+ const details =
30
+ typeof input === "object" && input !== null ? (input as Record<string, unknown>) : {};
31
+ const name = [details.name, details.subagent_type, details.agent, details.role].find(
32
+ (value): value is string => typeof value === "string" && value.length > 0,
33
+ );
34
+ const model = typeof details.model === "string" && details.model ? details.model : undefined;
35
+ return {
36
+ id: `claude-code:${toolUseId}`,
37
+ role: "subagent",
38
+ parentId: claudeAgent(parentToolUseId).id,
39
+ spawnToolUseId: toolUseId,
40
+ ...(name ? { name } : {}),
41
+ ...(model ? { model } : {}),
42
+ };
43
+ };
44
+
11
45
  const ContentBlockSchema = z.discriminatedUnion("type", [
12
46
  z.object({ type: z.literal("text"), text: z.string() }),
13
47
  z.object({ type: z.literal("tool_use"), id: z.string(), name: z.string(), input: z.unknown() }),
@@ -88,6 +122,7 @@ type PendingTurn = {
88
122
  asked: boolean;
89
123
  thought: boolean;
90
124
  text?: string;
125
+ agent: AgentAttribution;
91
126
  };
92
127
 
93
128
  type AssistantContent = {
@@ -96,6 +131,7 @@ type AssistantContent = {
96
131
  asked: boolean;
97
132
  thought: boolean;
98
133
  firstText?: string;
134
+ agent: AgentAttribution;
99
135
  };
100
136
 
101
137
  const questionTextOf = (raw: unknown): string =>
@@ -225,6 +261,30 @@ class ClaudeEventParser {
225
261
  if (!block) continue;
226
262
  events.push(block);
227
263
  if (block.tag !== "progress") continue;
264
+ const resultPayload = block.progress.payload as {
265
+ name?: unknown;
266
+ toolUseId?: unknown;
267
+ isError?: unknown;
268
+ };
269
+ if (
270
+ typeof resultPayload.name === "string" &&
271
+ isSubagentTool(resultPayload.name) &&
272
+ typeof resultPayload.toolUseId === "string"
273
+ ) {
274
+ events.push({
275
+ tag: "progress",
276
+ progress: {
277
+ id: `agent-completed-${resultPayload.toolUseId}`,
278
+ kind: "agent-completed",
279
+ payload: {
280
+ agentId: `claude-code:${resultPayload.toolUseId}`,
281
+ status: resultPayload.isError === true ? "failed" : "completed",
282
+ },
283
+ agent: claudeAgent(),
284
+ ts: block.progress.ts,
285
+ },
286
+ });
287
+ }
228
288
  const text = (block.progress.payload as { text?: unknown }).text;
229
289
  const failure = typeof text === "string" ? toolInfrastructureFailure(text) : null;
230
290
  if (failure) {
@@ -244,20 +304,21 @@ class ClaudeEventParser {
244
304
 
245
305
  const toolUseId = block.tool_use_id;
246
306
  const content = block.content ?? [];
307
+ const progress = toolResultProgress(
308
+ this.eventSequence++,
309
+ {
310
+ name: this.toolNames.get(toolUseId) ?? "unknown",
311
+ toolUseId,
312
+ content,
313
+ isError: block.is_error === true,
314
+ text: toolResultText(content),
315
+ ...(parentToolUseId ? { parentToolUseId } : {}),
316
+ },
317
+ Date.now(),
318
+ );
247
319
  return {
248
320
  tag: "progress",
249
- progress: toolResultProgress(
250
- this.eventSequence++,
251
- {
252
- name: this.toolNames.get(toolUseId) ?? "unknown",
253
- toolUseId,
254
- content,
255
- isError: block.is_error === true,
256
- text: toolResultText(content),
257
- ...(parentToolUseId ? { parentToolUseId } : {}),
258
- },
259
- Date.now(),
260
- ),
321
+ progress: { ...progress, agent: claudeAgent(parentToolUseId) },
261
322
  };
262
323
  }
263
324
 
@@ -284,6 +345,7 @@ class ClaudeEventParser {
284
345
  tools: [],
285
346
  asked: false,
286
347
  thought: false,
348
+ agent: claudeAgent(event.parent_tool_use_id),
287
349
  };
288
350
  for (const rawBlock of event.message.content) {
289
351
  const block = ContentBlockSchema.safeParse(rawBlock).data;
@@ -323,6 +385,7 @@ class ClaudeEventParser {
323
385
  kind: "agent-message",
324
386
  payload: { text, ...(parentToolUseId ? { parentToolUseId } : {}) },
325
387
  ts: Date.now(),
388
+ agent: claudeAgent(parentToolUseId),
326
389
  },
327
390
  });
328
391
  }
@@ -354,8 +417,29 @@ class ClaudeEventParser {
354
417
  ...(parentToolUseId ? { parentToolUseId } : {}),
355
418
  },
356
419
  ts: Date.now(),
420
+ agent: claudeAgent(parentToolUseId),
357
421
  },
358
422
  });
423
+ if (isSubagentTool(block.name)) {
424
+ const agent = spawnedClaudeAgent(block.id, block.input, parentToolUseId);
425
+ const details =
426
+ typeof block.input === "object" && block.input !== null
427
+ ? (block.input as Record<string, unknown>)
428
+ : {};
429
+ content.events.push({
430
+ tag: "progress",
431
+ progress: {
432
+ id: `agent-spawn-${block.id}`,
433
+ kind: "agent-spawn",
434
+ payload: {
435
+ agent,
436
+ ...(typeof details.prompt === "string" ? { prompt: details.prompt } : {}),
437
+ },
438
+ agent: claudeAgent(parentToolUseId),
439
+ ts: Date.now(),
440
+ },
441
+ });
442
+ }
359
443
  }
360
444
 
361
445
  private recordTurn(
@@ -376,6 +460,7 @@ class ClaudeEventParser {
376
460
  tools: [],
377
461
  asked: false,
378
462
  thought: false,
463
+ agent: content.agent,
379
464
  };
380
465
  this.pendingTurn.usage = usage;
381
466
  this.pendingTurn.asked ||= content.asked;
@@ -451,6 +536,7 @@ class ClaudeEventParser {
451
536
  kind: "usage",
452
537
  payload: { ...this.pendingTurn.usage, ...(label ? { label } : {}) },
453
538
  ts: Date.now(),
539
+ agent: this.pendingTurn.agent,
454
540
  },
455
541
  };
456
542
  this.pendingTurn = null;
@@ -1,5 +1,7 @@
1
1
  import {
2
+ type AgentAttribution,
2
3
  type ParsedDriverEvent,
4
+ type Progress,
3
5
  type TokenUsage,
4
6
  TURN_GLYPHS,
5
7
  toolResultProgress,
@@ -20,8 +22,98 @@ const resultText = (content: unknown): string => {
20
22
  .join("\n");
21
23
  };
22
24
 
25
+ const agentCompletionStatus = (status: unknown): "completed" | "failed" | "cancelled" => {
26
+ if (status === "failed") return "failed";
27
+ if (status === "cancelled" || status === "interrupted") return "cancelled";
28
+ return "completed";
29
+ };
30
+
31
+ export const appAgentLifecycleProgress = (
32
+ item: RpcRecord,
33
+ phase: "started" | "completed",
34
+ actor?: AgentAttribution,
35
+ ts = Date.now(),
36
+ ): Progress[] => {
37
+ if (item.type !== "collabAgentToolCall" || typeof item.id !== "string") return [];
38
+ const receiverThreadIds = Array.isArray(item.receiverThreadIds)
39
+ ? item.receiverThreadIds.filter((value): value is string => typeof value === "string")
40
+ : [];
41
+ const completionStatus = agentCompletionStatus(item.status);
42
+ return receiverThreadIds.map((receiverThreadId) =>
43
+ phase === "started"
44
+ ? {
45
+ id: `agent-spawn-${item.id}-${receiverThreadId}`,
46
+ kind: "agent-spawn",
47
+ payload: {
48
+ agent: {
49
+ id: receiverThreadId,
50
+ role: "subagent",
51
+ parentId: actor?.id ?? "codex",
52
+ spawnToolUseId: item.id as string,
53
+ ...(typeof item.model === "string" ? { model: item.model } : {}),
54
+ },
55
+ ...(typeof item.prompt === "string" ? { prompt: item.prompt } : {}),
56
+ },
57
+ ...(actor ? { agent: actor } : {}),
58
+ ts,
59
+ }
60
+ : {
61
+ id: `agent-completed-${item.id}-${receiverThreadId}`,
62
+ kind: "agent-completed",
63
+ payload: {
64
+ agentId: receiverThreadId,
65
+ status: completionStatus,
66
+ },
67
+ ...(actor ? { agent: actor } : {}),
68
+ ts,
69
+ },
70
+ );
71
+ };
72
+
23
73
  export type RpcId = string | number;
24
74
  export type RpcRecord = Record<string, unknown>;
75
+
76
+ export class CodexSubagentTracker {
77
+ private readonly spawnToolByThread = new Map<string, string>();
78
+ private readonly parentAgentByThread = new Map<string, string>();
79
+
80
+ record(
81
+ item: RpcRecord | null,
82
+ spawningThreadId: string | undefined,
83
+ mainThreadId: string | null,
84
+ ): void {
85
+ if (item?.type !== "collabAgentToolCall" || typeof item.id !== "string") return;
86
+ const receiverThreadIds = Array.isArray(item.receiverThreadIds)
87
+ ? item.receiverThreadIds.filter((value): value is string => typeof value === "string")
88
+ : [];
89
+ for (const receiverThreadId of receiverThreadIds) {
90
+ this.spawnToolByThread.set(receiverThreadId, item.id);
91
+ if (spawningThreadId) {
92
+ this.parentAgentByThread.set(
93
+ receiverThreadId,
94
+ spawningThreadId === mainThreadId ? "codex" : spawningThreadId,
95
+ );
96
+ }
97
+ }
98
+ }
99
+
100
+ attribution(
101
+ threadId: string | undefined,
102
+ mainThreadId: string | null,
103
+ ): AgentAttribution | undefined {
104
+ if (!threadId) return undefined;
105
+ const spawnToolUseId = this.spawnToolByThread.get(threadId);
106
+ if (!spawnToolUseId) {
107
+ return { id: threadId === mainThreadId ? "codex" : threadId, role: "main" };
108
+ }
109
+ return {
110
+ id: threadId,
111
+ role: "subagent",
112
+ parentId: this.parentAgentByThread.get(threadId) ?? mainThreadId ?? "codex",
113
+ spawnToolUseId,
114
+ };
115
+ }
116
+ }
25
117
  export type AppTurnFacets = {
26
118
  tools: string[];
27
119
  asked: boolean;
@@ -131,8 +223,10 @@ const appItemResultText = (item: RpcRecord, error: RpcRecord | null, content: un
131
223
  export const appItemProgress = (
132
224
  item: RpcRecord,
133
225
  phase: "started" | "completed",
134
- parentToolUseId?: string,
226
+ attribution?: AgentAttribution | string,
135
227
  ): ParsedDriverEvent | null => {
228
+ const agent = typeof attribution === "string" ? undefined : attribution;
229
+ const parentToolUseId = typeof attribution === "string" ? attribution : agent?.spawnToolUseId;
136
230
  const id = typeof item.id === "string" ? item.id : `${String(item.type)}-${Date.now()}`;
137
231
  if (item.type === "agentMessage" && phase === "completed" && typeof item.text === "string") {
138
232
  return {
@@ -142,6 +236,7 @@ export const appItemProgress = (
142
236
  kind: "agent-message",
143
237
  payload: { text: item.text, ...(parentToolUseId ? { parentToolUseId } : {}) },
144
238
  ts: Date.now(),
239
+ ...(agent ? { agent } : {}),
145
240
  },
146
241
  };
147
242
  }
@@ -162,6 +257,7 @@ export const appItemProgress = (
162
257
  ...(parentToolUseId ? { parentToolUseId } : {}),
163
258
  },
164
259
  ts: Date.now(),
260
+ ...(agent ? { agent } : {}),
165
261
  },
166
262
  };
167
263
  }
@@ -184,6 +280,7 @@ export const appItemProgress = (
184
280
  ...(parentToolUseId ? { parentToolUseId } : {}),
185
281
  },
186
282
  ts: Date.now(),
283
+ ...(agent ? { agent } : {}),
187
284
  },
188
285
  };
189
286
  }
@@ -199,20 +296,23 @@ export const appItemProgress = (
199
296
  error !== null;
200
297
  return {
201
298
  tag: "progress",
202
- progress: toolResultProgress(
203
- 0,
204
- {
205
- name: tool,
206
- toolUseId: id,
207
- content,
208
- ...(asRecord(structuredContent)
209
- ? { structuredContent: asRecord(structuredContent) ?? {} }
210
- : {}),
211
- isError,
212
- text: appItemResultText(item, error, content),
213
- ...(parentToolUseId ? { parentToolUseId } : {}),
214
- },
215
- Date.now(),
216
- ),
299
+ progress: {
300
+ ...toolResultProgress(
301
+ 0,
302
+ {
303
+ name: tool,
304
+ toolUseId: id,
305
+ content,
306
+ ...(asRecord(structuredContent)
307
+ ? { structuredContent: asRecord(structuredContent) ?? {} }
308
+ : {}),
309
+ isError,
310
+ text: appItemResultText(item, error, content),
311
+ ...(parentToolUseId ? { parentToolUseId } : {}),
312
+ },
313
+ Date.now(),
314
+ ),
315
+ ...(agent ? { agent } : {}),
316
+ },
217
317
  };
218
318
  };
@@ -24,11 +24,13 @@ import { copyRegularFileNoFollow } from "../shared/secure-copy.js";
24
24
  import { subprocessDriver } from "../subprocess/index.js";
25
25
  import {
26
26
  type AppTurnFacets,
27
+ appAgentLifecycleProgress,
27
28
  appItemProgress,
28
29
  appItemToolName,
29
30
  appTurnLabel,
30
31
  appUsage,
31
32
  asRecord,
33
+ CodexSubagentTracker,
32
34
  messageOf,
33
35
  type RpcId,
34
36
  type RpcRecord,
@@ -316,7 +318,7 @@ class CodexAppServerDriverImpl implements AgentDriver {
316
318
  { resolve: (result: RpcRecord) => void; reject: (error: Error) => void }
317
319
  >();
318
320
  private readonly interrupts = new Map<string, { id: RpcId; method: string; params: RpcRecord }>();
319
- private readonly subagentParents = new Map<string, string>();
321
+ private readonly subagents = new CodexSubagentTracker();
320
322
  private readonly turnFacets = new Map<string, AppTurnFacets>();
321
323
  private interruptCount = 0;
322
324
  private idleTimer: NodeJS.Timeout | null = null;
@@ -702,22 +704,22 @@ class CodexAppServerDriverImpl implements AgentDriver {
702
704
  const toolName = item ? appItemToolName(item) : null;
703
705
  if (toolName && !facets.tools.includes(toolName)) facets.tools.push(toolName);
704
706
  if (turnId) this.turnFacets.set(turnId, facets);
705
- this.recordSubagentParents(item);
706
- const parentToolUseId =
707
- typeof params.threadId === "string" ? this.subagentParents.get(params.threadId) : undefined;
707
+ const threadId = typeof params.threadId === "string" ? params.threadId : undefined;
708
+ this.subagents.record(item, threadId, this.threadId);
709
+ const agent = this.subagents.attribution(threadId, this.threadId);
708
710
  const event = item
709
- ? appItemProgress(item, method === "item/started" ? "started" : "completed", parentToolUseId)
711
+ ? appItemProgress(item, method === "item/started" ? "started" : "completed", agent)
710
712
  : null;
711
713
  if (event?.tag === "progress") await this.emitter.emit("progress", event.progress);
712
- }
713
-
714
- private recordSubagentParents(item: RpcRecord | null): void {
715
- if (item?.type !== "collabAgentToolCall" || typeof item.id !== "string") return;
716
- const receiverThreadIds = Array.isArray(item.receiverThreadIds)
717
- ? item.receiverThreadIds.filter((value): value is string => typeof value === "string")
718
- : [];
719
- for (const receiverThreadId of receiverThreadIds) {
720
- this.subagentParents.set(receiverThreadId, item.id);
714
+ if (item) {
715
+ for (const lifecycle of appAgentLifecycleProgress(
716
+ item,
717
+ method === "item/started" ? "started" : "completed",
718
+ agent,
719
+ event?.tag === "progress" ? event.progress.ts : Date.now(),
720
+ )) {
721
+ await this.emitter.emit("progress", lifecycle);
722
+ }
721
723
  }
722
724
  }
723
725
 
@@ -82,6 +82,7 @@ const KILL_GRACE_MS = 5_000;
82
82
  const DEFAULT_MAX_LINE_BYTES = 10_000_000;
83
83
  const MAX_STDERR_TAIL_CHARS = 8_192;
84
84
  const DEFAULT_MAX_ARTIFACT_FILES = 500;
85
+ const MAX_FIXTURE_VALUES_BYTES = 64 * 1024;
85
86
  const ARTIFACT_SNAPSHOT_MODULE = fileURLToPath(
86
87
  new URL("../../core/artifact-snapshot.js", import.meta.url),
87
88
  );
@@ -139,6 +140,10 @@ class SubprocessDriverImpl implements AgentDriver {
139
140
 
140
141
  async start(ctx: StartContext): Promise<ReadyState> {
141
142
  this.artifactsDir = ctx.artifactsDir;
143
+ const fixtureValues = JSON.stringify(ctx.fixtureValues ?? []);
144
+ if (Buffer.byteLength(fixtureValues) > MAX_FIXTURE_VALUES_BYTES) {
145
+ throw new Error(`subprocess fixture values exceed ${MAX_FIXTURE_VALUES_BYTES} bytes`);
146
+ }
142
147
  const managed = spawnManagedProcess(this.config.command, this.config.args ?? [], {
143
148
  cwd: this.config.cwd ?? ctx.runDir,
144
149
  env: {
@@ -147,6 +152,7 @@ class SubprocessDriverImpl implements AgentDriver {
147
152
  ...(ctx.caseId ? { LUX_CASE_ID: ctx.caseId } : {}),
148
153
  LUX_RUN_DIR: ctx.runDir,
149
154
  LUX_ARTIFACTS_DIR: ctx.artifactsDir,
155
+ LUX_FIXTURE_VALUES: fixtureValues,
150
156
  LUX_PROMPT: ctx.prompt,
151
157
  LUX_ARTIFACT_SNAPSHOT_MODULE: ARTIFACT_SNAPSHOT_MODULE,
152
158
  },