@prismatic-io/lux 0.0.2-preview.18 → 0.0.2-preview.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (244) hide show
  1. package/lib/answerers/persona/index.d.ts +5 -1
  2. package/lib/answerers/persona/index.d.ts.map +1 -1
  3. package/lib/assertions/rubric/index.d.ts +3 -1
  4. package/lib/assertions/rubric/index.d.ts.map +1 -1
  5. package/lib/authoring.d.ts +3 -1
  6. package/lib/authoring.d.ts.map +1 -1
  7. package/lib/authoring.js +2 -1
  8. package/lib/authoring.js.map +1 -1
  9. package/lib/cli/harness-plan.d.ts +26 -0
  10. package/lib/cli/harness-plan.d.ts.map +1 -0
  11. package/lib/cli/harness-plan.js +75 -0
  12. package/lib/cli/harness-plan.js.map +1 -0
  13. package/lib/cli/init-templates.d.ts +2 -2
  14. package/lib/cli/init-templates.d.ts.map +1 -1
  15. package/lib/cli/init-templates.js +11 -4
  16. package/lib/cli/init-templates.js.map +1 -1
  17. package/lib/cli/init.d.ts +2 -0
  18. package/lib/cli/init.d.ts.map +1 -1
  19. package/lib/cli/init.js +6 -3
  20. package/lib/cli/init.js.map +1 -1
  21. package/lib/cli/output-schemas.d.ts +7 -0
  22. package/lib/cli/output-schemas.d.ts.map +1 -1
  23. package/lib/cli/program.d.ts +5 -10
  24. package/lib/cli/program.d.ts.map +1 -1
  25. package/lib/cli/program.js +44 -77
  26. package/lib/cli/program.js.map +1 -1
  27. package/lib/cli/run-options.d.ts +19 -0
  28. package/lib/cli/run-options.d.ts.map +1 -1
  29. package/lib/cli/run-options.js +92 -0
  30. package/lib/cli/run-options.js.map +1 -1
  31. package/lib/core/case.d.ts +2 -0
  32. package/lib/core/case.d.ts.map +1 -1
  33. package/lib/core/case.js +3 -0
  34. package/lib/core/case.js.map +1 -1
  35. package/lib/core/driver.d.ts +13 -0
  36. package/lib/core/driver.d.ts.map +1 -1
  37. package/lib/core/driver.js +10 -0
  38. package/lib/core/driver.js.map +1 -1
  39. package/lib/core/harness-catalog.d.ts +35 -0
  40. package/lib/core/harness-catalog.d.ts.map +1 -0
  41. package/lib/core/harness-catalog.js +45 -0
  42. package/lib/core/harness-catalog.js.map +1 -0
  43. package/lib/core/harness-catalogs/antigravity.d.ts +11 -0
  44. package/lib/core/harness-catalogs/antigravity.d.ts.map +1 -0
  45. package/lib/core/harness-catalogs/antigravity.js +23 -0
  46. package/lib/core/harness-catalogs/antigravity.js.map +1 -0
  47. package/lib/core/harness-catalogs/claude-code.d.ts +12 -0
  48. package/lib/core/harness-catalogs/claude-code.d.ts.map +1 -0
  49. package/lib/core/harness-catalogs/claude-code.js +24 -0
  50. package/lib/core/harness-catalogs/claude-code.js.map +1 -0
  51. package/lib/core/harness-catalogs/codex.d.ts +11 -0
  52. package/lib/core/harness-catalogs/codex.d.ts.map +1 -0
  53. package/lib/core/harness-catalogs/codex.js +26 -0
  54. package/lib/core/harness-catalogs/codex.js.map +1 -0
  55. package/lib/core/harness-catalogs/copilot.d.ts +29 -0
  56. package/lib/core/harness-catalogs/copilot.d.ts.map +1 -0
  57. package/lib/core/harness-catalogs/copilot.js +45 -0
  58. package/lib/core/harness-catalogs/copilot.js.map +1 -0
  59. package/lib/core/harness-catalogs/cursor.d.ts +2 -0
  60. package/lib/core/harness-catalogs/cursor.d.ts.map +1 -0
  61. package/lib/core/harness-catalogs/cursor.js +13 -0
  62. package/lib/core/harness-catalogs/cursor.js.map +1 -0
  63. package/lib/core/harness-catalogs/index.d.ts +67 -0
  64. package/lib/core/harness-catalogs/index.d.ts.map +1 -0
  65. package/lib/core/harness-catalogs/index.js +15 -0
  66. package/lib/core/harness-catalogs/index.js.map +1 -0
  67. package/lib/core/harness-matrix.d.ts +70 -0
  68. package/lib/core/harness-matrix.d.ts.map +1 -0
  69. package/lib/core/harness-matrix.js +194 -0
  70. package/lib/core/harness-matrix.js.map +1 -0
  71. package/lib/core/index.d.ts +3 -0
  72. package/lib/core/index.d.ts.map +1 -1
  73. package/lib/core/index.js +3 -0
  74. package/lib/core/index.js.map +1 -1
  75. package/lib/core/model-cli.d.ts +4 -11
  76. package/lib/core/model-cli.d.ts.map +1 -1
  77. package/lib/core/model-cli.js +23 -17
  78. package/lib/core/model-cli.js.map +1 -1
  79. package/lib/core/platform-process.d.ts +1 -0
  80. package/lib/core/platform-process.d.ts.map +1 -1
  81. package/lib/core/platform-process.js +1 -1
  82. package/lib/core/platform-process.js.map +1 -1
  83. package/lib/core/run.d.ts +4 -0
  84. package/lib/core/run.d.ts.map +1 -1
  85. package/lib/core/run.js +2 -0
  86. package/lib/core/run.js.map +1 -1
  87. package/lib/drivers/antigravity/config.d.ts +46 -0
  88. package/lib/drivers/antigravity/config.d.ts.map +1 -0
  89. package/lib/drivers/antigravity/config.js +39 -0
  90. package/lib/drivers/antigravity/config.js.map +1 -0
  91. package/lib/drivers/antigravity/events.d.ts +4 -0
  92. package/lib/drivers/antigravity/events.d.ts.map +1 -0
  93. package/lib/drivers/antigravity/events.js +193 -0
  94. package/lib/drivers/antigravity/events.js.map +1 -0
  95. package/lib/drivers/antigravity/index.d.ts +60 -0
  96. package/lib/drivers/antigravity/index.d.ts.map +1 -0
  97. package/lib/drivers/antigravity/index.js +90 -0
  98. package/lib/drivers/antigravity/index.js.map +1 -0
  99. package/lib/drivers/claude-code/index.d.ts +93 -7
  100. package/lib/drivers/claude-code/index.d.ts.map +1 -1
  101. package/lib/drivers/claude-code/index.js +14 -8
  102. package/lib/drivers/claude-code/index.js.map +1 -1
  103. package/lib/drivers/codex/config.d.ts +61 -4
  104. package/lib/drivers/codex/config.d.ts.map +1 -1
  105. package/lib/drivers/codex/config.js +19 -5
  106. package/lib/drivers/codex/config.js.map +1 -1
  107. package/lib/drivers/codex/index.d.ts +63 -5
  108. package/lib/drivers/codex/index.d.ts.map +1 -1
  109. package/lib/drivers/codex/index.js +11 -3
  110. package/lib/drivers/codex/index.js.map +1 -1
  111. package/lib/drivers/copilot/config.d.ts +81 -0
  112. package/lib/drivers/copilot/config.d.ts.map +1 -0
  113. package/lib/drivers/copilot/config.js +54 -0
  114. package/lib/drivers/copilot/config.js.map +1 -0
  115. package/lib/drivers/copilot/index.d.ts +103 -0
  116. package/lib/drivers/copilot/index.d.ts.map +1 -0
  117. package/lib/drivers/copilot/index.js +256 -0
  118. package/lib/drivers/copilot/index.js.map +1 -0
  119. package/lib/drivers/cursor/config.d.ts +6 -1
  120. package/lib/drivers/cursor/config.d.ts.map +1 -1
  121. package/lib/drivers/cursor/config.js +2 -3
  122. package/lib/drivers/cursor/config.js.map +1 -1
  123. package/lib/drivers/cursor/index.d.ts +7 -2
  124. package/lib/drivers/cursor/index.d.ts.map +1 -1
  125. package/lib/drivers/cursor/index.js +9 -6
  126. package/lib/drivers/cursor/index.js.map +1 -1
  127. package/lib/drivers/cursor/interaction.d.ts +1 -2
  128. package/lib/drivers/cursor/interaction.d.ts.map +1 -1
  129. package/lib/drivers/cursor/interaction.js +3 -14
  130. package/lib/drivers/cursor/interaction.js.map +1 -1
  131. package/lib/drivers/{cursor/events.d.ts → shared/acp-events.d.ts} +5 -3
  132. package/lib/drivers/shared/acp-events.d.ts.map +1 -0
  133. package/lib/drivers/{cursor/events.js → shared/acp-events.js} +20 -10
  134. package/lib/drivers/shared/acp-events.js.map +1 -0
  135. package/lib/drivers/shared/acp-interaction.d.ts +7 -0
  136. package/lib/drivers/shared/acp-interaction.d.ts.map +1 -0
  137. package/lib/drivers/shared/acp-interaction.js +20 -0
  138. package/lib/drivers/shared/acp-interaction.js.map +1 -0
  139. package/lib/drivers/{cursor → shared}/acp-transport.d.ts +2 -0
  140. package/lib/drivers/shared/acp-transport.d.ts.map +1 -0
  141. package/lib/drivers/{cursor → shared}/acp-transport.js +10 -10
  142. package/lib/drivers/shared/acp-transport.js.map +1 -0
  143. package/lib/index.d.ts +7 -2
  144. package/lib/index.d.ts.map +1 -1
  145. package/lib/index.js +7 -2
  146. package/lib/index.js.map +1 -1
  147. package/lib/optimization/reflective.d.ts +2 -0
  148. package/lib/optimization/reflective.d.ts.map +1 -1
  149. package/lib/orchestrator/comparison-identity.d.ts.map +1 -1
  150. package/lib/orchestrator/comparison-identity.js +1 -0
  151. package/lib/orchestrator/comparison-identity.js.map +1 -1
  152. package/lib/orchestrator/config.d.ts +7 -0
  153. package/lib/orchestrator/config.d.ts.map +1 -1
  154. package/lib/orchestrator/config.js +8 -1
  155. package/lib/orchestrator/config.js.map +1 -1
  156. package/lib/orchestrator/doctor.d.ts.map +1 -1
  157. package/lib/orchestrator/doctor.js +72 -2
  158. package/lib/orchestrator/doctor.js.map +1 -1
  159. package/lib/orchestrator/experiment-context.d.ts +1 -1
  160. package/lib/orchestrator/harness-catalog-drift.d.ts +22 -0
  161. package/lib/orchestrator/harness-catalog-drift.d.ts.map +1 -0
  162. package/lib/orchestrator/harness-catalog-drift.js +90 -0
  163. package/lib/orchestrator/harness-catalog-drift.js.map +1 -0
  164. package/lib/orchestrator/orchestrator.d.ts +9 -1
  165. package/lib/orchestrator/orchestrator.d.ts.map +1 -1
  166. package/lib/orchestrator/orchestrator.js +14 -22
  167. package/lib/orchestrator/orchestrator.js.map +1 -1
  168. package/lib/orchestrator/run-dir.d.ts +2 -0
  169. package/lib/orchestrator/run-dir.d.ts.map +1 -1
  170. package/lib/orchestrator/run-dir.js +1 -1
  171. package/lib/orchestrator/run-dir.js.map +1 -1
  172. package/lib/orchestrator/run-execution.d.ts +1 -0
  173. package/lib/orchestrator/run-execution.d.ts.map +1 -1
  174. package/lib/orchestrator/run-execution.js +2 -0
  175. package/lib/orchestrator/run-execution.js.map +1 -1
  176. package/lib/orchestrator/suite-compare.d.ts +3 -0
  177. package/lib/orchestrator/suite-compare.d.ts.map +1 -1
  178. package/lib/orchestrator/suite-compare.js +27 -17
  179. package/lib/orchestrator/suite-compare.js.map +1 -1
  180. package/lib/orchestrator/suite-runs.d.ts +3 -0
  181. package/lib/orchestrator/suite-runs.d.ts.map +1 -1
  182. package/lib/orchestrator/suite-runs.js +2 -0
  183. package/lib/orchestrator/suite-runs.js.map +1 -1
  184. package/lib/orchestrator/suite-summary.d.ts +9 -0
  185. package/lib/orchestrator/suite-summary.d.ts.map +1 -1
  186. package/lib/orchestrator/suite-summary.js +14 -4
  187. package/lib/orchestrator/suite-summary.js.map +1 -1
  188. package/package.json +1 -1
  189. package/skills/lux-answerer/SKILL.md +1 -1
  190. package/src/authoring.ts +10 -3
  191. package/src/cli/harness-plan.ts +117 -0
  192. package/src/cli/init-templates.ts +12 -5
  193. package/src/cli/init.ts +12 -5
  194. package/src/cli/program.ts +50 -92
  195. package/src/cli/run-options.ts +123 -0
  196. package/src/core/case.ts +3 -0
  197. package/src/core/driver.ts +26 -0
  198. package/src/core/harness-catalog.ts +100 -0
  199. package/src/core/harness-catalogs/antigravity.ts +24 -0
  200. package/src/core/harness-catalogs/claude-code.ts +25 -0
  201. package/src/core/harness-catalogs/codex.ts +27 -0
  202. package/src/core/harness-catalogs/copilot.ts +46 -0
  203. package/src/core/harness-catalogs/cursor.ts +13 -0
  204. package/src/core/harness-catalogs/index.ts +22 -0
  205. package/src/core/harness-matrix.ts +298 -0
  206. package/src/core/index.ts +3 -0
  207. package/src/core/model-cli.ts +38 -21
  208. package/src/core/platform-process.ts +4 -4
  209. package/src/core/run.ts +2 -0
  210. package/src/drivers/antigravity/README.md +46 -0
  211. package/src/drivers/antigravity/config.ts +44 -0
  212. package/src/drivers/antigravity/events.ts +235 -0
  213. package/src/drivers/antigravity/index.ts +109 -0
  214. package/src/drivers/claude-code/index.ts +18 -7
  215. package/src/drivers/codex/config.ts +23 -5
  216. package/src/drivers/codex/index.ts +15 -2
  217. package/src/drivers/copilot/README.md +52 -0
  218. package/src/drivers/copilot/config.ts +58 -0
  219. package/src/drivers/copilot/index.ts +292 -0
  220. package/src/drivers/cursor/README.md +9 -6
  221. package/src/drivers/cursor/config.ts +2 -3
  222. package/src/drivers/cursor/index.ts +10 -6
  223. package/src/drivers/cursor/interaction.ts +3 -19
  224. package/src/drivers/{cursor/events.ts → shared/acp-events.ts} +20 -9
  225. package/src/drivers/shared/acp-interaction.ts +27 -0
  226. package/src/drivers/{cursor → shared}/acp-transport.ts +15 -10
  227. package/src/index.ts +44 -3
  228. package/src/orchestrator/comparison-identity.ts +1 -0
  229. package/src/orchestrator/config.ts +8 -0
  230. package/src/orchestrator/doctor.ts +125 -3
  231. package/src/orchestrator/harness-catalog-drift.ts +132 -0
  232. package/src/orchestrator/orchestrator.ts +31 -26
  233. package/src/orchestrator/run-dir.ts +8 -1
  234. package/src/orchestrator/run-execution.ts +3 -0
  235. package/src/orchestrator/suite-compare.ts +29 -19
  236. package/src/orchestrator/suite-runs.ts +3 -1
  237. package/src/orchestrator/suite-summary.ts +19 -4
  238. package/lib/drivers/cursor/acp-transport.d.ts.map +0 -1
  239. package/lib/drivers/cursor/acp-transport.js.map +0 -1
  240. package/lib/drivers/cursor/events.d.ts.map +0 -1
  241. package/lib/drivers/cursor/events.js.map +0 -1
  242. package/skills/lux/SKILL.md +0 -89
  243. package/skills/lux/references/cli.md +0 -128
  244. package/skills/lux/references/eval-authoring.md +0 -266
package/src/cli/init.ts CHANGED
@@ -13,19 +13,26 @@ import {
13
13
  STARTER_PROMPT_BODY,
14
14
  } from "./init-templates.js";
15
15
 
16
+ const EXPLICIT_MODEL_DRIVERS = new Set(["cursor", "copilot", "antigravity"]);
17
+
16
18
  const InitOptionsSchema = z
17
19
  .object({
18
20
  cwd: z.string().min(1),
19
21
  force: z.boolean().default(false),
20
22
  experiment: z.boolean().default(false),
21
- driver: z.enum(["subprocess", "claude-code", "codex", "cursor"]).optional(),
23
+ driver: z
24
+ .enum(["subprocess", "claude-code", "codex", "cursor", "copilot", "antigravity"])
25
+ .optional(),
22
26
  model: z.string().min(1).optional(),
23
27
  })
24
28
  .strict()
25
- .refine((options) => options.driver !== "cursor" || options.model !== undefined, {
26
- path: ["model"],
27
- message: "Cursor requires an explicit --model",
28
- })
29
+ .refine(
30
+ (options) => !EXPLICIT_MODEL_DRIVERS.has(options.driver ?? "") || options.model !== undefined,
31
+ {
32
+ path: ["model"],
33
+ message: "Cursor, Copilot, and Antigravity require an explicit --model",
34
+ },
35
+ )
29
36
  .refine((options) => !(options.experiment && options.driver === "subprocess"), {
30
37
  path: ["driver"],
31
38
  message: "The deterministic starter is for runs; choose a coding agent for experiments",
@@ -2,14 +2,16 @@ import { randomUUID } from "node:crypto";
2
2
  import { existsSync } from "node:fs";
3
3
  import { lstat, mkdir, realpath, rename, rm, stat, writeFile } from "node:fs/promises";
4
4
  import { createRequire } from "node:module";
5
- import { basename, dirname, extname, join, relative, resolve, sep } from "node:path";
5
+ import { basename, dirname, join, relative, resolve, sep } from "node:path";
6
6
  import { fileURLToPath } from "node:url";
7
7
  import { Cli, Errors, z } from "incur";
8
8
  import { createRubricAssertion } from "../assertions/rubric/index.js";
9
9
  import {
10
10
  type Candidate,
11
11
  GraderAnnotationSchema,
12
+ type HarnessFocus,
12
13
  matchesAnyGlob,
14
+ ReasoningEffortSchema,
13
15
  valueHash,
14
16
  } from "../core/index.js";
15
17
  import {
@@ -17,7 +19,6 @@ import {
17
19
  buildRegistryFromConfig,
18
20
  compareRuns,
19
21
  compareSuiteRuns,
20
- type DiscoveredCase,
21
22
  DoctorReportSchema,
22
23
  discoverCases,
23
24
  ExperimentCampaignPlanSchema,
@@ -46,8 +47,6 @@ import {
46
47
  resolveExperimentCampaignPath,
47
48
  runExperimentCampaign,
48
49
  type SuiteCaseResult,
49
- type SuiteItem,
50
- selectCases,
51
50
  upsertGraderAnnotation,
52
51
  verifyExperimentDecisionIntegrity,
53
52
  verifyExperimentRunIntegrity,
@@ -63,10 +62,15 @@ import { initProject } from "./init.js";
63
62
  import { CompareOutputSchema, ViewOutputSchema } from "./output-schemas.js";
64
63
  import { ConsoleReporter } from "./render/reporter.js";
65
64
  import {
65
+ expandLoop,
66
66
  parseAnnotationAssertion,
67
+ partitionFilters,
68
+ planSuiteItems,
67
69
  type RunOptions,
68
70
  resolveAnswererOverride,
69
71
  resolveConcurrency,
72
+ selectSuite,
73
+ suiteTitle,
70
74
  } from "./run-options.js";
71
75
  import { type ViewOptions, viewCommand, webViewCommand } from "./view.js";
72
76
 
@@ -92,7 +96,14 @@ const initOutput = z.object({
92
96
  const runOutput = z.union([
93
97
  z.object({
94
98
  caseCount: z.number().int(),
95
- cases: z.array(z.object({ path: z.string(), id: z.string() })),
99
+ cases: z.array(
100
+ z.object({
101
+ path: z.string(),
102
+ id: z.string(),
103
+ caseId: z.string().optional(),
104
+ permutationId: z.string().optional(),
105
+ }),
106
+ ),
96
107
  }),
97
108
  z.object({
98
109
  caseCount: z.number().int(),
@@ -156,7 +167,7 @@ export const buildCli = () =>
156
167
  force: z.boolean().optional().describe("Overwrite existing files"),
157
168
  experiment: z.boolean().optional().describe("Also scaffold an optimization campaign"),
158
169
  driver: z
159
- .enum(["subprocess", "claude-code", "codex", "cursor"])
170
+ .enum(["subprocess", "claude-code", "codex", "cursor", "copilot", "antigravity"])
160
171
  .optional()
161
172
  .describe("Starter driver (default: deterministic subprocess)"),
162
173
  model: nonEmptyString.optional().describe("Explicit subject model; required for Cursor"),
@@ -203,6 +214,11 @@ export const buildCli = () =>
203
214
  skipJudge: z.boolean().optional().describe("Skip rubric LLM judge assertions"),
204
215
  runsRoot: nonEmptyString.optional().describe("Run directory root"),
205
216
  profile: nonEmptyString.optional().describe("Named driver and answerer profile"),
217
+ driver: nonEmptyString
218
+ .optional()
219
+ .describe("Focus on one harness driver, replacing the matrix"),
220
+ model: nonEmptyString.optional().describe("Focus on one subject model"),
221
+ effort: ReasoningEffortSchema.optional().describe("Focus on one reasoning effort"),
206
222
  subjectRoot: nonEmptyString.optional().describe("Root bound into subjectPath values"),
207
223
  verbose: z.boolean().optional().describe("Include phase timings and run directories"),
208
224
  }),
@@ -217,6 +233,11 @@ export const buildCli = () =>
217
233
  options: { tag: [] },
218
234
  description: "Run matching cases",
219
235
  },
236
+ {
237
+ args: { filters: ["smoke"] },
238
+ options: { tag: [], driver: "codex", model: "gpt-5.6-luna", effort: "low" },
239
+ description: "Focus on one harness permutation",
240
+ },
220
241
  ],
221
242
  run: async (c) => {
222
243
  const result = await runCommand(c.args.filters, c.options as RunOptions, !c.agent);
@@ -620,8 +641,6 @@ const compareCommand = async (
620
641
  return comparison;
621
642
  };
622
643
 
623
- const caseIdOf = (casePath: string): string => basename(casePath, extname(casePath));
624
-
625
644
  const SIGINT_EXIT_CODE = 130;
626
645
 
627
646
  const physicalDestinationPath = async (path: string): Promise<string> => {
@@ -694,86 +713,6 @@ const isFailure = (outcome: { exitReason: string; casePassed: boolean | null }):
694
713
 
695
714
  const firstLine = (message: string): string => (message.split("\n")[0] ?? message).trim();
696
715
 
697
- const suiteTitle = (
698
- patterns: string[],
699
- tags: string[],
700
- explicit: SuiteItem[],
701
- loop: number,
702
- ): string => {
703
- const parts = [...patterns, ...explicit.map((e) => e.id), ...tags.map((t) => `#${t}`)];
704
- const filter = parts.length > 0 ? ` ${parts.join(" ")}` : "";
705
- const repeat = loop > 1 ? ` ×${loop}` : "";
706
- return `run${filter}${repeat}`;
707
- };
708
-
709
- type FilterPartition = { explicit: SuiteItem[]; patterns: string[] };
710
-
711
- // A filter that names an existing `.ts` file runs that file directly — it may
712
- // sit outside the discovery root. Everything else is a substring pattern — but
713
- // a `.ts` filter that resolves to nothing is almost always a typo'd path, so
714
- // the silent downgrade to a pattern gets called out.
715
- const partitionFilters = (filters: string[], cwd: string): FilterPartition => {
716
- const explicit: SuiteItem[] = [];
717
- const patterns: string[] = [];
718
- for (const f of filters) {
719
- const abs = resolve(cwd, f);
720
- if (f.endsWith(".ts") && existsSync(abs)) {
721
- explicit.push({ path: abs, id: caseIdOf(f) });
722
- } else {
723
- if (f.endsWith(".ts")) {
724
- process.stderr.write(`lux: ${abs} does not exist; treating "${f}" as a pattern\n`);
725
- }
726
- patterns.push(f);
727
- }
728
- }
729
- return { explicit, patterns };
730
- };
731
-
732
- // The selected set — substring/tag matches plus explicitly named files — is
733
- // the same whether it is run, `--list`ed, or `--list-tags`ed. No filters at
734
- // all means "everything discovered"; any filter narrows.
735
- const selectSuite = (
736
- cases: DiscoveredCase[],
737
- { explicit, patterns }: FilterPartition,
738
- tags: string[],
739
- ): DiscoveredCase[] => {
740
- const hasFilter = explicit.length > 0 || patterns.length > 0 || tags.length > 0;
741
- const byPath = new Map<string, DiscoveredCase>();
742
- if (!hasFilter) {
743
- for (const c of cases) byPath.set(c.path, c);
744
- } else {
745
- if (patterns.length > 0 || tags.length > 0) {
746
- for (const c of selectCases(cases, { patterns, tags })) byPath.set(c.path, c);
747
- }
748
- for (const e of explicit) {
749
- const discovered = cases.find((c) => c.path === e.path);
750
- // Discovery knows the case's declared id; the filename-derived one is
751
- // only a fallback for files discovery never saw.
752
- byPath.set(e.path, {
753
- path: e.path,
754
- id: discovered?.id ?? e.id,
755
- relPath: discovered?.relPath ?? e.id,
756
- tags: discovered?.tags ?? [],
757
- });
758
- }
759
- }
760
- return [...byPath.values()].sort((a, b) => a.id.localeCompare(b.id));
761
- };
762
-
763
- // `--loop` repeats each eval before moving to the next; each repeat gets a
764
- // `#n` display suffix so the reporter keeps them apart, while `caseId` keeps
765
- // every repeat filed under one run history.
766
- const expandLoop = (base: SuiteItem[], loop: number): SuiteItem[] =>
767
- loop > 1
768
- ? base.flatMap((b) =>
769
- Array.from({ length: loop }, (_, i) => ({
770
- path: b.path,
771
- id: `${b.id} #${i + 1}`,
772
- caseId: b.caseId ?? b.id,
773
- })),
774
- )
775
- : base;
776
-
777
716
  const experimentCommand = async (
778
717
  mode: "experiment" | "optimize",
779
718
  campaignPath: string,
@@ -824,6 +763,7 @@ const experimentCommand = async (
824
763
  return { experimentDir: result.experimentDir, ...result.manifest };
825
764
  };
826
765
 
766
+ /** A profile pins one driver, so callers pass the project matrix only without one. */
827
767
  const runCommand = async (filters: string[], options: RunOptions, human = false) => {
828
768
  if (options.interactive && options.claudeAnswerer) {
829
769
  throw new Errors.ParseError({
@@ -923,7 +863,17 @@ const runCommand = async (filters: string[], options: RunOptions, human = false)
923
863
  return result;
924
864
  }
925
865
 
926
- const base: SuiteItem[] = selected.map((c) => ({ path: c.path, id: c.id }));
866
+ const focus: HarnessFocus = {
867
+ driver: options.driver,
868
+ model: options.model,
869
+ effort: options.effort,
870
+ };
871
+ const base = await planSuiteItems(selected, {
872
+ matrix: profile ? undefined : config.matrix,
873
+ defaultDriver,
874
+ registry,
875
+ focus,
876
+ });
927
877
 
928
878
  // Collect-only: answer "what would run?" without spawning a single agent.
929
879
  if (options.list) {
@@ -931,7 +881,15 @@ const runCommand = async (filters: string[], options: RunOptions, human = false)
931
881
  process.stdout.write(
932
882
  `${base.length} cases\n${base.map(({ id, path }) => ` ${id} ${path}`).join("\n")}\n`,
933
883
  );
934
- return { caseCount: base.length, cases: base };
884
+ return {
885
+ caseCount: base.length,
886
+ cases: base.map(({ path, id, caseId, permutationId }) => ({
887
+ path,
888
+ id,
889
+ ...(caseId !== undefined ? { caseId } : {}),
890
+ ...(permutationId !== undefined ? { permutationId } : {}),
891
+ })),
892
+ };
935
893
  }
936
894
 
937
895
  const loop = options.loop ?? 1;
@@ -943,7 +901,7 @@ const runCommand = async (filters: string[], options: RunOptions, human = false)
943
901
  const reporter = human ? new ConsoleReporter({ verbose: options.verbose ?? false }) : undefined;
944
902
  using controller = abortOnInterrupt();
945
903
  reporter?.begin({
946
- title: suiteTitle(patterns, tags, explicit, loop),
904
+ title: suiteTitle(patterns, tags, explicit, loop, focus),
947
905
  caseIds: items.map((i) => i.id),
948
906
  runsRoot,
949
907
  loop,
@@ -991,4 +949,4 @@ const runCommand = async (filters: string[], options: RunOptions, human = false)
991
949
  };
992
950
 
993
951
  // Internal exports for testing
994
- export { expandLoop, isFailure, partitionFilters, resolveConcurrency, selectSuite };
952
+ export { isFailure, resolveConcurrency };
@@ -1,4 +1,9 @@
1
+ import { existsSync } from "node:fs";
2
+ import { basename, extname, resolve } from "node:path";
1
3
  import { Errors } from "incur";
4
+ import type { HarnessFocus, ReasoningEffort } from "../core/index.js";
5
+ import { type DiscoveredCase, type SuiteItem, selectCases } from "../orchestrator/index.js";
6
+ import { planHarnessItems } from "./harness-plan.js";
2
7
 
3
8
  export type RunOptions = {
4
9
  tag?: string[];
@@ -11,6 +16,9 @@ export type RunOptions = {
11
16
  skipJudge?: boolean;
12
17
  runsRoot?: string;
13
18
  profile?: string;
19
+ driver?: string;
20
+ model?: string;
21
+ effort?: ReasoningEffort;
14
22
  subjectRoot?: string;
15
23
  verbose?: boolean;
16
24
  };
@@ -64,3 +72,118 @@ export const resolveAnswererOverride = (options: RunOptions): string | undefined
64
72
  if (options.claudeAnswerer) return "claude-code";
65
73
  return undefined;
66
74
  };
75
+
76
+ const caseIdOf = (casePath: string): string => basename(casePath, extname(casePath));
77
+
78
+ export const suiteTitle = (
79
+ patterns: string[],
80
+ tags: string[],
81
+ explicit: SuiteItem[],
82
+ loop: number,
83
+ focus: HarnessFocus,
84
+ ): string => {
85
+ const parts = [...patterns, ...explicit.map((e) => e.id), ...tags.map((t) => `#${t}`)];
86
+ const filter = parts.length > 0 ? ` ${parts.join(" ")}` : "";
87
+ const focusParts = [focus.driver, focus.model, focus.effort].filter(
88
+ (part): part is string => part !== undefined,
89
+ );
90
+ const focused = focusParts.length > 0 ? ` @${focusParts.join(":")}` : "";
91
+ const repeat = loop > 1 ? ` ×${loop}` : "";
92
+ return `run${filter}${focused}${repeat}`;
93
+ };
94
+
95
+ type FilterPartition = { explicit: SuiteItem[]; patterns: string[] };
96
+
97
+ // A filter that names an existing `.ts` file runs that file directly — it may
98
+ // sit outside the discovery root. Everything else is a substring pattern — but
99
+ // a `.ts` filter that resolves to nothing is almost always a typo'd path, so
100
+ // the silent downgrade to a pattern gets called out.
101
+ export const partitionFilters = (filters: string[], cwd: string): FilterPartition => {
102
+ const explicit: SuiteItem[] = [];
103
+ const patterns: string[] = [];
104
+ for (const f of filters) {
105
+ const abs = resolve(cwd, f);
106
+ if (f.endsWith(".ts") && existsSync(abs)) {
107
+ explicit.push({ path: abs, id: caseIdOf(f) });
108
+ } else {
109
+ if (f.endsWith(".ts")) {
110
+ process.stderr.write(`lux: ${abs} does not exist; treating "${f}" as a pattern\n`);
111
+ }
112
+ patterns.push(f);
113
+ }
114
+ }
115
+ return { explicit, patterns };
116
+ };
117
+
118
+ // The selected set — substring/tag matches plus explicitly named files — is
119
+ // the same whether it is run, `--list`ed, or `--list-tags`ed. No filters at
120
+ // all means "everything discovered"; any filter narrows.
121
+ export const selectSuite = (
122
+ cases: DiscoveredCase[],
123
+ { explicit, patterns }: FilterPartition,
124
+ tags: string[],
125
+ ): DiscoveredCase[] => {
126
+ const hasFilter = explicit.length > 0 || patterns.length > 0 || tags.length > 0;
127
+ const byPath = new Map<string, DiscoveredCase>();
128
+ if (!hasFilter) {
129
+ for (const c of cases) byPath.set(c.path, c);
130
+ } else {
131
+ if (patterns.length > 0 || tags.length > 0) {
132
+ for (const c of selectCases(cases, { patterns, tags })) byPath.set(c.path, c);
133
+ }
134
+ for (const e of explicit) {
135
+ const discovered = cases.find((c) => c.path === e.path);
136
+ // Discovery knows the case's declared id; the filename-derived one is
137
+ // only a fallback for files discovery never saw.
138
+ byPath.set(e.path, {
139
+ path: e.path,
140
+ id: discovered?.id ?? e.id,
141
+ relPath: discovered?.relPath ?? e.id,
142
+ tags: discovered?.tags ?? [],
143
+ });
144
+ }
145
+ }
146
+ return [...byPath.values()].sort((a, b) => a.id.localeCompare(b.id));
147
+ };
148
+
149
+ // `--loop` repeats each eval before moving to the next; each repeat gets a
150
+ // `#n` display suffix so the reporter keeps them apart, while `caseId` keeps
151
+ // every repeat filed under one run history.
152
+ export const expandLoop = (base: SuiteItem[], loop: number): SuiteItem[] =>
153
+ loop > 1
154
+ ? base.flatMap((b) =>
155
+ Array.from({ length: loop }, (_, i) => ({
156
+ ...b,
157
+ id: `${b.id} #${i + 1}`,
158
+ caseId: b.caseId ?? b.id,
159
+ })),
160
+ )
161
+ : base;
162
+
163
+ export const planSuiteItems = async (
164
+ selected: readonly { path: string; id: string }[],
165
+ options: Parameters<typeof planHarnessItems>[1],
166
+ ): Promise<SuiteItem[]> => {
167
+ const plan = await planHarnessItems(selected, options);
168
+ for (const skipped of plan.unavailable) {
169
+ process.stderr.write(
170
+ `lux: skipping ${skipped.id}: harness command not found: ${skipped.commands.join(", ")}\n`,
171
+ );
172
+ }
173
+ if (plan.excluded.length > 0) {
174
+ process.stderr.write(
175
+ `lux: the harness focus excludes ${plan.excluded.length} pinned case${plan.excluded.length === 1 ? "" : "s"}: ${plan.excluded.join(", ")}\n`,
176
+ );
177
+ }
178
+ if (plan.items.length === 0) {
179
+ throw new Errors.IncurError({
180
+ code: plan.unavailable.length > 0 ? "NO_RUNNABLE_HARNESSES" : "NO_CASES_MATCHED",
181
+ message:
182
+ plan.unavailable.length > 0
183
+ ? "No cases remain: all selected matrix harnesses are unavailable."
184
+ : "No cases remain after applying the harness focus.",
185
+ exitCode: 1,
186
+ });
187
+ }
188
+ return plan.items;
189
+ };
package/src/core/case.ts CHANGED
@@ -3,6 +3,7 @@ import type { AssertionPlugin } from "./assertion.js";
3
3
  import { collectEvalCaseExtensions, setEvalCaseExtensions } from "./case-authoring-state.js";
4
4
  import { DriverCapabilitySchema } from "./driver.js";
5
5
  import { EvaluationMetadataSchema } from "./evaluation-clusters.js";
6
+ import { HarnessMatrixSchema } from "./harness-matrix.js";
6
7
  import type { LifecycleHooksPlugin } from "./lifecycle-hooks.js";
7
8
  import { formatZodIssues } from "./zod.js";
8
9
 
@@ -46,6 +47,8 @@ const EvalCaseObjectSchema = z.object({
46
47
  prompt: z.string().min(1, "prompt is required"),
47
48
  persona: z.string().min(1).optional(),
48
49
  driver: DriverRefSchema.optional(),
50
+ /** Harness permutations this case runs through instead of the project matrix. */
51
+ matrix: HarnessMatrixSchema.optional(),
49
52
  answerer: AnswererRefSchema.optional(),
50
53
  fixtures: FixturesRefSchema.optional(),
51
54
  requires: z.array(DriverCapabilitySchema).optional(),
@@ -1,6 +1,7 @@
1
1
  import { z } from "zod";
2
2
  import type { EventHandler } from "./emitter.js";
3
3
  import type { Answer, Done, ErrorEvent, Interrupt, Progress, ReadyState } from "./events.js";
4
+ import type { ReasoningEffort } from "./harness-catalog.js";
4
5
  import type { FixtureValueBinding } from "./lifecycle-hooks.js";
5
6
 
6
7
  export const ArtifactSchema = z.object({
@@ -97,6 +98,12 @@ export type DriverExperiment<Cfg> = {
97
98
  identity(config: Cfg, context: { rootDir: string }): unknown | Promise<unknown>;
98
99
  };
99
100
 
101
+ /** Model and reasoning effort chosen by a harness matrix permutation. */
102
+ export type HarnessSelection = {
103
+ model?: string | undefined;
104
+ effort?: ReasoningEffort | undefined;
105
+ };
106
+
100
107
  export type DriverPlugin<Cfg = unknown> = {
101
108
  name: string;
102
109
  configSchema?: z.ZodTypeAny;
@@ -104,8 +111,27 @@ export type DriverPlugin<Cfg = unknown> = {
104
111
  runtime?: (config: Cfg) => DriverRuntime;
105
112
  /** Ordinary runs remain supported when controlled experiments are not declared. */
106
113
  experiment?: DriverExperiment<Cfg>;
114
+ /**
115
+ * Apply a matrix selection to raw (unvalidated) configuration. Drivers
116
+ * without a model concept leave this undefined and reject selection.
117
+ */
118
+ selectHarness?: (config: unknown, selection: HarnessSelection) => unknown;
107
119
  };
108
120
 
121
+ /** Selection for drivers whose configuration uses `model` and `reasoningEffort`. */
122
+ export const selectModelAndReasoningEffort = (
123
+ config: unknown,
124
+ selection: HarnessSelection,
125
+ ): unknown => ({
126
+ ...(config && typeof config === "object" && !Array.isArray(config)
127
+ ? (config as Record<string, unknown>)
128
+ : {}),
129
+ ...(selection.model !== undefined ? { model: selection.model } : {}),
130
+ ...(selection.effort !== undefined || selection.model !== undefined
131
+ ? { reasoningEffort: selection.effort }
132
+ : {}),
133
+ });
134
+
109
135
  export const defineDriver = <S extends z.ZodType>(
110
136
  definition: Omit<DriverPlugin<z.output<S>>, "configSchema"> & { configSchema: S },
111
137
  ): Omit<DriverPlugin<z.output<S>>, "configSchema"> & { configSchema: S } => definition;
@@ -0,0 +1,100 @@
1
+ import { z } from "zod";
2
+
3
+ export const HARNESS_EFFORTS = [
4
+ "none",
5
+ "minimal",
6
+ "low",
7
+ "medium",
8
+ "high",
9
+ "xhigh",
10
+ "max",
11
+ "ultra",
12
+ ] as const;
13
+ export const ReasoningEffortSchema = z.enum(HARNESS_EFFORTS);
14
+ export type ReasoningEffort = z.infer<typeof ReasoningEffortSchema>;
15
+
16
+ export type CatalogModels = Readonly<Record<string, readonly ReasoningEffort[]>>;
17
+
18
+ export type HarnessCatalog<
19
+ D extends string = string,
20
+ E extends readonly ReasoningEffort[] = readonly ReasoningEffort[],
21
+ M extends CatalogModels | undefined = CatalogModels | undefined,
22
+ > = {
23
+ readonly driver: D;
24
+ readonly verified: string;
25
+ readonly sources: readonly string[];
26
+ readonly efforts: E;
27
+ readonly models: M;
28
+ };
29
+
30
+ export type ClosedHarnessCatalog = HarnessCatalog<
31
+ string,
32
+ readonly ReasoningEffort[],
33
+ CatalogModels
34
+ >;
35
+
36
+ export const defineHarnessCatalog = <
37
+ const D extends string,
38
+ const E extends readonly ReasoningEffort[],
39
+ const M extends CatalogModels | undefined,
40
+ >(
41
+ catalog: HarnessCatalog<D, E, M>,
42
+ ): HarnessCatalog<D, E, M> => catalog;
43
+
44
+ export const isClosedCatalog = (catalog: HarnessCatalog): catalog is ClosedHarnessCatalog =>
45
+ catalog.models !== undefined;
46
+
47
+ export const catalogModelIds = (catalog: HarnessCatalog): string[] =>
48
+ Object.keys(catalog.models ?? {});
49
+
50
+ export const catalogEfforts = (
51
+ catalog: HarnessCatalog,
52
+ model: string,
53
+ ): readonly ReasoningEffort[] | undefined => catalog.models?.[model];
54
+
55
+ export const defaultCatalogEffort = (
56
+ catalog: HarnessCatalog,
57
+ model: string,
58
+ ): ReasoningEffort | undefined => {
59
+ if (!isClosedCatalog(catalog)) return undefined;
60
+ const accepted = catalog.models[model] ?? catalog.efforts;
61
+ return accepted.includes("low") ? "low" : accepted[0];
62
+ };
63
+
64
+ export const describeAcceptedEfforts = (
65
+ catalog: HarnessCatalog,
66
+ model: string,
67
+ accepted: readonly ReasoningEffort[],
68
+ ): string =>
69
+ accepted.length === 0
70
+ ? `model '${model}' has no reasoning effort in ${catalog.driver}`
71
+ : `model '${model}' accepts reasoning effort ${accepted.join(", ")} in ${catalog.driver}`;
72
+
73
+ export const effortIssue = (
74
+ catalog: HarnessCatalog,
75
+ model: string,
76
+ effort: ReasoningEffort | undefined,
77
+ ): string | undefined => {
78
+ if (effort === undefined) return undefined;
79
+ const accepted = catalogEfforts(catalog, model);
80
+ if (accepted === undefined || accepted.includes(effort)) return undefined;
81
+ return describeAcceptedEfforts(catalog, model, accepted);
82
+ };
83
+
84
+ export type CatalogEffortConfig = {
85
+ model: string;
86
+ reasoningEffort?: ReasoningEffort | undefined;
87
+ };
88
+
89
+ export const applyCatalogEffort =
90
+ (catalog: HarnessCatalog) =>
91
+ <C extends CatalogEffortConfig>(config: C, ctx: z.RefinementCtx): C => {
92
+ const issue = effortIssue(catalog, config.model, config.reasoningEffort);
93
+ if (issue) {
94
+ ctx.addIssue({ code: "custom", path: ["reasoningEffort"], message: issue });
95
+ return config;
96
+ }
97
+ if (config.reasoningEffort !== undefined) return config;
98
+ const effort = defaultCatalogEffort(catalog, config.model);
99
+ return effort === undefined ? config : { ...config, reasoningEffort: effort };
100
+ };
@@ -0,0 +1,24 @@
1
+ import { defineHarnessCatalog } from "../harness-catalog.js";
2
+
3
+ const flash = ["low", "medium", "high"] as const;
4
+
5
+ export const antigravityCatalog = defineHarnessCatalog({
6
+ driver: "antigravity",
7
+ verified: "2026-09-18",
8
+ sources: [
9
+ "https://antigravity.google/docs/models/",
10
+ "https://antigravity.google/docs/cli/headless/",
11
+ "agy models (needs a login)",
12
+ ],
13
+ efforts: flash,
14
+ models: {
15
+ "gemini-3.8-flash": flash,
16
+ "gemini-3.7-flash": flash,
17
+ "gemini-3.6-flash": flash,
18
+ "gemini-3.5-flash": flash,
19
+ "gemini-3.1-pro": ["low", "high"],
20
+ "claude-sonnet-4-6": [],
21
+ "claude-opus-4-6-thinking": [],
22
+ "gpt-oss-120b-medium": [],
23
+ },
24
+ });
@@ -0,0 +1,25 @@
1
+ import { defineHarnessCatalog } from "../harness-catalog.js";
2
+
3
+ const full = ["low", "medium", "high", "xhigh", "max"] as const;
4
+ const withoutXhigh = ["low", "medium", "high", "max"] as const;
5
+
6
+ export const claudeCodeCatalog = defineHarnessCatalog({
7
+ driver: "claude-code",
8
+ verified: "2026-09-18",
9
+ sources: [
10
+ "https://code.claude.com/docs/en/model-config",
11
+ "https://platform.claude.com/docs/en/about-claude/models/overview",
12
+ ],
13
+ efforts: full,
14
+ models: {
15
+ "claude-fable-5-1": full,
16
+ "claude-fable-5": full,
17
+ "claude-opus-5": full,
18
+ "claude-opus-4-8": full,
19
+ "claude-opus-4-7": full,
20
+ "claude-sonnet-5": full,
21
+ "claude-opus-4-6": withoutXhigh,
22
+ "claude-sonnet-4-6": withoutXhigh,
23
+ "claude-haiku-4-5": [],
24
+ },
25
+ });
@@ -0,0 +1,27 @@
1
+ import { defineHarnessCatalog } from "../harness-catalog.js";
2
+
3
+ const throughUltra = ["low", "medium", "high", "xhigh", "max", "ultra"] as const;
4
+ const throughMax = ["low", "medium", "high", "xhigh", "max"] as const;
5
+ const throughXhigh = ["low", "medium", "high", "xhigh"] as const;
6
+ const noneThroughXhigh = ["none", "low", "medium", "high", "xhigh"] as const;
7
+
8
+ export const codexCatalog = defineHarnessCatalog({
9
+ driver: "codex",
10
+ verified: "2026-09-18",
11
+ sources: [
12
+ "https://learn.chatgpt.com/docs/models",
13
+ "https://developers.openai.com/api/docs/guides/reasoning",
14
+ "codex app-server model/list (mirrored in $CODEX_HOME/models_cache.json)",
15
+ ],
16
+ efforts: ["none", "low", "medium", "high", "xhigh", "max", "ultra"],
17
+ models: {
18
+ "gpt-6-astra": throughUltra,
19
+ "gpt-5.6-sol": throughUltra,
20
+ "gpt-5.6-terra": throughUltra,
21
+ "gpt-5.6-luna": throughMax,
22
+ "gpt-5.5": throughXhigh,
23
+ "gpt-5.4": noneThroughXhigh,
24
+ "gpt-5.4-mini": noneThroughXhigh,
25
+ "gpt-5.3-codex": throughXhigh,
26
+ },
27
+ });