@prismatic-io/lux 0.0.2-preview.18 → 0.0.2-preview.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/answerers/persona/index.d.ts +5 -1
- package/lib/answerers/persona/index.d.ts.map +1 -1
- package/lib/assertions/rubric/index.d.ts +3 -1
- package/lib/assertions/rubric/index.d.ts.map +1 -1
- package/lib/authoring.d.ts +3 -1
- package/lib/authoring.d.ts.map +1 -1
- package/lib/authoring.js +2 -1
- package/lib/authoring.js.map +1 -1
- package/lib/cli/harness-plan.d.ts +26 -0
- package/lib/cli/harness-plan.d.ts.map +1 -0
- package/lib/cli/harness-plan.js +75 -0
- package/lib/cli/harness-plan.js.map +1 -0
- package/lib/cli/init-templates.d.ts +2 -2
- package/lib/cli/init-templates.d.ts.map +1 -1
- package/lib/cli/init-templates.js +11 -4
- package/lib/cli/init-templates.js.map +1 -1
- package/lib/cli/init.d.ts +2 -0
- package/lib/cli/init.d.ts.map +1 -1
- package/lib/cli/init.js +6 -3
- package/lib/cli/init.js.map +1 -1
- package/lib/cli/output-schemas.d.ts +7 -0
- package/lib/cli/output-schemas.d.ts.map +1 -1
- package/lib/cli/program.d.ts +5 -10
- package/lib/cli/program.d.ts.map +1 -1
- package/lib/cli/program.js +44 -77
- package/lib/cli/program.js.map +1 -1
- package/lib/cli/run-options.d.ts +19 -0
- package/lib/cli/run-options.d.ts.map +1 -1
- package/lib/cli/run-options.js +92 -0
- package/lib/cli/run-options.js.map +1 -1
- package/lib/core/case.d.ts +2 -0
- package/lib/core/case.d.ts.map +1 -1
- package/lib/core/case.js +3 -0
- package/lib/core/case.js.map +1 -1
- package/lib/core/driver.d.ts +13 -0
- package/lib/core/driver.d.ts.map +1 -1
- package/lib/core/driver.js +10 -0
- package/lib/core/driver.js.map +1 -1
- package/lib/core/harness-catalog.d.ts +35 -0
- package/lib/core/harness-catalog.d.ts.map +1 -0
- package/lib/core/harness-catalog.js +45 -0
- package/lib/core/harness-catalog.js.map +1 -0
- package/lib/core/harness-catalogs/antigravity.d.ts +11 -0
- package/lib/core/harness-catalogs/antigravity.d.ts.map +1 -0
- package/lib/core/harness-catalogs/antigravity.js +23 -0
- package/lib/core/harness-catalogs/antigravity.js.map +1 -0
- package/lib/core/harness-catalogs/claude-code.d.ts +12 -0
- package/lib/core/harness-catalogs/claude-code.d.ts.map +1 -0
- package/lib/core/harness-catalogs/claude-code.js +24 -0
- package/lib/core/harness-catalogs/claude-code.js.map +1 -0
- package/lib/core/harness-catalogs/codex.d.ts +11 -0
- package/lib/core/harness-catalogs/codex.d.ts.map +1 -0
- package/lib/core/harness-catalogs/codex.js +26 -0
- package/lib/core/harness-catalogs/codex.js.map +1 -0
- package/lib/core/harness-catalogs/copilot.d.ts +29 -0
- package/lib/core/harness-catalogs/copilot.d.ts.map +1 -0
- package/lib/core/harness-catalogs/copilot.js +45 -0
- package/lib/core/harness-catalogs/copilot.js.map +1 -0
- package/lib/core/harness-catalogs/cursor.d.ts +2 -0
- package/lib/core/harness-catalogs/cursor.d.ts.map +1 -0
- package/lib/core/harness-catalogs/cursor.js +13 -0
- package/lib/core/harness-catalogs/cursor.js.map +1 -0
- package/lib/core/harness-catalogs/index.d.ts +67 -0
- package/lib/core/harness-catalogs/index.d.ts.map +1 -0
- package/lib/core/harness-catalogs/index.js +15 -0
- package/lib/core/harness-catalogs/index.js.map +1 -0
- package/lib/core/harness-matrix.d.ts +70 -0
- package/lib/core/harness-matrix.d.ts.map +1 -0
- package/lib/core/harness-matrix.js +194 -0
- package/lib/core/harness-matrix.js.map +1 -0
- package/lib/core/index.d.ts +3 -0
- package/lib/core/index.d.ts.map +1 -1
- package/lib/core/index.js +3 -0
- package/lib/core/index.js.map +1 -1
- package/lib/core/model-cli.d.ts +4 -11
- package/lib/core/model-cli.d.ts.map +1 -1
- package/lib/core/model-cli.js +23 -17
- package/lib/core/model-cli.js.map +1 -1
- package/lib/core/platform-process.d.ts +1 -0
- package/lib/core/platform-process.d.ts.map +1 -1
- package/lib/core/platform-process.js +1 -1
- package/lib/core/platform-process.js.map +1 -1
- package/lib/core/run.d.ts +4 -0
- package/lib/core/run.d.ts.map +1 -1
- package/lib/core/run.js +2 -0
- package/lib/core/run.js.map +1 -1
- package/lib/drivers/antigravity/config.d.ts +46 -0
- package/lib/drivers/antigravity/config.d.ts.map +1 -0
- package/lib/drivers/antigravity/config.js +39 -0
- package/lib/drivers/antigravity/config.js.map +1 -0
- package/lib/drivers/antigravity/events.d.ts +4 -0
- package/lib/drivers/antigravity/events.d.ts.map +1 -0
- package/lib/drivers/antigravity/events.js +193 -0
- package/lib/drivers/antigravity/events.js.map +1 -0
- package/lib/drivers/antigravity/index.d.ts +60 -0
- package/lib/drivers/antigravity/index.d.ts.map +1 -0
- package/lib/drivers/antigravity/index.js +90 -0
- package/lib/drivers/antigravity/index.js.map +1 -0
- package/lib/drivers/claude-code/index.d.ts +93 -7
- package/lib/drivers/claude-code/index.d.ts.map +1 -1
- package/lib/drivers/claude-code/index.js +14 -8
- package/lib/drivers/claude-code/index.js.map +1 -1
- package/lib/drivers/codex/config.d.ts +61 -4
- package/lib/drivers/codex/config.d.ts.map +1 -1
- package/lib/drivers/codex/config.js +19 -5
- package/lib/drivers/codex/config.js.map +1 -1
- package/lib/drivers/codex/index.d.ts +63 -5
- package/lib/drivers/codex/index.d.ts.map +1 -1
- package/lib/drivers/codex/index.js +11 -3
- package/lib/drivers/codex/index.js.map +1 -1
- package/lib/drivers/copilot/config.d.ts +81 -0
- package/lib/drivers/copilot/config.d.ts.map +1 -0
- package/lib/drivers/copilot/config.js +54 -0
- package/lib/drivers/copilot/config.js.map +1 -0
- package/lib/drivers/copilot/index.d.ts +103 -0
- package/lib/drivers/copilot/index.d.ts.map +1 -0
- package/lib/drivers/copilot/index.js +256 -0
- package/lib/drivers/copilot/index.js.map +1 -0
- package/lib/drivers/cursor/config.d.ts +6 -1
- package/lib/drivers/cursor/config.d.ts.map +1 -1
- package/lib/drivers/cursor/config.js +2 -3
- package/lib/drivers/cursor/config.js.map +1 -1
- package/lib/drivers/cursor/index.d.ts +7 -2
- package/lib/drivers/cursor/index.d.ts.map +1 -1
- package/lib/drivers/cursor/index.js +9 -6
- package/lib/drivers/cursor/index.js.map +1 -1
- package/lib/drivers/cursor/interaction.d.ts +1 -2
- package/lib/drivers/cursor/interaction.d.ts.map +1 -1
- package/lib/drivers/cursor/interaction.js +3 -14
- package/lib/drivers/cursor/interaction.js.map +1 -1
- package/lib/drivers/{cursor/events.d.ts → shared/acp-events.d.ts} +5 -3
- package/lib/drivers/shared/acp-events.d.ts.map +1 -0
- package/lib/drivers/{cursor/events.js → shared/acp-events.js} +20 -10
- package/lib/drivers/shared/acp-events.js.map +1 -0
- package/lib/drivers/shared/acp-interaction.d.ts +7 -0
- package/lib/drivers/shared/acp-interaction.d.ts.map +1 -0
- package/lib/drivers/shared/acp-interaction.js +20 -0
- package/lib/drivers/shared/acp-interaction.js.map +1 -0
- package/lib/drivers/{cursor → shared}/acp-transport.d.ts +2 -0
- package/lib/drivers/shared/acp-transport.d.ts.map +1 -0
- package/lib/drivers/{cursor → shared}/acp-transport.js +10 -10
- package/lib/drivers/shared/acp-transport.js.map +1 -0
- package/lib/index.d.ts +7 -2
- package/lib/index.d.ts.map +1 -1
- package/lib/index.js +7 -2
- package/lib/index.js.map +1 -1
- package/lib/optimization/reflective.d.ts +2 -0
- package/lib/optimization/reflective.d.ts.map +1 -1
- package/lib/orchestrator/comparison-identity.d.ts.map +1 -1
- package/lib/orchestrator/comparison-identity.js +1 -0
- package/lib/orchestrator/comparison-identity.js.map +1 -1
- package/lib/orchestrator/config.d.ts +7 -0
- package/lib/orchestrator/config.d.ts.map +1 -1
- package/lib/orchestrator/config.js +8 -1
- package/lib/orchestrator/config.js.map +1 -1
- package/lib/orchestrator/doctor.d.ts.map +1 -1
- package/lib/orchestrator/doctor.js +72 -2
- package/lib/orchestrator/doctor.js.map +1 -1
- package/lib/orchestrator/experiment-context.d.ts +1 -1
- package/lib/orchestrator/harness-catalog-drift.d.ts +22 -0
- package/lib/orchestrator/harness-catalog-drift.d.ts.map +1 -0
- package/lib/orchestrator/harness-catalog-drift.js +90 -0
- package/lib/orchestrator/harness-catalog-drift.js.map +1 -0
- package/lib/orchestrator/orchestrator.d.ts +9 -1
- package/lib/orchestrator/orchestrator.d.ts.map +1 -1
- package/lib/orchestrator/orchestrator.js +14 -22
- package/lib/orchestrator/orchestrator.js.map +1 -1
- package/lib/orchestrator/run-dir.d.ts +2 -0
- package/lib/orchestrator/run-dir.d.ts.map +1 -1
- package/lib/orchestrator/run-dir.js +1 -1
- package/lib/orchestrator/run-dir.js.map +1 -1
- package/lib/orchestrator/run-execution.d.ts +1 -0
- package/lib/orchestrator/run-execution.d.ts.map +1 -1
- package/lib/orchestrator/run-execution.js +2 -0
- package/lib/orchestrator/run-execution.js.map +1 -1
- package/lib/orchestrator/suite-compare.d.ts +3 -0
- package/lib/orchestrator/suite-compare.d.ts.map +1 -1
- package/lib/orchestrator/suite-compare.js +27 -17
- package/lib/orchestrator/suite-compare.js.map +1 -1
- package/lib/orchestrator/suite-runs.d.ts +3 -0
- package/lib/orchestrator/suite-runs.d.ts.map +1 -1
- package/lib/orchestrator/suite-runs.js +2 -0
- package/lib/orchestrator/suite-runs.js.map +1 -1
- package/lib/orchestrator/suite-summary.d.ts +9 -0
- package/lib/orchestrator/suite-summary.d.ts.map +1 -1
- package/lib/orchestrator/suite-summary.js +14 -4
- package/lib/orchestrator/suite-summary.js.map +1 -1
- package/package.json +1 -1
- package/skills/lux-answerer/SKILL.md +1 -1
- package/src/authoring.ts +10 -3
- package/src/cli/harness-plan.ts +117 -0
- package/src/cli/init-templates.ts +12 -5
- package/src/cli/init.ts +12 -5
- package/src/cli/program.ts +50 -92
- package/src/cli/run-options.ts +123 -0
- package/src/core/case.ts +3 -0
- package/src/core/driver.ts +26 -0
- package/src/core/harness-catalog.ts +100 -0
- package/src/core/harness-catalogs/antigravity.ts +24 -0
- package/src/core/harness-catalogs/claude-code.ts +25 -0
- package/src/core/harness-catalogs/codex.ts +27 -0
- package/src/core/harness-catalogs/copilot.ts +46 -0
- package/src/core/harness-catalogs/cursor.ts +13 -0
- package/src/core/harness-catalogs/index.ts +22 -0
- package/src/core/harness-matrix.ts +298 -0
- package/src/core/index.ts +3 -0
- package/src/core/model-cli.ts +38 -21
- package/src/core/platform-process.ts +4 -4
- package/src/core/run.ts +2 -0
- package/src/drivers/antigravity/README.md +46 -0
- package/src/drivers/antigravity/config.ts +44 -0
- package/src/drivers/antigravity/events.ts +235 -0
- package/src/drivers/antigravity/index.ts +109 -0
- package/src/drivers/claude-code/index.ts +18 -7
- package/src/drivers/codex/config.ts +23 -5
- package/src/drivers/codex/index.ts +15 -2
- package/src/drivers/copilot/README.md +52 -0
- package/src/drivers/copilot/config.ts +58 -0
- package/src/drivers/copilot/index.ts +292 -0
- package/src/drivers/cursor/README.md +9 -6
- package/src/drivers/cursor/config.ts +2 -3
- package/src/drivers/cursor/index.ts +10 -6
- package/src/drivers/cursor/interaction.ts +3 -19
- package/src/drivers/{cursor/events.ts → shared/acp-events.ts} +20 -9
- package/src/drivers/shared/acp-interaction.ts +27 -0
- package/src/drivers/{cursor → shared}/acp-transport.ts +15 -10
- package/src/index.ts +44 -3
- package/src/orchestrator/comparison-identity.ts +1 -0
- package/src/orchestrator/config.ts +8 -0
- package/src/orchestrator/doctor.ts +125 -3
- package/src/orchestrator/harness-catalog-drift.ts +132 -0
- package/src/orchestrator/orchestrator.ts +31 -26
- package/src/orchestrator/run-dir.ts +8 -1
- package/src/orchestrator/run-execution.ts +3 -0
- package/src/orchestrator/suite-compare.ts +29 -19
- package/src/orchestrator/suite-runs.ts +3 -1
- package/src/orchestrator/suite-summary.ts +19 -4
- package/lib/drivers/cursor/acp-transport.d.ts.map +0 -1
- package/lib/drivers/cursor/acp-transport.js.map +0 -1
- package/lib/drivers/cursor/events.d.ts.map +0 -1
- package/lib/drivers/cursor/events.js.map +0 -1
- package/skills/lux/SKILL.md +0 -89
- package/skills/lux/references/cli.md +0 -128
- package/skills/lux/references/eval-authoring.md +0 -266
package/src/cli/init.ts
CHANGED
|
@@ -13,19 +13,26 @@ import {
|
|
|
13
13
|
STARTER_PROMPT_BODY,
|
|
14
14
|
} from "./init-templates.js";
|
|
15
15
|
|
|
16
|
+
const EXPLICIT_MODEL_DRIVERS = new Set(["cursor", "copilot", "antigravity"]);
|
|
17
|
+
|
|
16
18
|
const InitOptionsSchema = z
|
|
17
19
|
.object({
|
|
18
20
|
cwd: z.string().min(1),
|
|
19
21
|
force: z.boolean().default(false),
|
|
20
22
|
experiment: z.boolean().default(false),
|
|
21
|
-
driver: z
|
|
23
|
+
driver: z
|
|
24
|
+
.enum(["subprocess", "claude-code", "codex", "cursor", "copilot", "antigravity"])
|
|
25
|
+
.optional(),
|
|
22
26
|
model: z.string().min(1).optional(),
|
|
23
27
|
})
|
|
24
28
|
.strict()
|
|
25
|
-
.refine(
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
+
.refine(
|
|
30
|
+
(options) => !EXPLICIT_MODEL_DRIVERS.has(options.driver ?? "") || options.model !== undefined,
|
|
31
|
+
{
|
|
32
|
+
path: ["model"],
|
|
33
|
+
message: "Cursor, Copilot, and Antigravity require an explicit --model",
|
|
34
|
+
},
|
|
35
|
+
)
|
|
29
36
|
.refine((options) => !(options.experiment && options.driver === "subprocess"), {
|
|
30
37
|
path: ["driver"],
|
|
31
38
|
message: "The deterministic starter is for runs; choose a coding agent for experiments",
|
package/src/cli/program.ts
CHANGED
|
@@ -2,14 +2,16 @@ import { randomUUID } from "node:crypto";
|
|
|
2
2
|
import { existsSync } from "node:fs";
|
|
3
3
|
import { lstat, mkdir, realpath, rename, rm, stat, writeFile } from "node:fs/promises";
|
|
4
4
|
import { createRequire } from "node:module";
|
|
5
|
-
import { basename, dirname,
|
|
5
|
+
import { basename, dirname, join, relative, resolve, sep } from "node:path";
|
|
6
6
|
import { fileURLToPath } from "node:url";
|
|
7
7
|
import { Cli, Errors, z } from "incur";
|
|
8
8
|
import { createRubricAssertion } from "../assertions/rubric/index.js";
|
|
9
9
|
import {
|
|
10
10
|
type Candidate,
|
|
11
11
|
GraderAnnotationSchema,
|
|
12
|
+
type HarnessFocus,
|
|
12
13
|
matchesAnyGlob,
|
|
14
|
+
ReasoningEffortSchema,
|
|
13
15
|
valueHash,
|
|
14
16
|
} from "../core/index.js";
|
|
15
17
|
import {
|
|
@@ -17,7 +19,6 @@ import {
|
|
|
17
19
|
buildRegistryFromConfig,
|
|
18
20
|
compareRuns,
|
|
19
21
|
compareSuiteRuns,
|
|
20
|
-
type DiscoveredCase,
|
|
21
22
|
DoctorReportSchema,
|
|
22
23
|
discoverCases,
|
|
23
24
|
ExperimentCampaignPlanSchema,
|
|
@@ -46,8 +47,6 @@ import {
|
|
|
46
47
|
resolveExperimentCampaignPath,
|
|
47
48
|
runExperimentCampaign,
|
|
48
49
|
type SuiteCaseResult,
|
|
49
|
-
type SuiteItem,
|
|
50
|
-
selectCases,
|
|
51
50
|
upsertGraderAnnotation,
|
|
52
51
|
verifyExperimentDecisionIntegrity,
|
|
53
52
|
verifyExperimentRunIntegrity,
|
|
@@ -63,10 +62,15 @@ import { initProject } from "./init.js";
|
|
|
63
62
|
import { CompareOutputSchema, ViewOutputSchema } from "./output-schemas.js";
|
|
64
63
|
import { ConsoleReporter } from "./render/reporter.js";
|
|
65
64
|
import {
|
|
65
|
+
expandLoop,
|
|
66
66
|
parseAnnotationAssertion,
|
|
67
|
+
partitionFilters,
|
|
68
|
+
planSuiteItems,
|
|
67
69
|
type RunOptions,
|
|
68
70
|
resolveAnswererOverride,
|
|
69
71
|
resolveConcurrency,
|
|
72
|
+
selectSuite,
|
|
73
|
+
suiteTitle,
|
|
70
74
|
} from "./run-options.js";
|
|
71
75
|
import { type ViewOptions, viewCommand, webViewCommand } from "./view.js";
|
|
72
76
|
|
|
@@ -92,7 +96,14 @@ const initOutput = z.object({
|
|
|
92
96
|
const runOutput = z.union([
|
|
93
97
|
z.object({
|
|
94
98
|
caseCount: z.number().int(),
|
|
95
|
-
cases: z.array(
|
|
99
|
+
cases: z.array(
|
|
100
|
+
z.object({
|
|
101
|
+
path: z.string(),
|
|
102
|
+
id: z.string(),
|
|
103
|
+
caseId: z.string().optional(),
|
|
104
|
+
permutationId: z.string().optional(),
|
|
105
|
+
}),
|
|
106
|
+
),
|
|
96
107
|
}),
|
|
97
108
|
z.object({
|
|
98
109
|
caseCount: z.number().int(),
|
|
@@ -156,7 +167,7 @@ export const buildCli = () =>
|
|
|
156
167
|
force: z.boolean().optional().describe("Overwrite existing files"),
|
|
157
168
|
experiment: z.boolean().optional().describe("Also scaffold an optimization campaign"),
|
|
158
169
|
driver: z
|
|
159
|
-
.enum(["subprocess", "claude-code", "codex", "cursor"])
|
|
170
|
+
.enum(["subprocess", "claude-code", "codex", "cursor", "copilot", "antigravity"])
|
|
160
171
|
.optional()
|
|
161
172
|
.describe("Starter driver (default: deterministic subprocess)"),
|
|
162
173
|
model: nonEmptyString.optional().describe("Explicit subject model; required for Cursor"),
|
|
@@ -203,6 +214,11 @@ export const buildCli = () =>
|
|
|
203
214
|
skipJudge: z.boolean().optional().describe("Skip rubric LLM judge assertions"),
|
|
204
215
|
runsRoot: nonEmptyString.optional().describe("Run directory root"),
|
|
205
216
|
profile: nonEmptyString.optional().describe("Named driver and answerer profile"),
|
|
217
|
+
driver: nonEmptyString
|
|
218
|
+
.optional()
|
|
219
|
+
.describe("Focus on one harness driver, replacing the matrix"),
|
|
220
|
+
model: nonEmptyString.optional().describe("Focus on one subject model"),
|
|
221
|
+
effort: ReasoningEffortSchema.optional().describe("Focus on one reasoning effort"),
|
|
206
222
|
subjectRoot: nonEmptyString.optional().describe("Root bound into subjectPath values"),
|
|
207
223
|
verbose: z.boolean().optional().describe("Include phase timings and run directories"),
|
|
208
224
|
}),
|
|
@@ -217,6 +233,11 @@ export const buildCli = () =>
|
|
|
217
233
|
options: { tag: [] },
|
|
218
234
|
description: "Run matching cases",
|
|
219
235
|
},
|
|
236
|
+
{
|
|
237
|
+
args: { filters: ["smoke"] },
|
|
238
|
+
options: { tag: [], driver: "codex", model: "gpt-5.6-luna", effort: "low" },
|
|
239
|
+
description: "Focus on one harness permutation",
|
|
240
|
+
},
|
|
220
241
|
],
|
|
221
242
|
run: async (c) => {
|
|
222
243
|
const result = await runCommand(c.args.filters, c.options as RunOptions, !c.agent);
|
|
@@ -620,8 +641,6 @@ const compareCommand = async (
|
|
|
620
641
|
return comparison;
|
|
621
642
|
};
|
|
622
643
|
|
|
623
|
-
const caseIdOf = (casePath: string): string => basename(casePath, extname(casePath));
|
|
624
|
-
|
|
625
644
|
const SIGINT_EXIT_CODE = 130;
|
|
626
645
|
|
|
627
646
|
const physicalDestinationPath = async (path: string): Promise<string> => {
|
|
@@ -694,86 +713,6 @@ const isFailure = (outcome: { exitReason: string; casePassed: boolean | null }):
|
|
|
694
713
|
|
|
695
714
|
const firstLine = (message: string): string => (message.split("\n")[0] ?? message).trim();
|
|
696
715
|
|
|
697
|
-
const suiteTitle = (
|
|
698
|
-
patterns: string[],
|
|
699
|
-
tags: string[],
|
|
700
|
-
explicit: SuiteItem[],
|
|
701
|
-
loop: number,
|
|
702
|
-
): string => {
|
|
703
|
-
const parts = [...patterns, ...explicit.map((e) => e.id), ...tags.map((t) => `#${t}`)];
|
|
704
|
-
const filter = parts.length > 0 ? ` ${parts.join(" ")}` : "";
|
|
705
|
-
const repeat = loop > 1 ? ` ×${loop}` : "";
|
|
706
|
-
return `run${filter}${repeat}`;
|
|
707
|
-
};
|
|
708
|
-
|
|
709
|
-
type FilterPartition = { explicit: SuiteItem[]; patterns: string[] };
|
|
710
|
-
|
|
711
|
-
// A filter that names an existing `.ts` file runs that file directly — it may
|
|
712
|
-
// sit outside the discovery root. Everything else is a substring pattern — but
|
|
713
|
-
// a `.ts` filter that resolves to nothing is almost always a typo'd path, so
|
|
714
|
-
// the silent downgrade to a pattern gets called out.
|
|
715
|
-
const partitionFilters = (filters: string[], cwd: string): FilterPartition => {
|
|
716
|
-
const explicit: SuiteItem[] = [];
|
|
717
|
-
const patterns: string[] = [];
|
|
718
|
-
for (const f of filters) {
|
|
719
|
-
const abs = resolve(cwd, f);
|
|
720
|
-
if (f.endsWith(".ts") && existsSync(abs)) {
|
|
721
|
-
explicit.push({ path: abs, id: caseIdOf(f) });
|
|
722
|
-
} else {
|
|
723
|
-
if (f.endsWith(".ts")) {
|
|
724
|
-
process.stderr.write(`lux: ${abs} does not exist; treating "${f}" as a pattern\n`);
|
|
725
|
-
}
|
|
726
|
-
patterns.push(f);
|
|
727
|
-
}
|
|
728
|
-
}
|
|
729
|
-
return { explicit, patterns };
|
|
730
|
-
};
|
|
731
|
-
|
|
732
|
-
// The selected set — substring/tag matches plus explicitly named files — is
|
|
733
|
-
// the same whether it is run, `--list`ed, or `--list-tags`ed. No filters at
|
|
734
|
-
// all means "everything discovered"; any filter narrows.
|
|
735
|
-
const selectSuite = (
|
|
736
|
-
cases: DiscoveredCase[],
|
|
737
|
-
{ explicit, patterns }: FilterPartition,
|
|
738
|
-
tags: string[],
|
|
739
|
-
): DiscoveredCase[] => {
|
|
740
|
-
const hasFilter = explicit.length > 0 || patterns.length > 0 || tags.length > 0;
|
|
741
|
-
const byPath = new Map<string, DiscoveredCase>();
|
|
742
|
-
if (!hasFilter) {
|
|
743
|
-
for (const c of cases) byPath.set(c.path, c);
|
|
744
|
-
} else {
|
|
745
|
-
if (patterns.length > 0 || tags.length > 0) {
|
|
746
|
-
for (const c of selectCases(cases, { patterns, tags })) byPath.set(c.path, c);
|
|
747
|
-
}
|
|
748
|
-
for (const e of explicit) {
|
|
749
|
-
const discovered = cases.find((c) => c.path === e.path);
|
|
750
|
-
// Discovery knows the case's declared id; the filename-derived one is
|
|
751
|
-
// only a fallback for files discovery never saw.
|
|
752
|
-
byPath.set(e.path, {
|
|
753
|
-
path: e.path,
|
|
754
|
-
id: discovered?.id ?? e.id,
|
|
755
|
-
relPath: discovered?.relPath ?? e.id,
|
|
756
|
-
tags: discovered?.tags ?? [],
|
|
757
|
-
});
|
|
758
|
-
}
|
|
759
|
-
}
|
|
760
|
-
return [...byPath.values()].sort((a, b) => a.id.localeCompare(b.id));
|
|
761
|
-
};
|
|
762
|
-
|
|
763
|
-
// `--loop` repeats each eval before moving to the next; each repeat gets a
|
|
764
|
-
// `#n` display suffix so the reporter keeps them apart, while `caseId` keeps
|
|
765
|
-
// every repeat filed under one run history.
|
|
766
|
-
const expandLoop = (base: SuiteItem[], loop: number): SuiteItem[] =>
|
|
767
|
-
loop > 1
|
|
768
|
-
? base.flatMap((b) =>
|
|
769
|
-
Array.from({ length: loop }, (_, i) => ({
|
|
770
|
-
path: b.path,
|
|
771
|
-
id: `${b.id} #${i + 1}`,
|
|
772
|
-
caseId: b.caseId ?? b.id,
|
|
773
|
-
})),
|
|
774
|
-
)
|
|
775
|
-
: base;
|
|
776
|
-
|
|
777
716
|
const experimentCommand = async (
|
|
778
717
|
mode: "experiment" | "optimize",
|
|
779
718
|
campaignPath: string,
|
|
@@ -824,6 +763,7 @@ const experimentCommand = async (
|
|
|
824
763
|
return { experimentDir: result.experimentDir, ...result.manifest };
|
|
825
764
|
};
|
|
826
765
|
|
|
766
|
+
/** A profile pins one driver, so callers pass the project matrix only without one. */
|
|
827
767
|
const runCommand = async (filters: string[], options: RunOptions, human = false) => {
|
|
828
768
|
if (options.interactive && options.claudeAnswerer) {
|
|
829
769
|
throw new Errors.ParseError({
|
|
@@ -923,7 +863,17 @@ const runCommand = async (filters: string[], options: RunOptions, human = false)
|
|
|
923
863
|
return result;
|
|
924
864
|
}
|
|
925
865
|
|
|
926
|
-
const
|
|
866
|
+
const focus: HarnessFocus = {
|
|
867
|
+
driver: options.driver,
|
|
868
|
+
model: options.model,
|
|
869
|
+
effort: options.effort,
|
|
870
|
+
};
|
|
871
|
+
const base = await planSuiteItems(selected, {
|
|
872
|
+
matrix: profile ? undefined : config.matrix,
|
|
873
|
+
defaultDriver,
|
|
874
|
+
registry,
|
|
875
|
+
focus,
|
|
876
|
+
});
|
|
927
877
|
|
|
928
878
|
// Collect-only: answer "what would run?" without spawning a single agent.
|
|
929
879
|
if (options.list) {
|
|
@@ -931,7 +881,15 @@ const runCommand = async (filters: string[], options: RunOptions, human = false)
|
|
|
931
881
|
process.stdout.write(
|
|
932
882
|
`${base.length} cases\n${base.map(({ id, path }) => ` ${id} ${path}`).join("\n")}\n`,
|
|
933
883
|
);
|
|
934
|
-
return {
|
|
884
|
+
return {
|
|
885
|
+
caseCount: base.length,
|
|
886
|
+
cases: base.map(({ path, id, caseId, permutationId }) => ({
|
|
887
|
+
path,
|
|
888
|
+
id,
|
|
889
|
+
...(caseId !== undefined ? { caseId } : {}),
|
|
890
|
+
...(permutationId !== undefined ? { permutationId } : {}),
|
|
891
|
+
})),
|
|
892
|
+
};
|
|
935
893
|
}
|
|
936
894
|
|
|
937
895
|
const loop = options.loop ?? 1;
|
|
@@ -943,7 +901,7 @@ const runCommand = async (filters: string[], options: RunOptions, human = false)
|
|
|
943
901
|
const reporter = human ? new ConsoleReporter({ verbose: options.verbose ?? false }) : undefined;
|
|
944
902
|
using controller = abortOnInterrupt();
|
|
945
903
|
reporter?.begin({
|
|
946
|
-
title: suiteTitle(patterns, tags, explicit, loop),
|
|
904
|
+
title: suiteTitle(patterns, tags, explicit, loop, focus),
|
|
947
905
|
caseIds: items.map((i) => i.id),
|
|
948
906
|
runsRoot,
|
|
949
907
|
loop,
|
|
@@ -991,4 +949,4 @@ const runCommand = async (filters: string[], options: RunOptions, human = false)
|
|
|
991
949
|
};
|
|
992
950
|
|
|
993
951
|
// Internal exports for testing
|
|
994
|
-
export {
|
|
952
|
+
export { isFailure, resolveConcurrency };
|
package/src/cli/run-options.ts
CHANGED
|
@@ -1,4 +1,9 @@
|
|
|
1
|
+
import { existsSync } from "node:fs";
|
|
2
|
+
import { basename, extname, resolve } from "node:path";
|
|
1
3
|
import { Errors } from "incur";
|
|
4
|
+
import type { HarnessFocus, ReasoningEffort } from "../core/index.js";
|
|
5
|
+
import { type DiscoveredCase, type SuiteItem, selectCases } from "../orchestrator/index.js";
|
|
6
|
+
import { planHarnessItems } from "./harness-plan.js";
|
|
2
7
|
|
|
3
8
|
export type RunOptions = {
|
|
4
9
|
tag?: string[];
|
|
@@ -11,6 +16,9 @@ export type RunOptions = {
|
|
|
11
16
|
skipJudge?: boolean;
|
|
12
17
|
runsRoot?: string;
|
|
13
18
|
profile?: string;
|
|
19
|
+
driver?: string;
|
|
20
|
+
model?: string;
|
|
21
|
+
effort?: ReasoningEffort;
|
|
14
22
|
subjectRoot?: string;
|
|
15
23
|
verbose?: boolean;
|
|
16
24
|
};
|
|
@@ -64,3 +72,118 @@ export const resolveAnswererOverride = (options: RunOptions): string | undefined
|
|
|
64
72
|
if (options.claudeAnswerer) return "claude-code";
|
|
65
73
|
return undefined;
|
|
66
74
|
};
|
|
75
|
+
|
|
76
|
+
const caseIdOf = (casePath: string): string => basename(casePath, extname(casePath));
|
|
77
|
+
|
|
78
|
+
export const suiteTitle = (
|
|
79
|
+
patterns: string[],
|
|
80
|
+
tags: string[],
|
|
81
|
+
explicit: SuiteItem[],
|
|
82
|
+
loop: number,
|
|
83
|
+
focus: HarnessFocus,
|
|
84
|
+
): string => {
|
|
85
|
+
const parts = [...patterns, ...explicit.map((e) => e.id), ...tags.map((t) => `#${t}`)];
|
|
86
|
+
const filter = parts.length > 0 ? ` ${parts.join(" ")}` : "";
|
|
87
|
+
const focusParts = [focus.driver, focus.model, focus.effort].filter(
|
|
88
|
+
(part): part is string => part !== undefined,
|
|
89
|
+
);
|
|
90
|
+
const focused = focusParts.length > 0 ? ` @${focusParts.join(":")}` : "";
|
|
91
|
+
const repeat = loop > 1 ? ` ×${loop}` : "";
|
|
92
|
+
return `run${filter}${focused}${repeat}`;
|
|
93
|
+
};
|
|
94
|
+
|
|
95
|
+
type FilterPartition = { explicit: SuiteItem[]; patterns: string[] };
|
|
96
|
+
|
|
97
|
+
// A filter that names an existing `.ts` file runs that file directly — it may
|
|
98
|
+
// sit outside the discovery root. Everything else is a substring pattern — but
|
|
99
|
+
// a `.ts` filter that resolves to nothing is almost always a typo'd path, so
|
|
100
|
+
// the silent downgrade to a pattern gets called out.
|
|
101
|
+
export const partitionFilters = (filters: string[], cwd: string): FilterPartition => {
|
|
102
|
+
const explicit: SuiteItem[] = [];
|
|
103
|
+
const patterns: string[] = [];
|
|
104
|
+
for (const f of filters) {
|
|
105
|
+
const abs = resolve(cwd, f);
|
|
106
|
+
if (f.endsWith(".ts") && existsSync(abs)) {
|
|
107
|
+
explicit.push({ path: abs, id: caseIdOf(f) });
|
|
108
|
+
} else {
|
|
109
|
+
if (f.endsWith(".ts")) {
|
|
110
|
+
process.stderr.write(`lux: ${abs} does not exist; treating "${f}" as a pattern\n`);
|
|
111
|
+
}
|
|
112
|
+
patterns.push(f);
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
return { explicit, patterns };
|
|
116
|
+
};
|
|
117
|
+
|
|
118
|
+
// The selected set — substring/tag matches plus explicitly named files — is
|
|
119
|
+
// the same whether it is run, `--list`ed, or `--list-tags`ed. No filters at
|
|
120
|
+
// all means "everything discovered"; any filter narrows.
|
|
121
|
+
export const selectSuite = (
|
|
122
|
+
cases: DiscoveredCase[],
|
|
123
|
+
{ explicit, patterns }: FilterPartition,
|
|
124
|
+
tags: string[],
|
|
125
|
+
): DiscoveredCase[] => {
|
|
126
|
+
const hasFilter = explicit.length > 0 || patterns.length > 0 || tags.length > 0;
|
|
127
|
+
const byPath = new Map<string, DiscoveredCase>();
|
|
128
|
+
if (!hasFilter) {
|
|
129
|
+
for (const c of cases) byPath.set(c.path, c);
|
|
130
|
+
} else {
|
|
131
|
+
if (patterns.length > 0 || tags.length > 0) {
|
|
132
|
+
for (const c of selectCases(cases, { patterns, tags })) byPath.set(c.path, c);
|
|
133
|
+
}
|
|
134
|
+
for (const e of explicit) {
|
|
135
|
+
const discovered = cases.find((c) => c.path === e.path);
|
|
136
|
+
// Discovery knows the case's declared id; the filename-derived one is
|
|
137
|
+
// only a fallback for files discovery never saw.
|
|
138
|
+
byPath.set(e.path, {
|
|
139
|
+
path: e.path,
|
|
140
|
+
id: discovered?.id ?? e.id,
|
|
141
|
+
relPath: discovered?.relPath ?? e.id,
|
|
142
|
+
tags: discovered?.tags ?? [],
|
|
143
|
+
});
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
return [...byPath.values()].sort((a, b) => a.id.localeCompare(b.id));
|
|
147
|
+
};
|
|
148
|
+
|
|
149
|
+
// `--loop` repeats each eval before moving to the next; each repeat gets a
|
|
150
|
+
// `#n` display suffix so the reporter keeps them apart, while `caseId` keeps
|
|
151
|
+
// every repeat filed under one run history.
|
|
152
|
+
export const expandLoop = (base: SuiteItem[], loop: number): SuiteItem[] =>
|
|
153
|
+
loop > 1
|
|
154
|
+
? base.flatMap((b) =>
|
|
155
|
+
Array.from({ length: loop }, (_, i) => ({
|
|
156
|
+
...b,
|
|
157
|
+
id: `${b.id} #${i + 1}`,
|
|
158
|
+
caseId: b.caseId ?? b.id,
|
|
159
|
+
})),
|
|
160
|
+
)
|
|
161
|
+
: base;
|
|
162
|
+
|
|
163
|
+
export const planSuiteItems = async (
|
|
164
|
+
selected: readonly { path: string; id: string }[],
|
|
165
|
+
options: Parameters<typeof planHarnessItems>[1],
|
|
166
|
+
): Promise<SuiteItem[]> => {
|
|
167
|
+
const plan = await planHarnessItems(selected, options);
|
|
168
|
+
for (const skipped of plan.unavailable) {
|
|
169
|
+
process.stderr.write(
|
|
170
|
+
`lux: skipping ${skipped.id}: harness command not found: ${skipped.commands.join(", ")}\n`,
|
|
171
|
+
);
|
|
172
|
+
}
|
|
173
|
+
if (plan.excluded.length > 0) {
|
|
174
|
+
process.stderr.write(
|
|
175
|
+
`lux: the harness focus excludes ${plan.excluded.length} pinned case${plan.excluded.length === 1 ? "" : "s"}: ${plan.excluded.join(", ")}\n`,
|
|
176
|
+
);
|
|
177
|
+
}
|
|
178
|
+
if (plan.items.length === 0) {
|
|
179
|
+
throw new Errors.IncurError({
|
|
180
|
+
code: plan.unavailable.length > 0 ? "NO_RUNNABLE_HARNESSES" : "NO_CASES_MATCHED",
|
|
181
|
+
message:
|
|
182
|
+
plan.unavailable.length > 0
|
|
183
|
+
? "No cases remain: all selected matrix harnesses are unavailable."
|
|
184
|
+
: "No cases remain after applying the harness focus.",
|
|
185
|
+
exitCode: 1,
|
|
186
|
+
});
|
|
187
|
+
}
|
|
188
|
+
return plan.items;
|
|
189
|
+
};
|
package/src/core/case.ts
CHANGED
|
@@ -3,6 +3,7 @@ import type { AssertionPlugin } from "./assertion.js";
|
|
|
3
3
|
import { collectEvalCaseExtensions, setEvalCaseExtensions } from "./case-authoring-state.js";
|
|
4
4
|
import { DriverCapabilitySchema } from "./driver.js";
|
|
5
5
|
import { EvaluationMetadataSchema } from "./evaluation-clusters.js";
|
|
6
|
+
import { HarnessMatrixSchema } from "./harness-matrix.js";
|
|
6
7
|
import type { LifecycleHooksPlugin } from "./lifecycle-hooks.js";
|
|
7
8
|
import { formatZodIssues } from "./zod.js";
|
|
8
9
|
|
|
@@ -46,6 +47,8 @@ const EvalCaseObjectSchema = z.object({
|
|
|
46
47
|
prompt: z.string().min(1, "prompt is required"),
|
|
47
48
|
persona: z.string().min(1).optional(),
|
|
48
49
|
driver: DriverRefSchema.optional(),
|
|
50
|
+
/** Harness permutations this case runs through instead of the project matrix. */
|
|
51
|
+
matrix: HarnessMatrixSchema.optional(),
|
|
49
52
|
answerer: AnswererRefSchema.optional(),
|
|
50
53
|
fixtures: FixturesRefSchema.optional(),
|
|
51
54
|
requires: z.array(DriverCapabilitySchema).optional(),
|
package/src/core/driver.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
2
|
import type { EventHandler } from "./emitter.js";
|
|
3
3
|
import type { Answer, Done, ErrorEvent, Interrupt, Progress, ReadyState } from "./events.js";
|
|
4
|
+
import type { ReasoningEffort } from "./harness-catalog.js";
|
|
4
5
|
import type { FixtureValueBinding } from "./lifecycle-hooks.js";
|
|
5
6
|
|
|
6
7
|
export const ArtifactSchema = z.object({
|
|
@@ -97,6 +98,12 @@ export type DriverExperiment<Cfg> = {
|
|
|
97
98
|
identity(config: Cfg, context: { rootDir: string }): unknown | Promise<unknown>;
|
|
98
99
|
};
|
|
99
100
|
|
|
101
|
+
/** Model and reasoning effort chosen by a harness matrix permutation. */
|
|
102
|
+
export type HarnessSelection = {
|
|
103
|
+
model?: string | undefined;
|
|
104
|
+
effort?: ReasoningEffort | undefined;
|
|
105
|
+
};
|
|
106
|
+
|
|
100
107
|
export type DriverPlugin<Cfg = unknown> = {
|
|
101
108
|
name: string;
|
|
102
109
|
configSchema?: z.ZodTypeAny;
|
|
@@ -104,8 +111,27 @@ export type DriverPlugin<Cfg = unknown> = {
|
|
|
104
111
|
runtime?: (config: Cfg) => DriverRuntime;
|
|
105
112
|
/** Ordinary runs remain supported when controlled experiments are not declared. */
|
|
106
113
|
experiment?: DriverExperiment<Cfg>;
|
|
114
|
+
/**
|
|
115
|
+
* Apply a matrix selection to raw (unvalidated) configuration. Drivers
|
|
116
|
+
* without a model concept leave this undefined and reject selection.
|
|
117
|
+
*/
|
|
118
|
+
selectHarness?: (config: unknown, selection: HarnessSelection) => unknown;
|
|
107
119
|
};
|
|
108
120
|
|
|
121
|
+
/** Selection for drivers whose configuration uses `model` and `reasoningEffort`. */
|
|
122
|
+
export const selectModelAndReasoningEffort = (
|
|
123
|
+
config: unknown,
|
|
124
|
+
selection: HarnessSelection,
|
|
125
|
+
): unknown => ({
|
|
126
|
+
...(config && typeof config === "object" && !Array.isArray(config)
|
|
127
|
+
? (config as Record<string, unknown>)
|
|
128
|
+
: {}),
|
|
129
|
+
...(selection.model !== undefined ? { model: selection.model } : {}),
|
|
130
|
+
...(selection.effort !== undefined || selection.model !== undefined
|
|
131
|
+
? { reasoningEffort: selection.effort }
|
|
132
|
+
: {}),
|
|
133
|
+
});
|
|
134
|
+
|
|
109
135
|
export const defineDriver = <S extends z.ZodType>(
|
|
110
136
|
definition: Omit<DriverPlugin<z.output<S>>, "configSchema"> & { configSchema: S },
|
|
111
137
|
): Omit<DriverPlugin<z.output<S>>, "configSchema"> & { configSchema: S } => definition;
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
|
|
3
|
+
export const HARNESS_EFFORTS = [
|
|
4
|
+
"none",
|
|
5
|
+
"minimal",
|
|
6
|
+
"low",
|
|
7
|
+
"medium",
|
|
8
|
+
"high",
|
|
9
|
+
"xhigh",
|
|
10
|
+
"max",
|
|
11
|
+
"ultra",
|
|
12
|
+
] as const;
|
|
13
|
+
export const ReasoningEffortSchema = z.enum(HARNESS_EFFORTS);
|
|
14
|
+
export type ReasoningEffort = z.infer<typeof ReasoningEffortSchema>;
|
|
15
|
+
|
|
16
|
+
export type CatalogModels = Readonly<Record<string, readonly ReasoningEffort[]>>;
|
|
17
|
+
|
|
18
|
+
export type HarnessCatalog<
|
|
19
|
+
D extends string = string,
|
|
20
|
+
E extends readonly ReasoningEffort[] = readonly ReasoningEffort[],
|
|
21
|
+
M extends CatalogModels | undefined = CatalogModels | undefined,
|
|
22
|
+
> = {
|
|
23
|
+
readonly driver: D;
|
|
24
|
+
readonly verified: string;
|
|
25
|
+
readonly sources: readonly string[];
|
|
26
|
+
readonly efforts: E;
|
|
27
|
+
readonly models: M;
|
|
28
|
+
};
|
|
29
|
+
|
|
30
|
+
export type ClosedHarnessCatalog = HarnessCatalog<
|
|
31
|
+
string,
|
|
32
|
+
readonly ReasoningEffort[],
|
|
33
|
+
CatalogModels
|
|
34
|
+
>;
|
|
35
|
+
|
|
36
|
+
export const defineHarnessCatalog = <
|
|
37
|
+
const D extends string,
|
|
38
|
+
const E extends readonly ReasoningEffort[],
|
|
39
|
+
const M extends CatalogModels | undefined,
|
|
40
|
+
>(
|
|
41
|
+
catalog: HarnessCatalog<D, E, M>,
|
|
42
|
+
): HarnessCatalog<D, E, M> => catalog;
|
|
43
|
+
|
|
44
|
+
export const isClosedCatalog = (catalog: HarnessCatalog): catalog is ClosedHarnessCatalog =>
|
|
45
|
+
catalog.models !== undefined;
|
|
46
|
+
|
|
47
|
+
export const catalogModelIds = (catalog: HarnessCatalog): string[] =>
|
|
48
|
+
Object.keys(catalog.models ?? {});
|
|
49
|
+
|
|
50
|
+
export const catalogEfforts = (
|
|
51
|
+
catalog: HarnessCatalog,
|
|
52
|
+
model: string,
|
|
53
|
+
): readonly ReasoningEffort[] | undefined => catalog.models?.[model];
|
|
54
|
+
|
|
55
|
+
export const defaultCatalogEffort = (
|
|
56
|
+
catalog: HarnessCatalog,
|
|
57
|
+
model: string,
|
|
58
|
+
): ReasoningEffort | undefined => {
|
|
59
|
+
if (!isClosedCatalog(catalog)) return undefined;
|
|
60
|
+
const accepted = catalog.models[model] ?? catalog.efforts;
|
|
61
|
+
return accepted.includes("low") ? "low" : accepted[0];
|
|
62
|
+
};
|
|
63
|
+
|
|
64
|
+
export const describeAcceptedEfforts = (
|
|
65
|
+
catalog: HarnessCatalog,
|
|
66
|
+
model: string,
|
|
67
|
+
accepted: readonly ReasoningEffort[],
|
|
68
|
+
): string =>
|
|
69
|
+
accepted.length === 0
|
|
70
|
+
? `model '${model}' has no reasoning effort in ${catalog.driver}`
|
|
71
|
+
: `model '${model}' accepts reasoning effort ${accepted.join(", ")} in ${catalog.driver}`;
|
|
72
|
+
|
|
73
|
+
export const effortIssue = (
|
|
74
|
+
catalog: HarnessCatalog,
|
|
75
|
+
model: string,
|
|
76
|
+
effort: ReasoningEffort | undefined,
|
|
77
|
+
): string | undefined => {
|
|
78
|
+
if (effort === undefined) return undefined;
|
|
79
|
+
const accepted = catalogEfforts(catalog, model);
|
|
80
|
+
if (accepted === undefined || accepted.includes(effort)) return undefined;
|
|
81
|
+
return describeAcceptedEfforts(catalog, model, accepted);
|
|
82
|
+
};
|
|
83
|
+
|
|
84
|
+
export type CatalogEffortConfig = {
|
|
85
|
+
model: string;
|
|
86
|
+
reasoningEffort?: ReasoningEffort | undefined;
|
|
87
|
+
};
|
|
88
|
+
|
|
89
|
+
export const applyCatalogEffort =
|
|
90
|
+
(catalog: HarnessCatalog) =>
|
|
91
|
+
<C extends CatalogEffortConfig>(config: C, ctx: z.RefinementCtx): C => {
|
|
92
|
+
const issue = effortIssue(catalog, config.model, config.reasoningEffort);
|
|
93
|
+
if (issue) {
|
|
94
|
+
ctx.addIssue({ code: "custom", path: ["reasoningEffort"], message: issue });
|
|
95
|
+
return config;
|
|
96
|
+
}
|
|
97
|
+
if (config.reasoningEffort !== undefined) return config;
|
|
98
|
+
const effort = defaultCatalogEffort(catalog, config.model);
|
|
99
|
+
return effort === undefined ? config : { ...config, reasoningEffort: effort };
|
|
100
|
+
};
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { defineHarnessCatalog } from "../harness-catalog.js";
|
|
2
|
+
|
|
3
|
+
const flash = ["low", "medium", "high"] as const;
|
|
4
|
+
|
|
5
|
+
export const antigravityCatalog = defineHarnessCatalog({
|
|
6
|
+
driver: "antigravity",
|
|
7
|
+
verified: "2026-09-18",
|
|
8
|
+
sources: [
|
|
9
|
+
"https://antigravity.google/docs/models/",
|
|
10
|
+
"https://antigravity.google/docs/cli/headless/",
|
|
11
|
+
"agy models (needs a login)",
|
|
12
|
+
],
|
|
13
|
+
efforts: flash,
|
|
14
|
+
models: {
|
|
15
|
+
"gemini-3.8-flash": flash,
|
|
16
|
+
"gemini-3.7-flash": flash,
|
|
17
|
+
"gemini-3.6-flash": flash,
|
|
18
|
+
"gemini-3.5-flash": flash,
|
|
19
|
+
"gemini-3.1-pro": ["low", "high"],
|
|
20
|
+
"claude-sonnet-4-6": [],
|
|
21
|
+
"claude-opus-4-6-thinking": [],
|
|
22
|
+
"gpt-oss-120b-medium": [],
|
|
23
|
+
},
|
|
24
|
+
});
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { defineHarnessCatalog } from "../harness-catalog.js";
|
|
2
|
+
|
|
3
|
+
const full = ["low", "medium", "high", "xhigh", "max"] as const;
|
|
4
|
+
const withoutXhigh = ["low", "medium", "high", "max"] as const;
|
|
5
|
+
|
|
6
|
+
export const claudeCodeCatalog = defineHarnessCatalog({
|
|
7
|
+
driver: "claude-code",
|
|
8
|
+
verified: "2026-09-18",
|
|
9
|
+
sources: [
|
|
10
|
+
"https://code.claude.com/docs/en/model-config",
|
|
11
|
+
"https://platform.claude.com/docs/en/about-claude/models/overview",
|
|
12
|
+
],
|
|
13
|
+
efforts: full,
|
|
14
|
+
models: {
|
|
15
|
+
"claude-fable-5-1": full,
|
|
16
|
+
"claude-fable-5": full,
|
|
17
|
+
"claude-opus-5": full,
|
|
18
|
+
"claude-opus-4-8": full,
|
|
19
|
+
"claude-opus-4-7": full,
|
|
20
|
+
"claude-sonnet-5": full,
|
|
21
|
+
"claude-opus-4-6": withoutXhigh,
|
|
22
|
+
"claude-sonnet-4-6": withoutXhigh,
|
|
23
|
+
"claude-haiku-4-5": [],
|
|
24
|
+
},
|
|
25
|
+
});
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import { defineHarnessCatalog } from "../harness-catalog.js";
|
|
2
|
+
|
|
3
|
+
const throughUltra = ["low", "medium", "high", "xhigh", "max", "ultra"] as const;
|
|
4
|
+
const throughMax = ["low", "medium", "high", "xhigh", "max"] as const;
|
|
5
|
+
const throughXhigh = ["low", "medium", "high", "xhigh"] as const;
|
|
6
|
+
const noneThroughXhigh = ["none", "low", "medium", "high", "xhigh"] as const;
|
|
7
|
+
|
|
8
|
+
export const codexCatalog = defineHarnessCatalog({
|
|
9
|
+
driver: "codex",
|
|
10
|
+
verified: "2026-09-18",
|
|
11
|
+
sources: [
|
|
12
|
+
"https://learn.chatgpt.com/docs/models",
|
|
13
|
+
"https://developers.openai.com/api/docs/guides/reasoning",
|
|
14
|
+
"codex app-server model/list (mirrored in $CODEX_HOME/models_cache.json)",
|
|
15
|
+
],
|
|
16
|
+
efforts: ["none", "low", "medium", "high", "xhigh", "max", "ultra"],
|
|
17
|
+
models: {
|
|
18
|
+
"gpt-6-astra": throughUltra,
|
|
19
|
+
"gpt-5.6-sol": throughUltra,
|
|
20
|
+
"gpt-5.6-terra": throughUltra,
|
|
21
|
+
"gpt-5.6-luna": throughMax,
|
|
22
|
+
"gpt-5.5": throughXhigh,
|
|
23
|
+
"gpt-5.4": noneThroughXhigh,
|
|
24
|
+
"gpt-5.4-mini": noneThroughXhigh,
|
|
25
|
+
"gpt-5.3-codex": throughXhigh,
|
|
26
|
+
},
|
|
27
|
+
});
|