@prismatic-io/lux 0.0.2-preview.17 → 0.0.2-preview.20

package/src/cli/run-options.ts

@@ -1,4 +1,9 @@
+import { existsSync } from "node:fs";
+import { basename, extname, resolve } from "node:path";
 import { Errors } from "incur";
+import type { HarnessFocus, ReasoningEffort } from "../core/index.js";
+import { type DiscoveredCase, type SuiteItem, selectCases } from "../orchestrator/index.js";
+import { planHarnessItems } from "./harness-plan.js";
 
 export type RunOptions = {
   tag?: string[];
@@ -11,6 +16,9 @@ export type RunOptions = {
   skipJudge?: boolean;
   runsRoot?: string;
   profile?: string;
+  driver?: string;
+  model?: string;
+  effort?: ReasoningEffort;
   subjectRoot?: string;
   verbose?: boolean;
 };
@@ -64,3 +72,118 @@ export const resolveAnswererOverride = (options: RunOptions): string | undefined
   if (options.claudeAnswerer) return "claude-code";
   return undefined;
 };
+
+const caseIdOf = (casePath: string): string => basename(casePath, extname(casePath));
+
+export const suiteTitle = (
+  patterns: string[],
+  tags: string[],
+  explicit: SuiteItem[],
+  loop: number,
+  focus: HarnessFocus,
+): string => {
+  const parts = [...patterns, ...explicit.map((e) => e.id), ...tags.map((t) => `#${t}`)];
+  const filter = parts.length > 0 ? ` ${parts.join(" ")}` : "";
+  const focusParts = [focus.driver, focus.model, focus.effort].filter(
+    (part): part is string => part !== undefined,
+  );
+  const focused = focusParts.length > 0 ? ` @${focusParts.join(":")}` : "";
+  const repeat = loop > 1 ? ` ×${loop}` : "";
+  return `run${filter}${focused}${repeat}`;
+};
+
+type FilterPartition = { explicit: SuiteItem[]; patterns: string[] };
+
+// A filter that names an existing `.ts` file runs that file directly — it may
+// sit outside the discovery root. Everything else is a substring pattern — but
+// a `.ts` filter that resolves to nothing is almost always a typo'd path, so
+// the silent downgrade to a pattern gets called out.
+export const partitionFilters = (filters: string[], cwd: string): FilterPartition => {
+  const explicit: SuiteItem[] = [];
+  const patterns: string[] = [];
+  for (const f of filters) {
+    const abs = resolve(cwd, f);
+    if (f.endsWith(".ts") && existsSync(abs)) {
+      explicit.push({ path: abs, id: caseIdOf(f) });
+    } else {
+      if (f.endsWith(".ts")) {
+        process.stderr.write(`lux: ${abs} does not exist; treating "${f}" as a pattern\n`);
+      }
+      patterns.push(f);
+    }
+  }
+  return { explicit, patterns };
+};
+
+// The selected set — substring/tag matches plus explicitly named files — is
+// the same whether it is run, `--list`ed, or `--list-tags`ed. No filters at
+// all means "everything discovered"; any filter narrows.
+export const selectSuite = (
+  cases: DiscoveredCase[],
+  { explicit, patterns }: FilterPartition,
+  tags: string[],
+): DiscoveredCase[] => {
+  const hasFilter = explicit.length > 0 || patterns.length > 0 || tags.length > 0;
+  const byPath = new Map<string, DiscoveredCase>();
+  if (!hasFilter) {
+    for (const c of cases) byPath.set(c.path, c);
+  } else {
+    if (patterns.length > 0 || tags.length > 0) {
+      for (const c of selectCases(cases, { patterns, tags })) byPath.set(c.path, c);
+    }
+    for (const e of explicit) {
+      const discovered = cases.find((c) => c.path === e.path);
+      // Discovery knows the case's declared id; the filename-derived one is
+      // only a fallback for files discovery never saw.
+      byPath.set(e.path, {
+        path: e.path,
+        id: discovered?.id ?? e.id,
+        relPath: discovered?.relPath ?? e.id,
+        tags: discovered?.tags ?? [],
+      });
+    }
+  }
+  return [...byPath.values()].sort((a, b) => a.id.localeCompare(b.id));
+};
+
+// `--loop` repeats each eval before moving to the next; each repeat gets a
+// `#n` display suffix so the reporter keeps them apart, while `caseId` keeps
+// every repeat filed under one run history.
+export const expandLoop = (base: SuiteItem[], loop: number): SuiteItem[] =>
+  loop > 1
+    ? base.flatMap((b) =>
+        Array.from({ length: loop }, (_, i) => ({
+          ...b,
+          id: `${b.id} #${i + 1}`,
+          caseId: b.caseId ?? b.id,
+        })),
+      )
+    : base;
+
+export const planSuiteItems = async (
+  selected: readonly { path: string; id: string }[],
+  options: Parameters<typeof planHarnessItems>[1],
+): Promise<SuiteItem[]> => {
+  const plan = await planHarnessItems(selected, options);
+  for (const skipped of plan.unavailable) {
+    process.stderr.write(
+      `lux: skipping ${skipped.id}: harness command not found: ${skipped.commands.join(", ")}\n`,
+    );
+  }
+  if (plan.excluded.length > 0) {
+    process.stderr.write(
+      `lux: the harness focus excludes ${plan.excluded.length} pinned case${plan.excluded.length === 1 ? "" : "s"}: ${plan.excluded.join(", ")}\n`,
+    );
+  }
+  if (plan.items.length === 0) {
+    throw new Errors.IncurError({
+      code: plan.unavailable.length > 0 ? "NO_RUNNABLE_HARNESSES" : "NO_CASES_MATCHED",
+      message:
+        plan.unavailable.length > 0
+          ? "No cases remain: all selected matrix harnesses are unavailable."
+          : "No cases remain after applying the harness focus.",
+      exitCode: 1,
+    });
+  }
+  return plan.items;
+};

package/src/core/case.ts

@@ -3,6 +3,7 @@ import type { AssertionPlugin } from "./assertion.js";
 import { collectEvalCaseExtensions, setEvalCaseExtensions } from "./case-authoring-state.js";
 import { DriverCapabilitySchema } from "./driver.js";
 import { EvaluationMetadataSchema } from "./evaluation-clusters.js";
+import { HarnessMatrixSchema } from "./harness-matrix.js";
 import type { LifecycleHooksPlugin } from "./lifecycle-hooks.js";
 import { formatZodIssues } from "./zod.js";
 
@@ -46,6 +47,8 @@ const EvalCaseObjectSchema = z.object({
   prompt: z.string().min(1, "prompt is required"),
   persona: z.string().min(1).optional(),
   driver: DriverRefSchema.optional(),
+  /** Harness permutations this case runs through instead of the project matrix. */
+  matrix: HarnessMatrixSchema.optional(),
   answerer: AnswererRefSchema.optional(),
   fixtures: FixturesRefSchema.optional(),
   requires: z.array(DriverCapabilitySchema).optional(),

package/src/core/driver.ts

@@ -1,6 +1,7 @@
 import { z } from "zod";
 import type { EventHandler } from "./emitter.js";
 import type { Answer, Done, ErrorEvent, Interrupt, Progress, ReadyState } from "./events.js";
+import type { ReasoningEffort } from "./harness-catalog.js";
 import type { FixtureValueBinding } from "./lifecycle-hooks.js";
 
 export const ArtifactSchema = z.object({
@@ -97,6 +98,12 @@ export type DriverExperiment<Cfg> = {
   identity(config: Cfg, context: { rootDir: string }): unknown | Promise<unknown>;
 };
 
+/** Model and reasoning effort chosen by a harness matrix permutation. */
+export type HarnessSelection = {
+  model?: string | undefined;
+  effort?: ReasoningEffort | undefined;
+};
+
 export type DriverPlugin<Cfg = unknown> = {
   name: string;
   configSchema?: z.ZodTypeAny;
@@ -104,8 +111,27 @@ export type DriverPlugin<Cfg = unknown> = {
   runtime?: (config: Cfg) => DriverRuntime;
   /** Ordinary runs remain supported when controlled experiments are not declared. */
   experiment?: DriverExperiment<Cfg>;
+  /**
+   * Apply a matrix selection to raw (unvalidated) configuration. Drivers
+   * without a model concept leave this undefined and reject selection.
+   */
+  selectHarness?: (config: unknown, selection: HarnessSelection) => unknown;
 };
 
+/** Selection for drivers whose configuration uses `model` and `reasoningEffort`. */
+export const selectModelAndReasoningEffort = (
+  config: unknown,
+  selection: HarnessSelection,
+): unknown => ({
+  ...(config && typeof config === "object" && !Array.isArray(config)
+    ? (config as Record<string, unknown>)
+    : {}),
+  ...(selection.model !== undefined ? { model: selection.model } : {}),
+  ...(selection.effort !== undefined || selection.model !== undefined
+    ? { reasoningEffort: selection.effort }
+    : {}),
+});
+
 export const defineDriver = <S extends z.ZodType>(
   definition: Omit<DriverPlugin<z.output<S>>, "configSchema"> & { configSchema: S },
 ): Omit<DriverPlugin<z.output<S>>, "configSchema"> & { configSchema: S } => definition;

package/src/core/harness-catalog.ts

@@ -0,0 +1,100 @@
+import { z } from "zod";
+
+export const HARNESS_EFFORTS = [
+  "none",
+  "minimal",
+  "low",
+  "medium",
+  "high",
+  "xhigh",
+  "max",
+  "ultra",
+] as const;
+export const ReasoningEffortSchema = z.enum(HARNESS_EFFORTS);
+export type ReasoningEffort = z.infer<typeof ReasoningEffortSchema>;
+
+export type CatalogModels = Readonly<Record<string, readonly ReasoningEffort[]>>;
+
+export type HarnessCatalog<
+  D extends string = string,
+  E extends readonly ReasoningEffort[] = readonly ReasoningEffort[],
+  M extends CatalogModels | undefined = CatalogModels | undefined,
+> = {
+  readonly driver: D;
+  readonly verified: string;
+  readonly sources: readonly string[];
+  readonly efforts: E;
+  readonly models: M;
+};
+
+export type ClosedHarnessCatalog = HarnessCatalog<
+  string,
+  readonly ReasoningEffort[],
+  CatalogModels
+>;
+
+export const defineHarnessCatalog = <
+  const D extends string,
+  const E extends readonly ReasoningEffort[],
+  const M extends CatalogModels | undefined,
+>(
+  catalog: HarnessCatalog<D, E, M>,
+): HarnessCatalog<D, E, M> => catalog;
+
+export const isClosedCatalog = (catalog: HarnessCatalog): catalog is ClosedHarnessCatalog =>
+  catalog.models !== undefined;
+
+export const catalogModelIds = (catalog: HarnessCatalog): string[] =>
+  Object.keys(catalog.models ?? {});
+
+export const catalogEfforts = (
+  catalog: HarnessCatalog,
+  model: string,
+): readonly ReasoningEffort[] | undefined => catalog.models?.[model];
+
+export const defaultCatalogEffort = (
+  catalog: HarnessCatalog,
+  model: string,
+): ReasoningEffort | undefined => {
+  if (!isClosedCatalog(catalog)) return undefined;
+  const accepted = catalog.models[model] ?? catalog.efforts;
+  return accepted.includes("low") ? "low" : accepted[0];
+};
+
+export const describeAcceptedEfforts = (
+  catalog: HarnessCatalog,
+  model: string,
+  accepted: readonly ReasoningEffort[],
+): string =>
+  accepted.length === 0
+    ? `model '${model}' has no reasoning effort in ${catalog.driver}`
+    : `model '${model}' accepts reasoning effort ${accepted.join(", ")} in ${catalog.driver}`;
+
+export const effortIssue = (
+  catalog: HarnessCatalog,
+  model: string,
+  effort: ReasoningEffort | undefined,
+): string | undefined => {
+  if (effort === undefined) return undefined;
+  const accepted = catalogEfforts(catalog, model);
+  if (accepted === undefined || accepted.includes(effort)) return undefined;
+  return describeAcceptedEfforts(catalog, model, accepted);
+};
+
+export type CatalogEffortConfig = {
+  model: string;
+  reasoningEffort?: ReasoningEffort | undefined;
+};
+
+export const applyCatalogEffort =
+  (catalog: HarnessCatalog) =>
+  <C extends CatalogEffortConfig>(config: C, ctx: z.RefinementCtx): C => {
+    const issue = effortIssue(catalog, config.model, config.reasoningEffort);
+    if (issue) {
+      ctx.addIssue({ code: "custom", path: ["reasoningEffort"], message: issue });
+      return config;
+    }
+    if (config.reasoningEffort !== undefined) return config;
+    const effort = defaultCatalogEffort(catalog, config.model);
+    return effort === undefined ? config : { ...config, reasoningEffort: effort };
+  };

package/src/core/harness-catalogs/antigravity.ts

@@ -0,0 +1,24 @@
+import { defineHarnessCatalog } from "../harness-catalog.js";
+
+const flash = ["low", "medium", "high"] as const;
+
+export const antigravityCatalog = defineHarnessCatalog({
+  driver: "antigravity",
+  verified: "2026-09-18",
+  sources: [
+    "https://antigravity.google/docs/models/",
+    "https://antigravity.google/docs/cli/headless/",
+    "agy models (needs a login)",
+  ],
+  efforts: flash,
+  models: {
+    "gemini-3.8-flash": flash,
+    "gemini-3.7-flash": flash,
+    "gemini-3.6-flash": flash,
+    "gemini-3.5-flash": flash,
+    "gemini-3.1-pro": ["low", "high"],
+    "claude-sonnet-4-6": [],
+    "claude-opus-4-6-thinking": [],
+    "gpt-oss-120b-medium": [],
+  },
+});

package/src/core/harness-catalogs/claude-code.ts

@@ -0,0 +1,25 @@
+import { defineHarnessCatalog } from "../harness-catalog.js";
+
+const full = ["low", "medium", "high", "xhigh", "max"] as const;
+const withoutXhigh = ["low", "medium", "high", "max"] as const;
+
+export const claudeCodeCatalog = defineHarnessCatalog({
+  driver: "claude-code",
+  verified: "2026-09-18",
+  sources: [
+    "https://code.claude.com/docs/en/model-config",
+    "https://platform.claude.com/docs/en/about-claude/models/overview",
+  ],
+  efforts: full,
+  models: {
+    "claude-fable-5-1": full,
+    "claude-fable-5": full,
+    "claude-opus-5": full,
+    "claude-opus-4-8": full,
+    "claude-opus-4-7": full,
+    "claude-sonnet-5": full,
+    "claude-opus-4-6": withoutXhigh,
+    "claude-sonnet-4-6": withoutXhigh,
+    "claude-haiku-4-5": [],
+  },
+});

package/src/core/harness-catalogs/codex.ts

@@ -0,0 +1,27 @@
+import { defineHarnessCatalog } from "../harness-catalog.js";
+
+const throughUltra = ["low", "medium", "high", "xhigh", "max", "ultra"] as const;
+const throughMax = ["low", "medium", "high", "xhigh", "max"] as const;
+const throughXhigh = ["low", "medium", "high", "xhigh"] as const;
+const noneThroughXhigh = ["none", "low", "medium", "high", "xhigh"] as const;
+
+export const codexCatalog = defineHarnessCatalog({
+  driver: "codex",
+  verified: "2026-09-18",
+  sources: [
+    "https://learn.chatgpt.com/docs/models",
+    "https://developers.openai.com/api/docs/guides/reasoning",
+    "codex app-server model/list (mirrored in $CODEX_HOME/models_cache.json)",
+  ],
+  efforts: ["none", "low", "medium", "high", "xhigh", "max", "ultra"],
+  models: {
+    "gpt-6-astra": throughUltra,
+    "gpt-5.6-sol": throughUltra,
+    "gpt-5.6-terra": throughUltra,
+    "gpt-5.6-luna": throughMax,
+    "gpt-5.5": throughXhigh,
+    "gpt-5.4": noneThroughXhigh,
+    "gpt-5.4-mini": noneThroughXhigh,
+    "gpt-5.3-codex": throughXhigh,
+  },
+});

package/src/core/harness-catalogs/copilot.ts

@@ -0,0 +1,46 @@
+import { defineHarnessCatalog } from "../harness-catalog.js";
+
+const claude = ["low", "medium", "high", "xhigh", "max"] as const;
+const claudeWithoutXhigh = ["low", "medium", "high", "max"] as const;
+const gptThroughMax = ["none", "low", "medium", "high", "xhigh", "max"] as const;
+const gptThroughXhigh = ["none", "low", "medium", "high", "xhigh"] as const;
+const fixed = [] as const;
+
+export const copilotCatalog = defineHarnessCatalog({
+  driver: "copilot",
+  verified: "2026-09-18",
+  sources: [
+    "copilot help config (model values)",
+    "https://docs.github.com/en/copilot/reference/ai-models/supported-models",
+    "https://docs.github.com/en/copilot/reference/copilot-cli-reference/cli-programmatic-reference",
+  ],
+  efforts: ["none", "minimal", "low", "medium", "high", "xhigh", "max"],
+  models: {
+    "claude-sonnet-5": claude,
+    "claude-fable-5.1": claude,
+    "claude-fable-5": claude,
+    "claude-opus-5": claude,
+    "claude-opus-4.8": claude,
+    "claude-opus-4.8-fast": claude,
+    "claude-opus-4.7": claude,
+    "claude-sonnet-4.6": claudeWithoutXhigh,
+    "claude-haiku-4.5": fixed,
+    "gpt-5.6-sol": gptThroughMax,
+    "gpt-5.6-terra": gptThroughMax,
+    "gpt-5.6-luna": gptThroughMax,
+    "gpt-5.5": gptThroughXhigh,
+    "gpt-5.4": gptThroughXhigh,
+    "gpt-5.4-mini": gptThroughXhigh,
+    "gpt-5.3-codex": ["low", "medium", "high", "xhigh"],
+    "gpt-5-mini": fixed,
+    "mai-code-1.1-flash": fixed,
+    "mai-code-1-flash-picker": fixed,
+    "gemini-3.8-flash": fixed,
+    "gemini-3.7-flash": fixed,
+    "gemini-3.6-flash": fixed,
+    "gemini-3.5-flash": fixed,
+    "grok-4.5": fixed,
+    "kimi-k3": fixed,
+    "kimi-k2.7-code": fixed,
+  },
+});

package/src/core/harness-catalogs/cursor.ts

@@ -0,0 +1,13 @@
+import { defineHarnessCatalog } from "../harness-catalog.js";
+
+export const cursorCatalog = defineHarnessCatalog({
+  driver: "cursor",
+  verified: "2026-09-18",
+  sources: [
+    "agent --list-models (account-scoped, needs a login)",
+    "https://cursor.com/docs/models",
+    "https://cursor.com/docs/cli/acp",
+  ],
+  efforts: ["low", "medium", "high", "xhigh"],
+  models: undefined,
+});

package/src/core/harness-catalogs/index.ts

@@ -0,0 +1,22 @@
+import type { HarnessCatalog } from "../harness-catalog.js";
+import { antigravityCatalog } from "./antigravity.js";
+import { claudeCodeCatalog } from "./claude-code.js";
+import { codexCatalog } from "./codex.js";
+import { copilotCatalog } from "./copilot.js";
+import { cursorCatalog } from "./cursor.js";
+
+export { antigravityCatalog, claudeCodeCatalog, codexCatalog, copilotCatalog, cursorCatalog };
+
+export const builtInHarnessCatalogs = [
+  claudeCodeCatalog,
+  codexCatalog,
+  copilotCatalog,
+  cursorCatalog,
+  antigravityCatalog,
+] as const;
+
+export type BuiltInHarnessCatalog = (typeof builtInHarnessCatalogs)[number];
+export type BuiltInHarnessDriver = BuiltInHarnessCatalog["driver"];
+
+export const harnessCatalogFor = (driver: string): HarnessCatalog | undefined =>
+  builtInHarnessCatalogs.find((catalog) => catalog.driver === driver);