@prismatic-io/lux 0.0.2-preview.4 → 0.0.2-preview.40

package/src/cli/program.ts

@@ -2,47 +2,41 @@ import { randomUUID } from "node:crypto";
 import { existsSync } from "node:fs";
 import { lstat, mkdir, realpath, rename, rm, stat, writeFile } from "node:fs/promises";
 import { createRequire } from "node:module";
-import { basename, dirname, extname, join, relative, resolve, sep } from "node:path";
-import { Command, InvalidArgumentError } from "commander";
-import { createRubricAssertion } from "../assertions/rubric/index.js";
+import { basename, dirname, join, relative, resolve, sep } from "node:path";
+import { fileURLToPath } from "node:url";
+import { Cli, Errors, z } from "incur";
 import {
   type Candidate,
   GraderAnnotationSchema,
+  type HarnessFocus,
   matchesAnyGlob,
+  ReasoningEffortSchema,
   valueHash,
 } from "../core/index.js";
+import { registryWithHarness } from "../orchestrator/harness-registry.js";
 import {
   applyCandidate,
   buildRegistryFromConfig,
   compareRuns,
   compareSuiteRuns,
-  type DiscoveredCase,
+  DoctorReportSchema,
   discoverCases,
-  doctorProject,
+  ExperimentCampaignPlanSchema,
+  ExperimentDecisionReportSchema,
+  ExperimentRunSchema,
   experimentCampaignMode,
   formatComparisonText,
-  formatDoctorReport,
   formatExperimentDecisionMarkdown,
-  formatExperimentRun,
-  formatExperimentRunsTable,
-  formatRunDetail,
-  formatRunsTable,
   formatSuiteComparisonText,
-  formatSuiteRunDetail,
-  formatSuiteRunsTable,
+  GradingSnapshotSchema,
   hasRegression,
   hasSuiteRegression,
-  isExperimentRun,
-  listExperimentRuns,
-  listRuns,
-  listSuiteRuns,
   loadEvalCase,
   loadExperimentCampaign,
   loadExperimentRun,
   loadFrozenRegradableRun,
   loadLuxConfig,
   loadRubricDefinition,
-  loadRunDetail,
   loadSuiteRun,
   loadVerifiedExperimentDecisionSnapshot,
   Orchestrator,
@@ -53,591 +47,608 @@ import {
   resolveExperimentCampaignPath,
   runExperimentCampaign,
   type SuiteCaseResult,
-  type SuiteItem,
-  selectCases,
   upsertGraderAnnotation,
   verifyExperimentDecisionIntegrity,
   verifyExperimentRunIntegrity,
 } from "../orchestrator/index.js";
+import { resolveProfile } from "../orchestrator/profile.js";
 import {
   abortOnInterrupt,
   ensureDirectory,
   failWith,
   relativePathWithin,
 } from "./command-runtime.js";
-import { formatInitReport, initProject } from "./init.js";
+import { doctorCommand } from "./doctor.js";
+import { initProject } from "./init.js";
+import { CompareOutputSchema, ViewOutputSchema } from "./output-schemas.js";
 import { ConsoleReporter } from "./render/reporter.js";
 import {
-  collectTags,
-  collectValues,
+  expandLoop,
   parseAnnotationAssertion,
-  parsePositiveInt,
-  parseUnitInterval,
+  partitionFilters,
+  planSuiteItems,
   type RunOptions,
   resolveAnswererOverride,
   resolveConcurrency,
+  selectSuite,
+  suiteTitle,
 } from "./run-options.js";
-import {
-  type BundledSkillName,
-  formatSkillsInstall,
-  formatSkillsList,
-  installSkills,
-  type SkillsPlatform,
-} from "./skills.js";
+import { type ViewOptions, viewCommand, webViewCommand } from "./view.js";
 
 const requireFromHere = createRequire(import.meta.url);
 // From lib/cli/program.js (or src/cli/program.ts under test) to the package root.
 const PKG_VERSION = (requireFromHere("../../package.json") as { version: string }).version;
 
-const registerProjectCommands = (program: Command): void => {
-  program
-    .command("init")
-    .description("scaffold a new evals project in the current directory")
-    .option("--force", "overwrite existing files")
-    .option("--experiment", "also scaffold a prompt optimization campaign and split cases")
-    .action(async (options: { force?: boolean; experiment?: boolean }) => {
-      const cwd = process.cwd();
-      const result = await initProject({
-        cwd,
-        force: options.force ?? false,
-        experiment: options.experiment ?? false,
-      });
-      process.stdout.write(`${formatInitReport(result, cwd)}\n`);
-    });
-
-  const skills = program.command("skills").description("manage bundled agent skills");
-
-  skills
-    .command("list")
-    .description("list bundled skills and supported platforms")
-    .action(() => {
-      process.stdout.write(`${formatSkillsList()}\n`);
-    });
-
-  skills
-    .command("install")
-    .description("install the Lux eval-authoring skill for Claude Code or Codex")
-    .argument("<platform>", "claude, codex, or all")
-    .option("--project", "install into the current project instead of the user profile")
-    .option("--dir <path>", "install under an explicit skills directory")
-    .option("--name <name>", "install one bundled skill")
-    .action(
-      async (
-        platform: string,
-        options: {
-          project?: boolean;
-          dir?: string;
-          name?: string;
+const nonEmptyString = z.string().min(1);
+const positiveInteger = z.coerce.number().int().positive();
+const unitInterval = z.coerce.number().min(0).max(1);
+const tagList = z
+  .array(nonEmptyString)
+  .default([])
+  .describe("Repeatable list; each value may be comma-separated");
+const normalizeTags = (values: string[]): string[] =>
+  values.flatMap((value) => value.split(",").map((tag) => tag.trim())).filter(Boolean);
+const initOutput = z.object({
+  created: z.array(z.string()),
+  updated: z.array(z.string()),
+  skipped: z.array(z.object({ path: z.string(), reason: z.string() })),
+  experiment: z.boolean(),
+});
+const runOutput = z.union([
+  z.object({
+    caseCount: z.number().int(),
+    cases: z.array(
+      z.object({
+        path: z.string(),
+        id: z.string(),
+        caseId: z.string().optional(),
+        permutationId: z.string().optional(),
+      }),
+    ),
+  }),
+  z.object({
+    caseCount: z.number().int(),
+    tags: z.array(z.object({ tag: z.string(), count: z.number().int() })),
+  }),
+  z.object({
+    suiteDir: z.string(),
+    results: z.array(
+      z.object({
+        caseId: z.string(),
+        runDir: z.string().nullable(),
+        exitReason: z.string(),
+        casePassed: z.boolean().nullable(),
+        caseScore: z.number().nullable(),
+        assertionsPassed: z.number().int(),
+        assertionsTotal: z.number().int(),
+      }),
+    ),
+  }),
+]);
+const experimentOutput = z.union([
+  ExperimentCampaignPlanSchema,
+  ExperimentRunSchema.extend({ experimentDir: z.string() }),
+]);
+const gradingOutput = GradingSnapshotSchema.extend({ gradingPath: z.string() });
+const reportOutput = z.union([
+  ExperimentDecisionReportSchema,
+  z.object({ path: z.string(), report: ExperimentDecisionReportSchema }),
+]);
+
+const campaignArgs = z.object({ campaign: nonEmptyString.describe("Experiment campaign path") });
+const campaignOptions = z.object({
+  runsRoot: nonEmptyString.optional().describe("Run directory root overriding lux.config.ts"),
+  resume: nonEmptyString.optional().describe("Interrupted experiment directory to resume"),
+  plan: z.boolean().optional().describe("Validate and estimate without calling models"),
+});
+
+export const buildCli = () =>
+  Cli.create("lux", {
+    description:
+      "Coding-agent evaluation and improvement with deterministic assertions and human-in-the-loop runs",
+    version: PKG_VERSION,
+    sync: {
+      include: [fileURLToPath(new URL("../../skills/*", import.meta.url))],
+      depth: 2,
+      suggestions: [
+        "Use Lux to inspect the eval cases in this project",
+        "Use Lux to diagnose the latest failed eval run",
+        "Use Lux to plan an evidence-based improvement experiment",
+      ],
+    },
+    mcp: {
+      title: "Lux coding-agent evaluation",
+      instructions:
+        "Inspect and plan before running model-spending commands. Apply only a verified promoted experiment champion.",
+    },
+  })
+    .command("init", {
+      description: "Scaffold a new eval project in the current directory",
+      options: z.object({
+        force: z.boolean().optional().describe("Overwrite existing files"),
+        experiment: z.boolean().optional().describe("Also scaffold an optimization campaign"),
+        driver: z
+          .enum([
+            "subprocess",
+            "claude-code",
+            "codex",
+            "cursor",
+            "copilot",
+            "antigravity",
+            "grok-build",
+          ])
+          .optional()
+          .describe("Starter driver (default: deterministic subprocess)"),
+        model: nonEmptyString.optional().describe("Explicit subject model; required for Cursor"),
+      }),
+      output: initOutput,
+      destructive: true,
+      mcp: { annotations: { destructiveHint: true, openWorldHint: false } },
+      examples: [
+        { description: "Scaffold a basic eval project" },
+        { options: { experiment: true }, description: "Scaffold an optimization project" },
+      ],
+      run: async (c) =>
+        c.ok(await initCommand(c.options), { cta: { commands: ["doctor", "run --list"] } }),
+    })
+    .command("doctor", {
+      description: "Preflight project configuration, CLIs, cases, campaigns, and budgets",
+      args: z.object({ campaigns: z.array(z.string()).default([]).describe("Campaign paths") }),
+      options: z.object({
+        static: z
+          .boolean()
+          .optional()
+          .describe("Validate project files without checking installed executables (for CI)"),
+      }),
+      output: DoctorReportSchema,
+      mcp: {
+        annotations: { readOnlyHint: true, destructiveHint: false, openWorldHint: false },
+      },
+      run: async (c) =>
+        c.ok(await doctorCommand(c.args.campaigns, c.options), {
+          cta: { commands: ["run --list"] },
+        }),
+    })
+    .command("run", {
+      description: "Discover and run cases, filtered by name, path, or tag",
+      args: z.object({ filters: z.array(z.string()).default([]).describe("Case names or paths") }),
+      options: z.object({
+        tag: tagList.describe("Required case tag; repeatable and comma-separated"),
+        loop: positiveInteger.optional().describe("Run each matched eval this many times"),
+        concurrency: positiveInteger.optional().describe("Maximum cases running concurrently"),
+        list: z.boolean().optional().describe("List matching cases without running"),
+        listTags: z.boolean().optional().describe("List tags across matching cases"),
+        interactive: z.boolean().optional().describe("Answer agent questions in this terminal"),
+        claudeAnswerer: z.boolean().optional().describe("Use Claude Code as the answerer"),
+        skipJudge: z.boolean().optional().describe("Skip rubric LLM judge assertions"),
+        runsRoot: nonEmptyString.optional().describe("Run directory root"),
+        profile: nonEmptyString.optional().describe("Named driver and answerer profile"),
+        driver: nonEmptyString
+          .optional()
+          .describe("Focus on one harness driver, replacing the matrix"),
+        model: nonEmptyString.optional().describe("Focus on one subject model"),
+        effort: ReasoningEffortSchema.optional().describe("Focus on one reasoning effort"),
+        subjectRoot: nonEmptyString.optional().describe("Root bound into subjectPath values"),
+        verbose: z.boolean().optional().describe("Include phase timings and run directories"),
+      }),
+      alias: { tag: "t", concurrency: "j", interactive: "i", verbose: "v" },
+      output: runOutput,
+      outputPolicy: "agent-only",
+      mcp: { annotations: { readOnlyHint: false, destructiveHint: false, openWorldHint: true } },
+      examples: [
+        { options: { list: true, tag: [] }, description: "List all discovered cases" },
+        {
+          args: { filters: ["smoke"] },
+          options: { tag: [] },
+          description: "Run matching cases",
         },
-      ) => {
-        if (!["claude", "claude-code", "codex", "all"].includes(platform)) {
-          throw new InvalidArgumentError("platform must be claude, codex, or all");
-        }
-        if (options.name && !["lux", "lux-answerer"].includes(options.name)) {
-          throw new InvalidArgumentError("skill name must be lux or lux-answerer");
-        }
-        const cwd = process.cwd();
-        const installed = await installSkills({
-          cwd,
-          platform: platform as SkillsPlatform,
-          project: options.project ?? false,
-          ...(options.dir ? { dir: options.dir } : {}),
-          ...(options.name ? { name: options.name as BundledSkillName } : {}),
+        {
+          args: { filters: ["smoke"] },
+          options: { tag: [], driver: "codex", model: "gpt-5.6-luna", effort: "low" },
+          description: "Focus on one harness permutation",
+        },
+      ],
+      run: async (c) => {
+        const result = await runCommand(c.args.filters, c.options as RunOptions, !c.agent);
+        const suiteDir = "suiteDir" in result ? result.suiteDir : undefined;
+        return c.ok(result, {
+          cta: {
+            description: "Inspect saved results:",
+            commands: suiteDir ? [{ command: "view", args: { runDir: suiteDir } }] : [],
+          },
         });
-        process.stdout.write(`${formatSkillsInstall(installed, cwd)}\n`);
       },
-    );
-};
-
-const registerExperimentCommands = (program: Command): void => {
-  program
-    .command("doctor [campaigns...]")
-    .description("preflight project configuration, CLIs, cases, campaigns, and budgets")
-    .option("--json", "emit machine-readable JSON")
-    .action(async (campaigns: string[], options: { json?: boolean }) => {
-      const report = await doctorProject({ cwd: process.cwd(), campaigns });
-      process.stdout.write(
-        options.json ? `${JSON.stringify(report, null, 2)}\n` : `${formatDoctorReport(report)}\n`,
-      );
-      if (!report.ok) failWith(1);
+    })
+    .command("experiment", {
+      description: "Evaluate controlled source variants across cases and model profiles",
+      args: campaignArgs,
+      options: campaignOptions,
+      output: experimentOutput,
+      mcp: { annotations: { readOnlyHint: false, destructiveHint: false, openWorldHint: true } },
+      run: async (c) => experimentCommand("experiment", c.args.campaign, c.options),
+    })
+    .command("optimize", {
+      description: "Evolve candidates from grading feedback and promote on held-out cases",
+      args: campaignArgs,
+      options: campaignOptions,
+      output: experimentOutput,
+      destructive: true,
+      mcp: { annotations: { readOnlyHint: false, destructiveHint: true, openWorldHint: true } },
+      run: async (c) => experimentCommand("optimize", c.args.campaign, c.options),
+    })
+    .command("grade", {
+      description: "Re-run current graders against a persisted run",
+      args: z.object({ runDir: nonEmptyString.describe("Persisted run directory") }),
+      options: z.object({ case: nonEmptyString.optional().describe("Current case definition") }),
+      output: gradingOutput,
+      mcp: { annotations: { readOnlyHint: false, destructiveHint: false, openWorldHint: true } },
+      run: async (c) => gradeCommand(c.args.runDir, c.options),
+    })
+    .command("annotate", {
+      description: "Add or update a human grader-alignment label",
+      args: z.object({ runDir: nonEmptyString.describe("Persisted run directory") }),
+      options: z.object({
+        out: nonEmptyString.describe("JSON annotation dataset to create or update"),
+        rubric: nonEmptyString.describe("Candidate-relative declarative rubric path"),
+        id: nonEmptyString.optional().describe("Stable annotation ID"),
+        label: z.enum(["pass", "fail"]).optional().describe("Expected overall verdict"),
+        score: unitInterval.optional().describe("Expected score from zero to one"),
+        assertion: z
+          .array(nonEmptyString)
+          .default([])
+          .describe("Assertion label id=pass|fail[:score]"),
+        split: z.enum(["train", "validation", "test"]).optional().describe("Dataset split"),
+        tag: tagList.describe("Selection tag; repeatable and comma-separated"),
+        feedback: nonEmptyString.optional().describe("Human rationale for mismatches"),
+        annotator: nonEmptyString.optional().describe("Label author"),
+        reviewer: nonEmptyString.optional().describe("Independent reviewer"),
+        source: z.enum(["human", "synthetic"]).default("human").describe("Label provenance"),
+      }),
+      output: z.object({ id: z.string(), path: z.string(), labels: z.number().int() }),
+      destructive: true,
+      mcp: {
+        annotations: {
+          readOnlyHint: false,
+          destructiveHint: false,
+          idempotentHint: true,
+          openWorldHint: false,
+        },
+      },
+      run: async (c) => annotateCommand(c.args.runDir, c.options),
+    })
+    .command("apply", {
+      description: "Apply the verified promoted experiment champion",
+      args: z.object({
+        experimentDir: nonEmptyString.describe("Experiment directory"),
+        candidateId: nonEmptyString.optional().describe("Expected promoted candidate ID"),
+      }),
+      output: z.object({
+        candidateId: z.string(),
+        subjectRoot: z.string(),
+        sourceBytes: z.number(),
+      }),
+      destructive: true,
+      mcp: {
+        annotations: { readOnlyHint: false, destructiveHint: true, openWorldHint: false },
+      },
+      run: async (c) => applyCommand(c.args.experimentDir, c.args.candidateId),
+    })
+    .command("report", {
+      description: "Render a PR-ready report from a verified experiment record",
+      args: z.object({ experimentDir: nonEmptyString.describe("Experiment directory") }),
+      options: z.object({
+        out: nonEmptyString.optional().describe("Write Markdown to this path"),
+        force: z.boolean().optional().describe("Overwrite an existing output file"),
+      }),
+      output: reportOutput,
+      mcp: { annotations: { readOnlyHint: false, destructiveHint: false, openWorldHint: false } },
+      run: async (c) => reportCommand(c.args.experimentDir, c.options),
+    })
+    .command("view", {
+      description: "List past runs or show one run in detail",
+      args: z.object({ runDir: nonEmptyString.optional().describe("Run or suite directory") }),
+      options: z.object({
+        runsRoot: nonEmptyString.optional().describe("Run directory root"),
+        case: nonEmptyString.optional().describe("Filter lists to one case slug"),
+        suites: z.boolean().optional().describe("List logical suite runs"),
+        experiments: z.boolean().optional().describe("List experiment runs"),
+        web: z.boolean().optional().describe("Open the Lux Review web app"),
+        port: z.coerce.number().int().min(0).max(65535).optional().describe("Review server port"),
+        open: z.boolean().default(true).describe("Open a browser in web mode"),
+      }),
+      output: ViewOutputSchema,
+      mcp: { annotations: { readOnlyHint: true, destructiveHint: false, openWorldHint: false } },
+      run: (c) => {
+        if (c.options.web) {
+          if (c.formatExplicit && c.format !== "jsonl") {
+            throw new Errors.ParseError({
+              message: "--web requires streaming output; omit --format or use --format jsonl",
+            });
+          }
+          return webViewCommand(c.args.runDir, c.options as ViewOptions);
+        }
+        return viewCommand(c.args.runDir, c.options as ViewOptions).then((result) =>
+          ViewOutputSchema.parse(result),
+        );
+      },
+    })
+    .command("compare", {
+      description: "Diff two run or suite directories and fail on regression",
+      args: z.object({
+        runA: nonEmptyString.describe("Baseline run"),
+        runB: nonEmptyString.describe("Current run"),
+      }),
+      options: z.object({
+        passRateTolerance: unitInterval.optional().describe("Allowed per-case pass-rate decrease"),
+      }),
+      output: CompareOutputSchema,
+      mcp: { annotations: { readOnlyHint: true, destructiveHint: false, openWorldHint: false } },
+      run: async (c) =>
+        CompareOutputSchema.parse(await compareCommand(c.args.runA, c.args.runB, c.options)),
     });
 
-  program
-    .command("run [filters...]")
-    .description("discover and run cases; filter by name/path substring or --tag")
-    .option(
-      "-t, --tag <tag>",
-      "only cases whose meta.tags include this tag (repeatable, comma-separated)",
-      collectTags,
-      [],
-    )
-    .option("--loop <n>", "run each matched eval n times", parsePositiveInt)
-    .option(
-      "-j, --concurrency <n>",
-      "run up to n cases at once (overrides lux.config.ts)",
-      parsePositiveInt,
-    )
-    .option("--list", "print the matched cases and exit without running them")
-    .option("--list-tags", "print the tags across the matched cases and exit")
-    .option("-i, --interactive", "answer agent questions yourself in this terminal (HITL)")
-    .option(
-      "--claude-answerer",
-      "delegate questions through the Claude plugin's lux-answerer skill",
-    )
-    .option("--skip-judge", "skip rubric LLM judge assertions")
-    .option("--runs-root <path>", "run-dir root (overrides lux.config.ts)")
-    .option("--profile <id>", "named driver/answerer profile from lux.config.ts")
-    .option("--subject-root <path>", "root bound into subjectPath() values")
-    .option("-v, --verbose", "add phase timings and the run dir to every case")
-    .action(async (filters: string[], options: RunOptions) => {
-      await runCommand(filters, options);
+const initCommand = async (options: Omit<Parameters<typeof initProject>[0], "cwd">) =>
+  initProject({ cwd: process.cwd(), ...options });
+
+const gradeCommand = async (runDir: string, options: { case?: string | undefined }) => {
+  const directory = resolve(runDir);
+  if (!(await ensureDirectory("grade", directory))) failWith(1);
+  const config = await loadLuxConfig(process.cwd());
+  using controller = abortOnInterrupt();
+  const result = await regradeRun({
+    runDir: directory,
+    registry: buildRegistryFromConfig(config),
+    graderContext: config.harness ?? null,
+    ...(options.case ? { case: await loadEvalCase(resolve(options.case)) } : {}),
+    abortSignal: controller.signal,
+  });
+  if (controller.signal.aborted) failWith(SIGINT_EXIT_CODE);
+  if (result.snapshot.report.casePassed !== true) {
+    throw new Errors.IncurError({
+      code: "GRADING_FAILED",
+      message: `The persisted run did not pass regrading. Grading saved to ${result.path}.`,
+      exitCode: 1,
     });
+  }
+  return { gradingPath: result.path, ...result.snapshot };
+};
 
-  program
-    .command("experiment <campaign>")
-    .description("evaluate controlled source variants across cases and model profiles")
-    .option("--runs-root <path>", "run-dir root (overrides lux.config.ts)")
-    .option("--resume <experiment-dir>", "resume an interrupted experiment ledger")
-    .option("--plan", "validate and estimate the campaign without calling models")
-    .option("--json", "emit machine-readable JSON")
-    .action(
-      async (
-        campaign: string,
-        options: { runsRoot?: string; resume?: string; json?: boolean; plan?: boolean },
-      ) => {
-        await experimentCommand("experiment", campaign, options);
-      },
-    );
+type AnnotationOptions = {
+  out: string;
+  rubric: string;
+  id?: string | undefined;
+  label?: "pass" | "fail" | undefined;
+  score?: number | undefined;
+  assertion: string[];
+  split?: "train" | "validation" | "test" | undefined;
+  tag: string[];
+  feedback?: string | undefined;
+  annotator?: string | undefined;
+  reviewer?: string | undefined;
+  source: "human" | "synthetic";
 };
 
-const registerCandidateCommands = (program: Command): void => {
-  program
-    .command("grade <runDir>")
-    .description("re-run current graders against a persisted run without rerunning the agent")
-    .option("--case <path>", "grade with a current case definition instead of frozen case.json")
-    .option("--json", "emit machine-readable JSON")
-    .action(async (runDir: string, options: { case?: string; json?: boolean }) => {
-      const directory = resolve(runDir);
-      if (!(await ensureDirectory("grade", directory))) return;
-      const config = await loadLuxConfig(process.cwd());
-      const controller = abortOnInterrupt();
-      const result = await regradeRun({
-        runDir: directory,
-        registry: buildRegistryFromConfig(config),
-        graderContext: config.harness ?? null,
-        ...(options.case ? { case: await loadEvalCase(resolve(options.case)) } : {}),
-        abortSignal: controller.signal,
-      });
-      const report = result.snapshot.report;
-      let gradingStatus = "ungraded";
-      if (report.casePassed === true) gradingStatus = "PASS";
-      if (report.casePassed === false) gradingStatus = "FAIL";
-      let output: string;
-      if (options.json) {
-        output = `${JSON.stringify({ gradingPath: result.path, ...result.snapshot }, null, 2)}\n`;
-      } else {
-        const score = report.caseScore === null ? "—" : `${(report.caseScore * 100).toFixed(0)}%`;
-        output = [
-          `grading: ${gradingStatus}`,
-          `score:   ${score}`,
-          `checks:  ${report.assertions.filter((assertion) => assertion.passed).length}/${report.assertions.length}`,
-          `saved:   ${result.path}`,
-          "",
-        ].join("\n");
-      }
-      process.stdout.write(output);
-      if (controller.signal.aborted) failWith(SIGINT_EXIT_CODE);
-      else if (report.casePassed !== true) failWith(1);
+const annotateCommand = async (runDir: string, options: AnnotationOptions) => {
+  const directory = resolve(runDir);
+  if (!(await ensureDirectory("annotate", directory))) failWith(1);
+  const parsedAssertions = options.assertion.map(parseAnnotationAssertion);
+  const assertionIds = parsedAssertions.map(([id]) => id);
+  if (new Set(assertionIds).size !== assertionIds.length) {
+    throw new Errors.ParseError({ message: "--assertion IDs must not be repeated" });
+  }
+  const assertions = Object.fromEntries(parsedAssertions);
+  if (
+    options.label === undefined &&
+    options.score === undefined &&
+    Object.keys(assertions).length === 0
+  ) {
+    throw new Errors.ParseError({
+      message: "Provide --label, --score, or at least one --assertion",
     });
-
-  program
-    .command("annotate <runDir>")
-    .description("add or update a human grader-alignment label for a persisted run")
-    .requiredOption("--out <path>", "JSON annotation dataset to create or update")
-    .requiredOption("--rubric <path>", "candidate-relative declarative .json rubric path")
-    .option("--id <id>", "stable annotation id")
-    .option("--label <pass|fail>", "expected overall case verdict")
-    .option("--score <n>", "expected overall case score from 0 to 1", parseUnitInterval)
-    .option(
-      "--assertion <id=pass|fail[:score]>",
-      "expected stable assertion verdict and optional score (repeatable)",
-      collectValues,
-      [],
-    )
-    .option("--split <train|validation|test>", "dataset split, stored as a tag")
-    .option("--tag <tag>", "extra selection tag (repeatable, comma-separated)", collectTags, [])
-    .option("--feedback <text>", "human rationale shown to the rubric optimizer on mismatch")
-    .option("--annotator <id>", "person or system that authored the label")
-    .option("--reviewer <id>", "independent reviewer of the label")
-    .option("--source <human|synthetic>", "label provenance", "human")
-    .action(
-      async (
-        runDir: string,
-        options: {
-          out: string;
-          rubric: string;
-          id?: string;
-          label?: string;
-          score?: number;
-          assertion: string[];
-          split?: string;
-          tag: string[];
-          feedback?: string;
-          annotator?: string;
-          reviewer?: string;
-          source: string;
-        },
-      ) => {
-        const directory = resolve(runDir);
-        if (!(await ensureDirectory("annotate", directory))) return;
-        if (options.label && options.label !== "pass" && options.label !== "fail") {
-          throw new InvalidArgumentError("--label must be pass or fail");
-        }
-        if (options.split && !["train", "validation", "test"].includes(options.split)) {
-          throw new InvalidArgumentError("--split must be train, validation, or test");
-        }
-        if (options.source !== "human" && options.source !== "synthetic") {
-          throw new InvalidArgumentError("--source must be human or synthetic");
-        }
-        const parsedAssertions = options.assertion.map(parseAnnotationAssertion);
-        const assertionIds = parsedAssertions.map(([id]) => id);
-        if (new Set(assertionIds).size !== assertionIds.length) {
-          throw new InvalidArgumentError("--assertion ids must not be repeated");
-        }
-        const assertions = Object.fromEntries(parsedAssertions);
-        if (
-          options.label === undefined &&
-          options.score === undefined &&
-          Object.keys(assertions).length === 0
-        ) {
-          throw new Error("lux annotate: provide --label, --score, or at least one --assertion");
-        }
-        const out = resolve(options.out);
-        const run = await loadFrozenRegradableRun(directory);
-        try {
-          const rubric = await loadRubricDefinition(
-            await resolveAuthoredFile(options.rubric, "rubric"),
-          );
-          if (rubric.id !== run.case.id) {
-            throw new Error(
-              `lux annotate: rubric id '${rubric.id}' does not match persisted case '${run.case.id}'`,
-            );
-          }
-          const rubricIds = new Set(rubric.assertions.map((assertion) => assertion.id));
-          const unknownIds = assertionIds.filter((assertionId) => !rubricIds.has(assertionId));
-          if (unknownIds.length > 0) {
-            throw new Error(`lux annotate: unknown rubric assertion ids: ${unknownIds.join(", ")}`);
-          }
-          const tags = [...(options.split ? [options.split] : []), ...options.tag].filter(
-            (tag, index, all) => all.indexOf(tag) === index,
-          );
-          const caseId = run.case.id;
-          const portableRunDir = (relative(dirname(out), directory) || ".").split(sep).join("/");
-          const id =
-            options.id ??
-            `${caseId}-${basename(directory)}-${valueHash(portableRunDir).slice(0, 10)}`.replace(
-              /[^a-zA-Z0-9_-]+/g,
-              "-",
-            );
-          const annotation = GraderAnnotationSchema.parse({
-            id,
-            runDir: portableRunDir,
-            rubricPath: options.rubric,
-            tags,
-            source: options.source,
-            ...(options.annotator ? { annotator: options.annotator } : {}),
-            labeledAt: new Date().toISOString(),
-            ...(options.reviewer ? { reviewer: options.reviewer } : {}),
-            expected: {
-              ...(options.label ? { casePassed: options.label === "pass" } : {}),
-              ...(options.score !== undefined ? { caseScore: options.score } : {}),
-              assertions,
-            },
-            ...(options.feedback ? { feedback: options.feedback } : {}),
-          });
-          const dataset = await upsertGraderAnnotation(out, annotation);
-          process.stdout.write(
-            `annotated ${annotation.id} in ${out} (${dataset.annotations.length} labels)\n`,
-          );
-        } finally {
-          await releaseFrozenRegradableRun(run);
-        }
+  }
+  const out = resolve(options.out);
+  const run = await loadFrozenRegradableRun(directory);
+  try {
+    const rubric = await loadRubricDefinition(await resolveAuthoredFile(options.rubric, "rubric"));
+    if (rubric.id !== run.case.id) {
+      throw new Error(`Rubric ID '${rubric.id}' does not match persisted case '${run.case.id}'.`);
+    }
+    const rubricIds = new Set(rubric.assertions.map((assertion) => assertion.id));
+    const unknownIds = assertionIds.filter((assertionId) => !rubricIds.has(assertionId));
+    if (unknownIds.length > 0) {
+      throw new Errors.ParseError({
+        message: `Unknown rubric assertion IDs: ${unknownIds.join(", ")}`,
+      });
+    }
+    const annotationTags = [
+      ...(options.split ? [options.split] : []),
+      ...normalizeTags(options.tag),
+    ].filter((tag, index, all) => all.indexOf(tag) === index);
+    const portableRunDir = (relative(dirname(out), directory) || ".").split(sep).join("/");
+    const id =
+      options.id ??
+      `${run.case.id}-${basename(directory)}-${valueHash(portableRunDir).slice(0, 10)}`.replace(
+        /[^a-zA-Z0-9_-]+/g,
+        "-",
+      );
+    const annotation = GraderAnnotationSchema.parse({
+      id,
+      runDir: portableRunDir,
+      rubricPath: options.rubric,
+      tags: annotationTags,
+      source: options.source,
+      ...(options.annotator ? { annotator: options.annotator } : {}),
+      labeledAt: new Date().toISOString(),
+      ...(options.reviewer ? { reviewer: options.reviewer } : {}),
+      expected: {
+        ...(options.label ? { casePassed: options.label === "pass" } : {}),
+        ...(options.score !== undefined ? { caseScore: options.score } : {}),
+        assertions,
       },
-    );
+      ...(options.feedback ? { feedback: options.feedback } : {}),
+    });
+    const dataset = await upsertGraderAnnotation(out, annotation);
+    return { id: annotation.id, path: out, labels: dataset.annotations.length };
+  } finally {
+    await releaseFrozenRegradableRun(run);
+  }
 };
 
-const registerReportingCommands = (program: Command): void => {
-  program
-    .command("optimize <campaign>")
-    .description("evolve source candidates from grading feedback and promote on held-out cases")
-    .option("--runs-root <path>", "run-dir root (overrides lux.config.ts)")
-    .option("--resume <experiment-dir>", "resume an interrupted optimization ledger")
-    .option("--plan", "validate and estimate the campaign without calling models")
-    .option("--json", "emit machine-readable JSON")
-    .action(
-      async (
-        campaign: string,
-        options: { runsRoot?: string; resume?: string; json?: boolean; plan?: boolean },
-      ) => {
-        await experimentCommand("optimize", campaign, options);
-      },
-    );
-
-  program
-    .command("apply <experimentDir> [candidateId]")
-    .description("apply the promoted experiment champion after verifying the source base hash")
-    .action(async (experimentDir: string, candidateId?: string) => {
-      const directory = resolve(experimentDir);
-      const experiment = await loadExperimentRun(directory);
-      if (!experiment) {
-        process.stderr.write(`lux apply: experiment manifest not found at ${directory}\n`);
-        failWith(1);
-        return;
-      }
-      if (experiment.status !== "passed") {
-        process.stderr.write(
-          `lux apply: experiment status is '${experiment.status}', not 'passed'\n`,
-        );
-        failWith(1);
-        return;
-      }
-      const championId = experiment.championCandidateId;
-      if (!championId) {
-        process.stderr.write("lux apply: passed experiment has no promoted champion\n");
-        failWith(1);
-        return;
-      }
-      if (candidateId && candidateId !== championId) {
-        process.stderr.write(
-          `lux apply: candidate '${candidateId}' is not the promoted champion '${championId}'\n`,
-        );
-        failWith(1);
-        return;
-      }
-      let candidate: Candidate | undefined;
-      try {
-        const candidates = await verifyExperimentRunIntegrity(directory, experiment);
-        verifyExperimentDecisionIntegrity(experiment);
-        candidate = candidates.find((entry) => entry.id === championId);
-        if (!candidate) {
-          throw new Error(`champion candidate '${championId}' not found`);
-        }
-      } catch (error) {
-        const message = error instanceof Error ? error.message : String(error);
-        process.stderr.write(`lux apply: experiment integrity check failed: ${message}\n`);
-        failWith(1);
-        return;
-      }
-      await applyCandidate(experiment.subjectRoot, experiment.campaignSnapshot.subject, candidate);
-      process.stdout.write(
-        `applied candidate ${candidate.id} to ${experiment.subjectRoot} (${candidate.sourceBytes} bytes)\n`,
-      );
+const applyCommand = async (experimentDir: string, candidateId?: string) => {
+  const directory = resolve(experimentDir);
+  const experiment = await loadExperimentRun(directory);
+  if (!experiment) {
+    throw new Errors.IncurError({
+      code: "EXPERIMENT_NOT_FOUND",
+      message: `Experiment manifest not found at ${directory}.`,
+      exitCode: 1,
     });
-
-  program
-    .command("report <experimentDir>")
-    .description("render a PR-ready promotion report from a verified experiment record")
-    .option("--json", "emit the structured decision report as JSON")
-    .option("--out <path>", "write the report to a file instead of stdout")
-    .option("--force", "overwrite an existing output file")
-    .action(
-      async (experimentDir: string, options: { json?: boolean; out?: string; force?: boolean }) => {
-        const directory = resolve(experimentDir);
-        if (!(await ensureDirectory("report", directory))) return;
-        const snapshot = await loadVerifiedExperimentDecisionSnapshot(directory);
-        const { experiment, report } = snapshot;
-        const rendered = options.json
-          ? `${JSON.stringify(report, null, 2)}\n`
-          : `${formatExperimentDecisionMarkdown(report)}\n`;
-        if (!options.out) {
-          process.stdout.write(rendered);
-          return;
-        }
-        const out = resolve(options.out);
-        const outputStats = await lstat(out).catch((error: NodeJS.ErrnoException) => {
-          if (error.code === "ENOENT") return null;
-          throw error;
-        });
-        if (outputStats && (outputStats.isSymbolicLink() || !outputStats.isFile())) {
-          throw new Error("lux report: an existing --out must be a regular, non-symlink file");
-        }
-        const [physicalDirectory, physicalSubjectRoot, physicalOut] = await Promise.all([
-          realpath(directory),
-          realpath(experiment.subjectRoot),
-          physicalDestinationPath(out),
-        ]);
-        assertReportDestination(
-          physicalOut,
-          physicalDirectory,
-          physicalSubjectRoot,
-          experiment.campaignSnapshot.subject.exclude,
-        );
-        if (outputStats && !options.force) {
-          throw new Error(`lux report: output already exists (use --force): ${out}`);
-        }
-        await mkdir(dirname(out), { recursive: true });
-        const physicalParent = await realpath(dirname(out));
-        const finalPhysicalOut = resolve(physicalParent, basename(out));
-        assertReportDestination(
-          finalPhysicalOut,
-          physicalDirectory,
-          physicalSubjectRoot,
-          experiment.campaignSnapshot.subject.exclude,
-        );
-        await writeReportSafely(finalPhysicalOut, rendered, options.force ?? false);
-        process.stdout.write(`wrote experiment report to ${out}\n`);
-      },
-    );
-
-  program
-    .command("view [runDir]")
-    .description("list past runs, or show one run's detail when given a runDir")
-    .option("--runs-root <path>", "run-dir root (defaults to lux.config.ts)")
-    .option("--case <id>", "filter the run list to one case slug (list mode only)")
-    .option("--suites", "list logical suite runs instead of individual runs")
-    .option("--experiments", "list experiment runs instead of individual runs")
-    .option("--json", "emit machine-readable JSON")
-    .action(
-      async (
-        runDirArg: string | undefined,
-        options: {
-          runsRoot?: string;
-          case?: string;
-          suites?: boolean;
-          experiments?: boolean;
-          json?: boolean;
-        },
-      ) => {
-        if (runDirArg) {
-          const path = resolve(runDirArg);
-          if (!(await ensureDirectory("view", path))) return;
-          if (isExperimentRun(path)) {
-            const experiment = await loadExperimentRun(path);
-            if (!experiment) {
-              process.stderr.write(`lux view: invalid experiment at ${path}\n`);
-              failWith(1);
-              return;
-            }
-            process.stdout.write(
-              options.json
-                ? `${JSON.stringify({ experimentDir: path, ...experiment }, null, 2)}\n`
-                : `${formatExperimentRun(path, experiment)}\n`,
-            );
-            return;
-          }
-          const suite = await loadSuiteRun(path);
-          if (suite) {
-            process.stdout.write(
-              options.json
-                ? `${JSON.stringify({ suiteDir: path, ...suite }, null, 2)}\n`
-                : `${formatSuiteRunDetail(path, suite)}\n`,
-            );
-            return;
-          }
-          const detail = await loadRunDetail(path);
-          if (options.json) {
-            process.stdout.write(`${JSON.stringify(detail, null, 2)}\n`);
-          } else {
-            process.stdout.write(`${formatRunDetail(detail)}\n`);
-          }
-          return;
-        }
-
-        const cwd = process.cwd();
-        const config = await loadLuxConfig(cwd);
-        const runsRoot = options.runsRoot ? resolve(options.runsRoot) : config.runsRoot;
-        if (options.experiments) {
-          const experiments = await listExperimentRuns(runsRoot);
-          process.stdout.write(
-            options.json
-              ? `${JSON.stringify(experiments, null, 2)}\n`
-              : `experiments at ${runsRoot}:\n${formatExperimentRunsTable(experiments)}\n`,
-          );
-          return;
-        }
-        if (options.suites) {
-          const suites = await listSuiteRuns(runsRoot);
-          process.stdout.write(
-            options.json
-              ? `${JSON.stringify(suites, null, 2)}\n`
-              : `suite runs at ${runsRoot}:\n${formatSuiteRunsTable(suites)}\n`,
-          );
-          return;
-        }
-        const all = await listRuns(runsRoot);
-        const runs = options.case ? all.filter((r) => r.caseSlug === options.case) : all;
-        if (options.json) {
-          process.stdout.write(`${JSON.stringify(runs, null, 2)}\n`);
-        } else {
-          if (!options.case) {
-            process.stdout.write(`suite runs at ${runsRoot}:\n`);
-            process.stdout.write(`${formatSuiteRunsTable(await listSuiteRuns(runsRoot))}\n\n`);
-          }
-          process.stdout.write(`runs at ${runsRoot}:\n`);
-          process.stdout.write(`${formatRunsTable(runs)}\n`);
-        }
-      },
-    );
-
-  program
-    .command("compare <runA> <runB>")
-    .description("diff two run or suite directories")
-    .option(
-      "--pass-rate-tolerance <fraction>",
-      "allowed per-case pass-rate decrease for suite comparison",
-      parseUnitInterval,
-    )
-    .action(async (runA: string, runB: string, options: { passRateTolerance?: number }) => {
-      const pathA = resolve(runA);
-      const pathB = resolve(runB);
-      // Validate both up front so one typo'd path in CI cannot read as
-      // "0 regressed"; report every bad path, not just the first.
-      const okA = await ensureDirectory("compare", pathA);
-      const okB = await ensureDirectory("compare", pathB);
-      if (!okA || !okB) return;
-      const [suiteA, suiteB] = await Promise.all([loadSuiteRun(pathA), loadSuiteRun(pathB)]);
-      if (Boolean(suiteA) !== Boolean(suiteB)) {
-        process.stderr.write("lux compare: both paths must be runs or both must be suites\n");
-        failWith(1);
-        return;
-      }
-      if (suiteA && suiteB) {
-        const comparison = await compareSuiteRuns(pathA, pathB, {
-          ...(options.passRateTolerance !== undefined
-            ? { passRateTolerance: options.passRateTolerance }
-            : {}),
-        });
-        process.stdout.write(`${formatSuiteComparisonText(comparison)}\n`);
-        if (hasSuiteRegression(comparison)) failWith(1);
-        return;
-      }
-      const cmp = await compareRuns({ pathA, pathB });
-      process.stdout.write(`${formatComparisonText(cmp)}\n`);
-      if (hasRegression(cmp)) failWith(1);
+  }
+  if (experiment.status !== "passed") {
+    throw new Errors.IncurError({
+      code: "EXPERIMENT_NOT_PROMOTED",
+      message: `Experiment status is '${experiment.status}', not 'passed'.`,
+      exitCode: 1,
+    });
+  }
+  const championId = experiment.championCandidateId;
+  if (!championId) {
+    throw new Errors.IncurError({
+      code: "CHAMPION_NOT_FOUND",
+      message: "The passed experiment has no promoted champion.",
+      exitCode: 1,
+    });
+  }
+  if (candidateId && candidateId !== championId) {
+    throw new Errors.IncurError({
+      code: "CANDIDATE_NOT_PROMOTED",
+      message: `Candidate '${candidateId}' is not the promoted champion '${championId}'.`,
+      exitCode: 1,
+    });
+  }
+  let candidate: Candidate | undefined;
+  try {
+    const candidates = await verifyExperimentRunIntegrity(directory, experiment);
+    verifyExperimentDecisionIntegrity(experiment);
+    candidate = candidates.find((entry) => entry.id === championId);
+    if (!candidate) throw new Error(`Champion candidate '${championId}' not found.`);
+  } catch (error) {
+    throw new Errors.IncurError({
+      code: "EXPERIMENT_INTEGRITY_FAILED",
+      message: error instanceof Error ? error.message : String(error),
+      exitCode: 1,
     });
+  }
+  await applyCandidate(experiment.subjectRoot, experiment.campaignSnapshot.subject, candidate);
+  return {
+    candidateId: candidate.id,
+    subjectRoot: experiment.subjectRoot,
+    sourceBytes: candidate.sourceBytes,
+  };
 };
 
-export const buildProgram = (): Command => {
-  const program = new Command();
-  program
-    .name("lux")
-    .description(
-      "Coding-agent evaluation and improvement with deterministic assertions and human-in-the-loop runs",
-    )
-    .version(PKG_VERSION)
-    .addHelpText(
-      "after",
-      "\nAgent skills:\n  $ lux skills list\n  $ lux skills install codex --project\n  $ lux skills install claude --project",
-    );
-  registerProjectCommands(program);
-  registerExperimentCommands(program);
-  registerCandidateCommands(program);
-  registerReportingCommands(program);
-  program.action(() => program.outputHelp());
-  return program;
+const reportCommand = async (
+  experimentDir: string,
+  options: { out?: string | undefined; force?: boolean | undefined },
+) => {
+  const directory = resolve(experimentDir);
+  if (!(await ensureDirectory("report", directory))) failWith(1);
+  const snapshot = await loadVerifiedExperimentDecisionSnapshot(directory);
+  const { experiment, report } = snapshot;
+  if (!options.out) return report;
+  const out = resolve(options.out);
+  const outputStats = await lstat(out).catch((error: NodeJS.ErrnoException) => {
+    if (error.code === "ENOENT") return null;
+    throw error;
+  });
+  if (outputStats && (outputStats.isSymbolicLink() || !outputStats.isFile())) {
+    throw new Error("An existing --out must be a regular, non-symlink file.");
+  }
+  const [physicalDirectory, physicalSubjectRoot, physicalOut] = await Promise.all([
+    realpath(directory),
+    realpath(experiment.subjectRoot),
+    physicalDestinationPath(out),
+  ]);
+  assertReportDestination(
+    physicalOut,
+    physicalDirectory,
+    physicalSubjectRoot,
+    experiment.campaignSnapshot.subject.exclude,
+  );
+  if (outputStats && !options.force) {
+    throw new Error(`Output already exists (use --force): ${out}`);
+  }
+  await mkdir(dirname(out), { recursive: true });
+  const physicalParent = await realpath(dirname(out));
+  const finalPhysicalOut = resolve(physicalParent, basename(out));
+  assertReportDestination(
+    finalPhysicalOut,
+    physicalDirectory,
+    physicalSubjectRoot,
+    experiment.campaignSnapshot.subject.exclude,
+  );
+  await writeReportSafely(
+    finalPhysicalOut,
+    `${formatExperimentDecisionMarkdown(report)}\n`,
+    options.force ?? false,
+  );
+  return { path: out, report };
 };
 
-const caseIdOf = (casePath: string): string => basename(casePath, extname(casePath));
+const compareCommand = async (
+  runA: string,
+  runB: string,
+  options: { passRateTolerance?: number | undefined },
+) => {
+  const pathA = resolve(runA);
+  const pathB = resolve(runB);
+  const invalid = (
+    await Promise.all(
+      [pathA, pathB].map(async (path) =>
+        existsSync(path) && (await stat(path)).isDirectory() ? undefined : path,
+      ),
+    )
+  ).filter((path): path is string => path !== undefined);
+  if (invalid.length > 0) {
+    throw new Errors.IncurError({
+      code: "RUN_DIRECTORY_NOT_FOUND",
+      message: invalid.map((path) => `${path} is not a directory`).join("; "),
+      exitCode: 1,
+    });
+  }
+  const [suiteA, suiteB] = await Promise.all([loadSuiteRun(pathA), loadSuiteRun(pathB)]);
+  if (Boolean(suiteA) !== Boolean(suiteB)) {
+    throw new Errors.IncurError({
+      code: "INCOMPATIBLE_RUN_TYPES",
+      message: "Both paths must be runs or both must be suites.",
+      exitCode: 1,
+    });
+  }
+  if (suiteA && suiteB) {
+    const comparison = await compareSuiteRuns(pathA, pathB, {
+      ...(options.passRateTolerance !== undefined
+        ? { passRateTolerance: options.passRateTolerance }
+        : {}),
+    });
+    if (hasSuiteRegression(comparison)) {
+      throw new Errors.IncurError({
+        code: "REGRESSION_DETECTED",
+        message: formatSuiteComparisonText(comparison),
+        exitCode: 1,
+      });
+    }
+    return comparison;
+  }
+  const comparison = await compareRuns({ pathA, pathB });
+  if (hasRegression(comparison)) {
+    throw new Errors.IncurError({
+      code: "REGRESSION_DETECTED",
+      message: formatComparisonText(comparison),
+      exitCode: 1,
+    });
+  }
+  return comparison;
+};
 
 const SIGINT_EXIT_CODE = 130;
 
@@ -711,98 +722,22 @@ const isFailure = (outcome: { exitReason: string; casePassed: boolean | null }):
 
 const firstLine = (message: string): string => (message.split("\n")[0] ?? message).trim();
 
-const suiteTitle = (
-  patterns: string[],
-  tags: string[],
-  explicit: SuiteItem[],
-  loop: number,
-): string => {
-  const parts = [...patterns, ...explicit.map((e) => e.id), ...tags.map((t) => `#${t}`)];
-  const filter = parts.length > 0 ? ` ${parts.join(" ")}` : "";
-  const repeat = loop > 1 ? ` ×${loop}` : "";
-  return `run${filter}${repeat}`;
-};
-
-type FilterPartition = { explicit: SuiteItem[]; patterns: string[] };
-
-// A filter that names an existing `.ts` file runs that file directly — it may
-// sit outside the discovery root. Everything else is a substring pattern — but
-// a `.ts` filter that resolves to nothing is almost always a typo'd path, so
-// the silent downgrade to a pattern gets called out.
-const partitionFilters = (filters: string[], cwd: string): FilterPartition => {
-  const explicit: SuiteItem[] = [];
-  const patterns: string[] = [];
-  for (const f of filters) {
-    const abs = resolve(cwd, f);
-    if (f.endsWith(".ts") && existsSync(abs)) {
-      explicit.push({ path: abs, id: caseIdOf(f) });
-    } else {
-      if (f.endsWith(".ts")) {
-        process.stderr.write(`lux: ${abs} does not exist; treating "${f}" as a pattern\n`);
-      }
-      patterns.push(f);
-    }
-  }
-  return { explicit, patterns };
-};
-
-// The selected set — substring/tag matches plus explicitly named files — is
-// the same whether it is run, `--list`ed, or `--list-tags`ed. No filters at
-// all means "everything discovered"; any filter narrows.
-const selectSuite = (
-  cases: DiscoveredCase[],
-  { explicit, patterns }: FilterPartition,
-  tags: string[],
-): DiscoveredCase[] => {
-  const hasFilter = explicit.length > 0 || patterns.length > 0 || tags.length > 0;
-  const byPath = new Map<string, DiscoveredCase>();
-  if (!hasFilter) {
-    for (const c of cases) byPath.set(c.path, c);
-  } else {
-    if (patterns.length > 0 || tags.length > 0) {
-      for (const c of selectCases(cases, { patterns, tags })) byPath.set(c.path, c);
-    }
-    for (const e of explicit) {
-      const discovered = cases.find((c) => c.path === e.path);
-      // Discovery knows the case's declared id; the filename-derived one is
-      // only a fallback for files discovery never saw.
-      byPath.set(e.path, {
-        path: e.path,
-        id: discovered?.id ?? e.id,
-        relPath: discovered?.relPath ?? e.id,
-        tags: discovered?.tags ?? [],
-      });
-    }
-  }
-  return [...byPath.values()].sort((a, b) => a.id.localeCompare(b.id));
-};
-
-// `--loop` repeats each eval before moving to the next; each repeat gets a
-// `#n` display suffix so the reporter keeps them apart, while `caseId` keeps
-// every repeat filed under one run history.
-const expandLoop = (base: SuiteItem[], loop: number): SuiteItem[] =>
-  loop > 1
-    ? base.flatMap((b) =>
-        Array.from({ length: loop }, (_, i) => ({
-          path: b.path,
-          id: `${b.id} #${i + 1}`,
-          caseId: b.caseId ?? b.id,
-        })),
-      )
-    : base;
-
 const experimentCommand = async (
   mode: "experiment" | "optimize",
   campaignPath: string,
-  options: { runsRoot?: string; resume?: string; json?: boolean; plan?: boolean },
-): Promise<void> => {
+  options: {
+    runsRoot?: string | undefined;
+    resume?: string | undefined;
+    plan?: boolean | undefined;
+  },
+) => {
   const cwd = process.cwd();
   const config = await loadLuxConfig(cwd);
   const campaign = await resolveExperimentCampaignPath(config.rootDir, campaignPath);
   const loaded = await loadExperimentCampaign(campaign);
   experimentCampaignMode(campaign, loaded.campaign, mode);
   const registry = buildRegistryFromConfig(config);
-  const controller = abortOnInterrupt();
+  using controller = abortOnInterrupt();
   const execution = {
     config,
     registry,
@@ -813,64 +748,55 @@ const experimentCommand = async (
   };
   if (options.plan) {
     const plan = await planExperimentCampaign(execution, mode);
-    let output: string;
-    if (options.json) {
-      output = `${JSON.stringify(plan, null, 2)}\n`;
-    } else {
-      const projectedStatus = plan.fitsProjectedCallBudgets ? "yes" : "no";
-      output = [
-        `campaign:      ${plan.campaignId}`,
-        `mode:          ${plan.mode}`,
-        `evaluation:    ${plan.evaluationKind}`,
-        `profiles:      ${plan.profiles.join(", ")}`,
-        `splits:        ${plan.splits.map((split) => `${split.split}=${split.examples}`).join(", ")}`,
-        `components:    ${plan.mutableComponents.length}`,
-        `candidates:    ${plan.projected.candidates}`,
-        `metric calls:  ${plan.projected.metricCalls}/${plan.budget.maxMetricCalls}`,
-        `judge calls:   ${plan.projected.judgeCalls}/${plan.budget.maxJudgeCalls}`,
-        `proposal calls:${plan.projected.proposalCalls}/${plan.budget.maxProposalCalls}`,
-        `unpriced calls: observed at runtime (cap ${plan.budget.maxUnpricedModelCalls})`,
-        `projected work within budget: ${projectedStatus}`,
-        ...plan.warnings.map((warning) => `warning:       ${warning}`),
-        "",
-      ].join("\n");
+    if (!plan.fitsProjectedCallBudgets) {
+      throw new Errors.IncurError({
+        code: "EXPERIMENT_BUDGET_EXCEEDED",
+        message: "The projected experiment work exceeds its configured call budgets.",
+        exitCode: 1,
+      });
     }
-    process.stdout.write(output);
-    if (!plan.fitsProjectedCallBudgets) failWith(1);
-    return;
+    return plan;
   }
   const result =
     mode === "optimize"
       ? await optimizeExperimentCampaign(execution)
       : await runExperimentCampaign(execution);
-  process.stdout.write(
-    options.json
-      ? `${JSON.stringify({ experimentDir: result.experimentDir, ...result.manifest }, null, 2)}\n`
-      : `${formatExperimentRun(result.experimentDir, result.manifest)}\n`,
-  );
   if (controller.signal.aborted) failWith(SIGINT_EXIT_CODE);
-  else if (result.manifest.status !== "passed") failWith(1);
+  if (result.manifest.status !== "passed") {
+    throw new Errors.IncurError({
+      code: "EXPERIMENT_FAILED",
+      message: `Experiment ${result.manifest.experimentId} finished with status '${result.manifest.status}'.`,
+      exitCode: 1,
+    });
+  }
+  return { experimentDir: result.experimentDir, ...result.manifest };
 };
 
-const runCommand = async (filters: string[], options: RunOptions): Promise<void> => {
+/** A profile pins one driver, so callers pass the project matrix only without one. */
+const runCommand = async (filters: string[], options: RunOptions, human = false) => {
   if (options.interactive && options.claudeAnswerer) {
-    throw new Error("lux run: --interactive and --claude-answerer are mutually exclusive");
+    throw new Errors.ParseError({
+      message: "--interactive and --claude-answerer are mutually exclusive",
+    });
   }
   const cwd = process.cwd();
   const config = await loadLuxConfig(cwd);
-  const profile = options.profile ? config.profiles?.[options.profile] : undefined;
-  if (options.profile && !profile) {
+  const authoredProfile = options.profile ? config.profiles?.[options.profile] : undefined;
+  if (options.profile && !authoredProfile) {
     const available = Object.keys(config.profiles ?? {}).sort();
-    process.stderr.write(
-      `lux: unknown profile '${options.profile}'${available.length > 0 ? `; available: ${available.join(", ")}` : ""}\n`,
-    );
-    failWith(1);
-    return;
+    throw new Errors.IncurError({
+      code: "PROFILE_NOT_FOUND",
+      message: `Unknown profile '${options.profile}'${available.length > 0 ? `; available: ${available.join(", ")}` : ""}.`,
+      exitCode: 1,
+    });
   }
-  const harness = profile?.harness ?? config.harness;
-  const registry = buildRegistryFromConfig(config).withAssertion(createRubricAssertion(harness));
+  const profile = resolveProfile(config, authoredProfile);
+  const harness = profile.harness;
+  const registry = registryWithHarness(buildRegistryFromConfig(config), config, harness);
   const defaultDriver = profile?.driver ?? config.defaultDriver;
   const orchestrator = new Orchestrator({
+    driverDefaults: config.driverDefaults,
+    environment: profile?.environment ?? config.environment,
     registry,
     defaultAnswerer: profile?.answerer ?? config.defaultAnswerer,
     ...(defaultDriver ? { defaultDriver } : {}),
@@ -888,23 +814,26 @@ const runCommand = async (filters: string[], options: RunOptions): Promise<void>
   // broken config, not an empty suite — never mask it behind "no cases
   // matched" or a lucky explicit-file run.
   if (config.casesRoot && !existsSync(config.casesRoot)) {
-    process.stderr.write(`lux: casesRoot does not exist: ${config.casesRoot}\n`);
-    failWith(1);
-    return;
+    throw new Errors.IncurError({
+      code: "CASES_ROOT_NOT_FOUND",
+      message: `casesRoot does not exist: ${config.casesRoot}`,
+      exitCode: 1,
+    });
   }
 
   const { explicit, patterns } = partitionFilters(filters, cwd);
-  const tags = options.tag ?? [];
+  const tags = normalizeTags(options.tag ?? []);
   const hasFilter = explicit.length > 0 || patterns.length > 0 || tags.length > 0;
 
   // Explicitly named files need no discovery, so they run even without a
   // project; anything else would need a tree walk with nowhere safe to walk.
   if (config.configPath === null && !existsSync(root) && explicit.length === 0) {
-    process.stderr.write(
-      `lux: no lux.config.ts or cases/ directory in ${cwd} — run \`lux init\` to scaffold a project, or name a case file directly\n`,
-    );
-    failWith(1);
-    return;
+    throw new Errors.IncurError({
+      code: "PROJECT_NOT_INITIALIZED",
+      message: `No lux.config.ts or cases/ directory in ${cwd}.`,
+      hint: "Run `lux init` or name a case file directly.",
+      exitCode: 1,
+    });
   }
 
   const { cases, problems } = existsSync(root)
@@ -919,35 +848,60 @@ const runCommand = async (filters: string[], options: RunOptions): Promise<void>
   // An empty selection fails every mode the same way — running, `--list`, and
   // `--list-tags` all exit 1, so a typo'd filter can never read as a clean pass.
   if (selected.length === 0) {
-    process.stderr.write(
-      hasFilter
-        ? "lux: no cases matched the given filters\n"
-        : `lux: no cases discovered under ${root}\n`,
-    );
-    failWith(1);
-    return;
+    throw new Errors.IncurError({
+      code: "NO_CASES_MATCHED",
+      message: hasFilter
+        ? "No cases matched the given filters."
+        : `No cases were discovered under ${root}.`,
+      exitCode: 1,
+    });
   }
 
   // List the tag vocabulary of the selected cases — "which --tag values exist?".
   if (options.listTags) {
     const counts = new Map<string, number>();
     for (const c of selected) for (const t of c.tags) counts.set(t, (counts.get(t) ?? 0) + 1);
-    for (const [tag, n] of [...counts].sort((a, b) => a[0].localeCompare(b[0]))) {
-      process.stdout.write(`${tag}\t${n}\n`);
-    }
-    process.stderr.write(
-      `lux: ${counts.size} tag${counts.size === 1 ? "" : "s"} across ${selected.length} cases\n`,
-    );
-    return;
+    const result = {
+      caseCount: selected.length,
+      tags: [...counts]
+        .sort((a, b) => a[0].localeCompare(b[0]))
+        .map(([tag, count]) => ({ tag, count })),
+    };
+    if (human)
+      process.stdout.write(
+        `${result.caseCount} cases\n${result.tags.map(({ tag, count }) => `  ${tag}: ${count}`).join("\n")}\n`,
+      );
+    return result;
   }
 
-  const base: SuiteItem[] = selected.map((c) => ({ path: c.path, id: c.id }));
+  const focus: HarnessFocus = {
+    driver: options.driver,
+    model: options.model,
+    effort: options.effort,
+  };
+  const base = await planSuiteItems(selected, {
+    driverDefaults: config.driverDefaults,
+    matrix: authoredProfile ? undefined : config.matrix,
+    defaultDriver,
+    registry,
+    focus,
+  });
 
   // Collect-only: answer "what would run?" without spawning a single agent.
   if (options.list) {
-    for (const item of base) process.stdout.write(`${item.id}\n`);
-    process.stderr.write(`lux: ${base.length} case${base.length === 1 ? "" : "s"} matched\n`);
-    return;
+    if (human)
+      process.stdout.write(
+        `${base.length} cases\n${base.map(({ id, path }) => `  ${id}  ${path}`).join("\n")}\n`,
+      );
+    return {
+      caseCount: base.length,
+      cases: base.map(({ path, id, caseId, permutationId }) => ({
+        path,
+        id,
+        ...(caseId !== undefined ? { caseId } : {}),
+        ...(permutationId !== undefined ? { permutationId } : {}),
+      })),
+    };
   }
 
   const loop = options.loop ?? 1;
@@ -956,10 +910,10 @@ const runCommand = async (filters: string[], options: RunOptions): Promise<void>
   const { concurrency, note } = resolveConcurrency(options, config.concurrency);
   if (note) process.stderr.write(note);
 
-  const reporter = new ConsoleReporter({ verbose: options.verbose ?? false });
-  const controller = abortOnInterrupt();
-  reporter.begin({
-    title: suiteTitle(patterns, tags, explicit, loop),
+  const reporter = human ? new ConsoleReporter({ verbose: options.verbose ?? false }) : undefined;
+  using controller = abortOnInterrupt();
+  reporter?.begin({
+    title: suiteTitle(patterns, tags, explicit, loop, focus),
     caseIds: items.map((i) => i.id),
     runsRoot,
     loop,
@@ -972,28 +926,41 @@ const runCommand = async (filters: string[], options: RunOptions): Promise<void>
       fixturesRoot: config.fixturesRoot,
       answererOverride: resolveAnswererOverride(options),
       skipJudge: options.skipJudge,
-      observer: reporter,
+      ...(reporter ? { observer: reporter } : {}),
       abortSignal: controller.signal,
       concurrency,
       subjectRoot,
       profileId: options.profile,
     });
     results = suite.results;
-    reporter.setSuiteDir(suite.suiteDir);
+    reporter?.setSuiteDir(suite.suiteDir);
+    if (controller.signal.aborted) failWith(SIGINT_EXIT_CODE);
+    if (results.some(isFailure)) {
+      throw new Errors.IncurError({
+        code: "SUITE_FAILED",
+        message: `${results.filter(isFailure).length} of ${results.length} case attempts failed. Suite: ${suite.suiteDir}`,
+        exitCode: 1,
+      });
+    }
+    return {
+      suiteDir: suite.suiteDir,
+      results: results.map((result) => ({
+        caseId: result.caseId,
+        runDir: result.runDir ?? null,
+        exitReason: result.exitReason,
+        casePassed: result.casePassed,
+        caseScore: result.result?.grading?.caseScore ?? null,
+        assertionsPassed:
+          result.result?.grading?.assertions.filter((assertion) => assertion.passed).length ?? 0,
+        assertionsTotal:
+          result.result?.grading?.assertions.filter((assertion) => assertion.passed !== null)
+            .length ?? 0,
+      })),
+    };
   } finally {
-    reporter.finish();
+    reporter?.finish();
   }
-
-  if (controller.signal.aborted) failWith(SIGINT_EXIT_CODE);
-  else if (results.some(isFailure)) failWith(1);
 };
 
 // Internal exports for testing
-export {
-  expandLoop,
-  isFailure,
-  parsePositiveInt,
-  partitionFilters,
-  resolveConcurrency,
-  selectSuite,
-};
+export { isFailure, resolveConcurrency };