@prismatic-io/lux 0.0.1

package/src/cli/program.ts

@@ -0,0 +1,992 @@
+import { randomUUID } from "node:crypto";
+import { existsSync } from "node:fs";
+import { lstat, mkdir, realpath, rename, rm, stat, writeFile } from "node:fs/promises";
+import { createRequire } from "node:module";
+import { basename, dirname, extname, join, relative, resolve, sep } from "node:path";
+import { Command, InvalidArgumentError } from "commander";
+import { createRubricAssertion } from "../assertions/rubric/index.js";
+import {
+  type Candidate,
+  GraderAnnotationSchema,
+  matchesAnyGlob,
+  valueHash,
+} from "../core/index.js";
+import {
+  applyCandidate,
+  buildRegistryFromConfig,
+  compareRuns,
+  compareSuiteRuns,
+  type DiscoveredCase,
+  discoverCases,
+  doctorProject,
+  experimentCampaignMode,
+  formatComparisonText,
+  formatDoctorReport,
+  formatExperimentDecisionMarkdown,
+  formatExperimentRun,
+  formatExperimentRunsTable,
+  formatRunDetail,
+  formatRunsTable,
+  formatSuiteComparisonText,
+  formatSuiteRunDetail,
+  formatSuiteRunsTable,
+  hasRegression,
+  hasSuiteRegression,
+  isExperimentRun,
+  listExperimentRuns,
+  listRuns,
+  listSuiteRuns,
+  loadEvalCase,
+  loadExperimentCampaign,
+  loadExperimentRun,
+  loadFrozenRegradableRun,
+  loadLuxConfig,
+  loadRubricDefinition,
+  loadRunDetail,
+  loadSuiteRun,
+  loadVerifiedExperimentDecisionSnapshot,
+  Orchestrator,
+  optimizeExperimentCampaign,
+  planExperimentCampaign,
+  regradeRun,
+  releaseFrozenRegradableRun,
+  resolveExperimentCampaignPath,
+  runExperimentCampaign,
+  type SuiteCaseResult,
+  type SuiteItem,
+  selectCases,
+  upsertGraderAnnotation,
+  verifyExperimentDecisionIntegrity,
+  verifyExperimentRunIntegrity,
+} from "../orchestrator/index.js";
+import {
+  abortOnInterrupt,
+  ensureDirectory,
+  failWith,
+  relativePathWithin,
+} from "./command-runtime.js";
+import { formatInitReport, initProject } from "./init.js";
+import { ConsoleReporter } from "./render/reporter.js";
+import {
+  collectTags,
+  collectValues,
+  parseAnnotationAssertion,
+  parsePositiveInt,
+  parseUnitInterval,
+  type RunOptions,
+  resolveAnswererOverride,
+  resolveConcurrency,
+} from "./run-options.js";
+import {
+  type BundledSkillName,
+  formatSkillsInstall,
+  formatSkillsList,
+  installSkills,
+  type SkillsPlatform,
+} from "./skills.js";
+
+const requireFromHere = createRequire(import.meta.url);
+// From lib/cli/program.js (or src/cli/program.ts under test) to the package root.
+const PKG_VERSION = (requireFromHere("../../package.json") as { version: string }).version;
+
+const registerProjectCommands = (program: Command): void => {
+  program
+    .command("init")
+    .description("scaffold a new evals project in the current directory")
+    .option("--force", "overwrite existing files")
+    .option("--experiment", "also scaffold a prompt optimization campaign and split cases")
+    .action(async (options: { force?: boolean; experiment?: boolean }) => {
+      const cwd = process.cwd();
+      const result = await initProject({
+        cwd,
+        force: options.force ?? false,
+        experiment: options.experiment ?? false,
+      });
+      process.stdout.write(`${formatInitReport(result, cwd)}\n`);
+    });
+
+  const skills = program.command("skills").description("manage bundled agent skills");
+
+  skills
+    .command("list")
+    .description("list bundled skills and supported platforms")
+    .action(() => {
+      process.stdout.write(`${formatSkillsList()}\n`);
+    });
+
+  skills
+    .command("install")
+    .description("install the Lux eval-authoring skill for Claude Code or Codex")
+    .argument("<platform>", "claude, codex, or all")
+    .option("--project", "install into the current project instead of the user profile")
+    .option("--dir <path>", "install under an explicit skills directory")
+    .option("--name <name>", "install one bundled skill")
+    .action(
+      async (
+        platform: string,
+        options: {
+          project?: boolean;
+          dir?: string;
+          name?: string;
+        },
+      ) => {
+        if (!["claude", "claude-code", "codex", "all"].includes(platform)) {
+          throw new InvalidArgumentError("platform must be claude, codex, or all");
+        }
+        if (options.name && !["lux", "lux-answerer"].includes(options.name)) {
+          throw new InvalidArgumentError("skill name must be lux or lux-answerer");
+        }
+        const cwd = process.cwd();
+        const installed = await installSkills({
+          cwd,
+          platform: platform as SkillsPlatform,
+          project: options.project ?? false,
+          ...(options.dir ? { dir: options.dir } : {}),
+          ...(options.name ? { name: options.name as BundledSkillName } : {}),
+        });
+        process.stdout.write(`${formatSkillsInstall(installed, cwd)}\n`);
+      },
+    );
+};
+
+const registerExperimentCommands = (program: Command): void => {
+  program
+    .command("doctor [campaigns...]")
+    .description("preflight project configuration, CLIs, cases, campaigns, and budgets")
+    .option("--json", "emit machine-readable JSON")
+    .action(async (campaigns: string[], options: { json?: boolean }) => {
+      const report = await doctorProject({ cwd: process.cwd(), campaigns });
+      process.stdout.write(
+        options.json ? `${JSON.stringify(report, null, 2)}\n` : `${formatDoctorReport(report)}\n`,
+      );
+      if (!report.ok) failWith(1);
+    });
+
+  program
+    .command("run [filters...]", { isDefault: true })
+    .description("discover and run cases; filter by name/path substring or --tag")
+    .option(
+      "-t, --tag <tag>",
+      "only cases whose meta.tags include this tag (repeatable, comma-separated)",
+      collectTags,
+      [],
+    )
+    .option("--loop <n>", "run each matched eval n times", parsePositiveInt)
+    .option(
+      "-j, --concurrency <n>",
+      "run up to n cases at once (overrides lux.config.ts)",
+      parsePositiveInt,
+    )
+    .option("--list", "print the matched cases and exit without running them")
+    .option("--list-tags", "print the tags across the matched cases and exit")
+    .option("-i, --interactive", "answer agent questions yourself in this terminal (HITL)")
+    .option(
+      "--claude-answerer",
+      "delegate questions through the Claude plugin's lux-answerer skill",
+    )
+    .option("--runs-root <path>", "run-dir root (overrides lux.config.ts)")
+    .option("--profile <id>", "named driver/answerer profile from lux.config.ts")
+    .option("--subject-root <path>", "root bound into subjectPath() values")
+    .option("-v, --verbose", "add phase timings and the run dir to every case")
+    .action(async (filters: string[], options: RunOptions) => {
+      await runCommand(filters, options);
+    });
+
+  program
+    .command("experiment <campaign>")
+    .description("evaluate controlled source variants across cases and model profiles")
+    .option("--runs-root <path>", "run-dir root (overrides lux.config.ts)")
+    .option("--resume <experiment-dir>", "resume an interrupted experiment ledger")
+    .option("--plan", "validate and estimate the campaign without calling models")
+    .option("--json", "emit machine-readable JSON")
+    .action(
+      async (
+        campaign: string,
+        options: { runsRoot?: string; resume?: string; json?: boolean; plan?: boolean },
+      ) => {
+        await experimentCommand("experiment", campaign, options);
+      },
+    );
+};
+
+const registerCandidateCommands = (program: Command): void => {
+  program
+    .command("grade <runDir>")
+    .description("re-run current graders against a persisted run without rerunning the agent")
+    .option("--case <path>", "grade with a current case definition instead of frozen case.json")
+    .option("--json", "emit machine-readable JSON")
+    .action(async (runDir: string, options: { case?: string; json?: boolean }) => {
+      const directory = resolve(runDir);
+      if (!(await ensureDirectory("grade", directory))) return;
+      const config = await loadLuxConfig(process.cwd());
+      const controller = abortOnInterrupt();
+      const result = await regradeRun({
+        runDir: directory,
+        registry: buildRegistryFromConfig(config),
+        graderContext: config.harness ?? null,
+        ...(options.case ? { case: await loadEvalCase(resolve(options.case)) } : {}),
+        abortSignal: controller.signal,
+      });
+      const report = result.snapshot.report;
+      let gradingStatus = "ungraded";
+      if (report.casePassed === true) gradingStatus = "PASS";
+      if (report.casePassed === false) gradingStatus = "FAIL";
+      let output: string;
+      if (options.json) {
+        output = `${JSON.stringify({ gradingPath: result.path, ...result.snapshot }, null, 2)}\n`;
+      } else {
+        const score = report.caseScore === null ? "—" : `${(report.caseScore * 100).toFixed(0)}%`;
+        output = [
+          `grading: ${gradingStatus}`,
+          `score:   ${score}`,
+          `checks:  ${report.assertions.filter((assertion) => assertion.passed).length}/${report.assertions.length}`,
+          `saved:   ${result.path}`,
+          "",
+        ].join("\n");
+      }
+      process.stdout.write(output);
+      if (controller.signal.aborted) failWith(SIGINT_EXIT_CODE);
+      else if (report.casePassed !== true) failWith(1);
+    });
+
+  program
+    .command("annotate <runDir>")
+    .description("add or update a human grader-alignment label for a persisted run")
+    .requiredOption("--out <path>", "JSON annotation dataset to create or update")
+    .requiredOption("--rubric <path>", "candidate-relative declarative .json rubric path")
+    .option("--id <id>", "stable annotation id")
+    .option("--label <pass|fail>", "expected overall case verdict")
+    .option("--score <n>", "expected overall case score from 0 to 1", parseUnitInterval)
+    .option(
+      "--assertion <id=pass|fail[:score]>",
+      "expected stable assertion verdict and optional score (repeatable)",
+      collectValues,
+      [],
+    )
+    .option("--split <train|validation|test>", "dataset split, stored as a tag")
+    .option("--tag <tag>", "extra selection tag (repeatable, comma-separated)", collectTags, [])
+    .option("--feedback <text>", "human rationale shown to the rubric optimizer on mismatch")
+    .option("--annotator <id>", "person or system that authored the label")
+    .option("--reviewer <id>", "independent reviewer of the label")
+    .option("--source <human|synthetic>", "label provenance", "human")
+    .action(
+      async (
+        runDir: string,
+        options: {
+          out: string;
+          rubric: string;
+          id?: string;
+          label?: string;
+          score?: number;
+          assertion: string[];
+          split?: string;
+          tag: string[];
+          feedback?: string;
+          annotator?: string;
+          reviewer?: string;
+          source: string;
+        },
+      ) => {
+        const directory = resolve(runDir);
+        if (!(await ensureDirectory("annotate", directory))) return;
+        if (options.label && options.label !== "pass" && options.label !== "fail") {
+          throw new InvalidArgumentError("--label must be pass or fail");
+        }
+        if (options.split && !["train", "validation", "test"].includes(options.split)) {
+          throw new InvalidArgumentError("--split must be train, validation, or test");
+        }
+        if (options.source !== "human" && options.source !== "synthetic") {
+          throw new InvalidArgumentError("--source must be human or synthetic");
+        }
+        const parsedAssertions = options.assertion.map(parseAnnotationAssertion);
+        const assertionIds = parsedAssertions.map(([id]) => id);
+        if (new Set(assertionIds).size !== assertionIds.length) {
+          throw new InvalidArgumentError("--assertion ids must not be repeated");
+        }
+        const assertions = Object.fromEntries(parsedAssertions);
+        if (
+          options.label === undefined &&
+          options.score === undefined &&
+          Object.keys(assertions).length === 0
+        ) {
+          throw new Error("lux annotate: provide --label, --score, or at least one --assertion");
+        }
+        const out = resolve(options.out);
+        const run = await loadFrozenRegradableRun(directory);
+        try {
+          const rubric = await loadRubricDefinition(
+            await resolveAuthoredFile(options.rubric, "rubric"),
+          );
+          if (rubric.id !== run.case.id) {
+            throw new Error(
+              `lux annotate: rubric id '${rubric.id}' does not match persisted case '${run.case.id}'`,
+            );
+          }
+          const rubricIds = new Set(rubric.assertions.map((assertion) => assertion.id));
+          const unknownIds = assertionIds.filter((assertionId) => !rubricIds.has(assertionId));
+          if (unknownIds.length > 0) {
+            throw new Error(`lux annotate: unknown rubric assertion ids: ${unknownIds.join(", ")}`);
+          }
+          const tags = [...(options.split ? [options.split] : []), ...options.tag].filter(
+            (tag, index, all) => all.indexOf(tag) === index,
+          );
+          const caseId = run.case.id;
+          const portableRunDir = (relative(dirname(out), directory) || ".").split(sep).join("/");
+          const id =
+            options.id ??
+            `${caseId}-${basename(directory)}-${valueHash(portableRunDir).slice(0, 10)}`.replace(
+              /[^a-zA-Z0-9_-]+/g,
+              "-",
+            );
+          const annotation = GraderAnnotationSchema.parse({
+            id,
+            runDir: portableRunDir,
+            rubricPath: options.rubric,
+            tags,
+            source: options.source,
+            ...(options.annotator ? { annotator: options.annotator } : {}),
+            labeledAt: new Date().toISOString(),
+            ...(options.reviewer ? { reviewer: options.reviewer } : {}),
+            expected: {
+              ...(options.label ? { casePassed: options.label === "pass" } : {}),
+              ...(options.score !== undefined ? { caseScore: options.score } : {}),
+              assertions,
+            },
+            ...(options.feedback ? { feedback: options.feedback } : {}),
+          });
+          const dataset = await upsertGraderAnnotation(out, annotation);
+          process.stdout.write(
+            `annotated ${annotation.id} in ${out} (${dataset.annotations.length} labels)\n`,
+          );
+        } finally {
+          await releaseFrozenRegradableRun(run);
+        }
+      },
+    );
+};
+
+const registerReportingCommands = (program: Command): void => {
+  program
+    .command("optimize <campaign>")
+    .description("evolve source candidates from grading feedback and promote on held-out cases")
+    .option("--runs-root <path>", "run-dir root (overrides lux.config.ts)")
+    .option("--resume <experiment-dir>", "resume an interrupted optimization ledger")
+    .option("--plan", "validate and estimate the campaign without calling models")
+    .option("--json", "emit machine-readable JSON")
+    .action(
+      async (
+        campaign: string,
+        options: { runsRoot?: string; resume?: string; json?: boolean; plan?: boolean },
+      ) => {
+        await experimentCommand("optimize", campaign, options);
+      },
+    );
+
+  program
+    .command("apply <experimentDir> [candidateId]")
+    .description("apply the promoted experiment champion after verifying the source base hash")
+    .action(async (experimentDir: string, candidateId?: string) => {
+      const directory = resolve(experimentDir);
+      const experiment = await loadExperimentRun(directory);
+      if (!experiment) {
+        process.stderr.write(`lux apply: experiment manifest not found at ${directory}\n`);
+        failWith(1);
+        return;
+      }
+      if (experiment.status !== "passed") {
+        process.stderr.write(
+          `lux apply: experiment status is '${experiment.status}', not 'passed'\n`,
+        );
+        failWith(1);
+        return;
+      }
+      const championId = experiment.championCandidateId;
+      if (!championId) {
+        process.stderr.write("lux apply: passed experiment has no promoted champion\n");
+        failWith(1);
+        return;
+      }
+      if (candidateId && candidateId !== championId) {
+        process.stderr.write(
+          `lux apply: candidate '${candidateId}' is not the promoted champion '${championId}'\n`,
+        );
+        failWith(1);
+        return;
+      }
+      let candidate: Candidate | undefined;
+      try {
+        const candidates = await verifyExperimentRunIntegrity(directory, experiment);
+        verifyExperimentDecisionIntegrity(experiment);
+        candidate = candidates.find((entry) => entry.id === championId);
+        if (!candidate) {
+          throw new Error(`champion candidate '${championId}' not found`);
+        }
+      } catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        process.stderr.write(`lux apply: experiment integrity check failed: ${message}\n`);
+        failWith(1);
+        return;
+      }
+      await applyCandidate(experiment.subjectRoot, experiment.campaignSnapshot.subject, candidate);
+      process.stdout.write(
+        `applied candidate ${candidate.id} to ${experiment.subjectRoot} (${candidate.sourceBytes} bytes)\n`,
+      );
+    });
+
+  program
+    .command("report <experimentDir>")
+    .description("render a PR-ready promotion report from a verified experiment record")
+    .option("--json", "emit the structured decision report as JSON")
+    .option("--out <path>", "write the report to a file instead of stdout")
+    .option("--force", "overwrite an existing output file")
+    .action(
+      async (experimentDir: string, options: { json?: boolean; out?: string; force?: boolean }) => {
+        const directory = resolve(experimentDir);
+        if (!(await ensureDirectory("report", directory))) return;
+        const snapshot = await loadVerifiedExperimentDecisionSnapshot(directory);
+        const { experiment, report } = snapshot;
+        const rendered = options.json
+          ? `${JSON.stringify(report, null, 2)}\n`
+          : `${formatExperimentDecisionMarkdown(report)}\n`;
+        if (!options.out) {
+          process.stdout.write(rendered);
+          return;
+        }
+        const out = resolve(options.out);
+        const outputStats = await lstat(out).catch((error: NodeJS.ErrnoException) => {
+          if (error.code === "ENOENT") return null;
+          throw error;
+        });
+        if (outputStats && (outputStats.isSymbolicLink() || !outputStats.isFile())) {
+          throw new Error("lux report: an existing --out must be a regular, non-symlink file");
+        }
+        const [physicalDirectory, physicalSubjectRoot, physicalOut] = await Promise.all([
+          realpath(directory),
+          realpath(experiment.subjectRoot),
+          physicalDestinationPath(out),
+        ]);
+        assertReportDestination(
+          physicalOut,
+          physicalDirectory,
+          physicalSubjectRoot,
+          experiment.campaignSnapshot.subject.exclude,
+        );
+        if (outputStats && !options.force) {
+          throw new Error(`lux report: output already exists (use --force): ${out}`);
+        }
+        await mkdir(dirname(out), { recursive: true });
+        const physicalParent = await realpath(dirname(out));
+        const finalPhysicalOut = resolve(physicalParent, basename(out));
+        assertReportDestination(
+          finalPhysicalOut,
+          physicalDirectory,
+          physicalSubjectRoot,
+          experiment.campaignSnapshot.subject.exclude,
+        );
+        await writeReportSafely(finalPhysicalOut, rendered, options.force ?? false);
+        process.stdout.write(`wrote experiment report to ${out}\n`);
+      },
+    );
+
+  program
+    .command("view [runDir]")
+    .description("list past runs, or show one run's detail when given a runDir")
+    .option("--runs-root <path>", "run-dir root (defaults to lux.config.ts)")
+    .option("--case <id>", "filter the run list to one case slug (list mode only)")
+    .option("--suites", "list logical suite runs instead of individual runs")
+    .option("--experiments", "list experiment runs instead of individual runs")
+    .option("--json", "emit machine-readable JSON")
+    .action(
+      async (
+        runDirArg: string | undefined,
+        options: {
+          runsRoot?: string;
+          case?: string;
+          suites?: boolean;
+          experiments?: boolean;
+          json?: boolean;
+        },
+      ) => {
+        if (runDirArg) {
+          const path = resolve(runDirArg);
+          if (!(await ensureDirectory("view", path))) return;
+          if (isExperimentRun(path)) {
+            const experiment = await loadExperimentRun(path);
+            if (!experiment) {
+              process.stderr.write(`lux view: invalid experiment at ${path}\n`);
+              failWith(1);
+              return;
+            }
+            process.stdout.write(
+              options.json
+                ? `${JSON.stringify({ experimentDir: path, ...experiment }, null, 2)}\n`
+                : `${formatExperimentRun(path, experiment)}\n`,
+            );
+            return;
+          }
+          const suite = await loadSuiteRun(path);
+          if (suite) {
+            process.stdout.write(
+              options.json
+                ? `${JSON.stringify({ suiteDir: path, ...suite }, null, 2)}\n`
+                : `${formatSuiteRunDetail(path, suite)}\n`,
+            );
+            return;
+          }
+          const detail = await loadRunDetail(path);
+          if (options.json) {
+            process.stdout.write(`${JSON.stringify(detail, null, 2)}\n`);
+          } else {
+            process.stdout.write(`${formatRunDetail(detail)}\n`);
+          }
+          return;
+        }
+
+        const cwd = process.cwd();
+        const config = await loadLuxConfig(cwd);
+        const runsRoot = options.runsRoot ? resolve(options.runsRoot) : config.runsRoot;
+        if (options.experiments) {
+          const experiments = await listExperimentRuns(runsRoot);
+          process.stdout.write(
+            options.json
+              ? `${JSON.stringify(experiments, null, 2)}\n`
+              : `experiments at ${runsRoot}:\n${formatExperimentRunsTable(experiments)}\n`,
+          );
+          return;
+        }
+        if (options.suites) {
+          const suites = await listSuiteRuns(runsRoot);
+          process.stdout.write(
+            options.json
+              ? `${JSON.stringify(suites, null, 2)}\n`
+              : `suite runs at ${runsRoot}:\n${formatSuiteRunsTable(suites)}\n`,
+          );
+          return;
+        }
+        const all = await listRuns(runsRoot);
+        const runs = options.case ? all.filter((r) => r.caseSlug === options.case) : all;
+        if (options.json) {
+          process.stdout.write(`${JSON.stringify(runs, null, 2)}\n`);
+        } else {
+          if (!options.case) {
+            process.stdout.write(`suite runs at ${runsRoot}:\n`);
+            process.stdout.write(`${formatSuiteRunsTable(await listSuiteRuns(runsRoot))}\n\n`);
+          }
+          process.stdout.write(`runs at ${runsRoot}:\n`);
+          process.stdout.write(`${formatRunsTable(runs)}\n`);
+        }
+      },
+    );
+
+  program
+    .command("compare <runA> <runB>")
+    .description("diff two run or suite directories")
+    .option(
+      "--pass-rate-tolerance <fraction>",
+      "allowed per-case pass-rate decrease for suite comparison",
+      parseUnitInterval,
+    )
+    .action(async (runA: string, runB: string, options: { passRateTolerance?: number }) => {
+      const pathA = resolve(runA);
+      const pathB = resolve(runB);
+      // Validate both up front so one typo'd path in CI cannot read as
+      // "0 regressed"; report every bad path, not just the first.
+      const okA = await ensureDirectory("compare", pathA);
+      const okB = await ensureDirectory("compare", pathB);
+      if (!okA || !okB) return;
+      const [suiteA, suiteB] = await Promise.all([loadSuiteRun(pathA), loadSuiteRun(pathB)]);
+      if (Boolean(suiteA) !== Boolean(suiteB)) {
+        process.stderr.write("lux compare: both paths must be runs or both must be suites\n");
+        failWith(1);
+        return;
+      }
+      if (suiteA && suiteB) {
+        const comparison = await compareSuiteRuns(pathA, pathB, {
+          ...(options.passRateTolerance !== undefined
+            ? { passRateTolerance: options.passRateTolerance }
+            : {}),
+        });
+        process.stdout.write(`${formatSuiteComparisonText(comparison)}\n`);
+        if (hasSuiteRegression(comparison)) failWith(1);
+        return;
+      }
+      const cmp = await compareRuns({ pathA, pathB });
+      process.stdout.write(`${formatComparisonText(cmp)}\n`);
+      if (hasRegression(cmp)) failWith(1);
+    });
+};
+
+export const buildProgram = (): Command => {
+  const program = new Command();
+  program
+    .name("lux")
+    .description(
+      "Coding-agent evaluation and improvement with deterministic assertions and human-in-the-loop runs",
+    )
+    .version(PKG_VERSION);
+  registerProjectCommands(program);
+  registerExperimentCommands(program);
+  registerCandidateCommands(program);
+  registerReportingCommands(program);
+  return program;
+};
+
+const caseIdOf = (casePath: string): string => basename(casePath, extname(casePath));
+
+const SIGINT_EXIT_CODE = 130;
+
+const physicalDestinationPath = async (path: string): Promise<string> => {
+  const suffix: string[] = [];
+  let ancestor = resolve(path);
+  while (!existsSync(ancestor)) {
+    const parent = dirname(ancestor);
+    if (parent === ancestor) break;
+    suffix.unshift(basename(ancestor));
+    ancestor = parent;
+  }
+  return resolve(await realpath(ancestor), ...suffix);
+};
+
+const assertReportDestination = (
+  out: string,
+  experimentDirectory: string,
+  subjectRoot: string,
+  excludedSubjectPaths: readonly string[],
+): void => {
+  if (relativePathWithin(experimentDirectory, out) !== null) {
+    throw new Error("lux report: --out must not write into the managed experiment directory");
+  }
+  const fromSubject = relativePathWithin(subjectRoot, out);
+  if (fromSubject !== null && !matchesAnyGlob(fromSubject, excludedSubjectPaths)) {
+    throw new Error(
+      "lux report: --out would change the experiment subject identity; " +
+        "write outside the subject or under an excluded runs/report directory",
+    );
+  }
+};
+
+const writeReportSafely = async (path: string, body: string, force: boolean): Promise<void> => {
+  if (!force) {
+    await writeFile(path, body, { flag: "wx" });
+    return;
+  }
+  const temporary = join(dirname(path), `.${basename(path)}.${randomUUID()}.tmp`);
+  try {
+    await writeFile(temporary, body, { flag: "wx" });
+    await rename(temporary, path);
+  } finally {
+    await rm(temporary, { force: true });
+  }
+};
+
+const resolveAuthoredFile = async (value: string, label: string): Promise<string> => {
+  if (value.startsWith("/")) {
+    if (existsSync(value) && (await stat(value)).isFile()) return resolve(value);
+    throw new Error(`lux: ${label} file does not exist: ${value}`);
+  }
+  let directory = process.cwd();
+  for (;;) {
+    const candidate = resolve(directory, value);
+    if (existsSync(candidate) && (await stat(candidate)).isFile()) return candidate;
+    const parent = dirname(directory);
+    if (parent === directory) break;
+    directory = parent;
+  }
+  throw new Error(`lux: ${label} file does not exist from ${process.cwd()}: ${value}`);
+};
+
+/**
+ * A case is only a success if it finished cleanly AND nothing graded it as
+ * failing. A run that timed out carrying no assertions grades as `null`, which
+ * must not be mistaken for a pass.
+ */
+const isFailure = (outcome: { exitReason: string; casePassed: boolean | null }): boolean =>
+  outcome.exitReason !== "done" || outcome.casePassed !== true;
+
+const firstLine = (message: string): string => (message.split("\n")[0] ?? message).trim();
+
+const suiteTitle = (
+  patterns: string[],
+  tags: string[],
+  explicit: SuiteItem[],
+  loop: number,
+): string => {
+  const parts = [...patterns, ...explicit.map((e) => e.id), ...tags.map((t) => `#${t}`)];
+  const filter = parts.length > 0 ? ` ${parts.join(" ")}` : "";
+  const repeat = loop > 1 ? ` ×${loop}` : "";
+  return `run${filter}${repeat}`;
+};
+
+type FilterPartition = { explicit: SuiteItem[]; patterns: string[] };
+
+// A filter that names an existing `.ts` file runs that file directly — it may
+// sit outside the discovery root. Everything else is a substring pattern — but
+// a `.ts` filter that resolves to nothing is almost always a typo'd path, so
+// the silent downgrade to a pattern gets called out.
+const partitionFilters = (filters: string[], cwd: string): FilterPartition => {
+  const explicit: SuiteItem[] = [];
+  const patterns: string[] = [];
+  for (const f of filters) {
+    const abs = resolve(cwd, f);
+    if (f.endsWith(".ts") && existsSync(abs)) {
+      explicit.push({ path: abs, id: caseIdOf(f) });
+    } else {
+      if (f.endsWith(".ts")) {
+        process.stderr.write(`lux: ${abs} does not exist; treating "${f}" as a pattern\n`);
+      }
+      patterns.push(f);
+    }
+  }
+  return { explicit, patterns };
+};
+
+// The selected set — substring/tag matches plus explicitly named files — is
+// the same whether it is run, `--list`ed, or `--list-tags`ed. No filters at
+// all means "everything discovered"; any filter narrows.
+const selectSuite = (
+  cases: DiscoveredCase[],
+  { explicit, patterns }: FilterPartition,
+  tags: string[],
+): DiscoveredCase[] => {
+  const hasFilter = explicit.length > 0 || patterns.length > 0 || tags.length > 0;
+  const byPath = new Map<string, DiscoveredCase>();
+  if (!hasFilter) {
+    for (const c of cases) byPath.set(c.path, c);
+  } else {
+    if (patterns.length > 0 || tags.length > 0) {
+      for (const c of selectCases(cases, { patterns, tags })) byPath.set(c.path, c);
+    }
+    for (const e of explicit) {
+      const discovered = cases.find((c) => c.path === e.path);
+      // Discovery knows the case's declared id; the filename-derived one is
+      // only a fallback for files discovery never saw.
+      byPath.set(e.path, {
+        path: e.path,
+        id: discovered?.id ?? e.id,
+        relPath: discovered?.relPath ?? e.id,
+        tags: discovered?.tags ?? [],
+      });
+    }
+  }
+  return [...byPath.values()].sort((a, b) => a.id.localeCompare(b.id));
+};
+
+// `--loop` repeats each eval before moving to the next; each repeat gets a
+// `#n` display suffix so the reporter keeps them apart, while `caseId` keeps
+// every repeat filed under one run history.
+const expandLoop = (base: SuiteItem[], loop: number): SuiteItem[] =>
+  loop > 1
+    ? base.flatMap((b) =>
+        Array.from({ length: loop }, (_, i) => ({
+          path: b.path,
+          id: `${b.id} #${i + 1}`,
+          caseId: b.caseId ?? b.id,
+        })),
+      )
+    : base;
+
+const experimentCommand = async (
+  mode: "experiment" | "optimize",
+  campaignPath: string,
+  options: { runsRoot?: string; resume?: string; json?: boolean; plan?: boolean },
+): Promise<void> => {
+  const cwd = process.cwd();
+  const config = await loadLuxConfig(cwd);
+  const campaign = await resolveExperimentCampaignPath(config.rootDir, campaignPath);
+  const loaded = await loadExperimentCampaign(campaign);
+  experimentCampaignMode(campaign, loaded.campaign, mode);
+  const registry = buildRegistryFromConfig(config);
+  const controller = abortOnInterrupt();
+  const execution = {
+    config,
+    registry,
+    loaded,
+    ...(options.runsRoot ? { runsRoot: resolve(options.runsRoot) } : {}),
+    ...(options.resume ? { resumeFrom: resolve(options.resume) } : {}),
+    abortSignal: controller.signal,
+  };
+  if (options.plan) {
+    const plan = await planExperimentCampaign(execution, mode);
+    let output: string;
+    if (options.json) {
+      output = `${JSON.stringify(plan, null, 2)}\n`;
+    } else {
+      const projectedStatus = plan.fitsProjectedCallBudgets ? "yes" : "no";
+      output = [
+        `campaign:      ${plan.campaignId}`,
+        `mode:          ${plan.mode}`,
+        `evaluation:    ${plan.evaluationKind}`,
+        `profiles:      ${plan.profiles.join(", ")}`,
+        `splits:        ${plan.splits.map((split) => `${split.split}=${split.examples}`).join(", ")}`,
+        `components:    ${plan.mutableComponents.length}`,
+        `candidates:    ${plan.projected.candidates}`,
+        `metric calls:  ${plan.projected.metricCalls}/${plan.budget.maxMetricCalls}`,
+        `judge calls:   ${plan.projected.judgeCalls}/${plan.budget.maxJudgeCalls}`,
+        `proposal calls:${plan.projected.proposalCalls}/${plan.budget.maxProposalCalls}`,
+        `unpriced calls: observed at runtime (cap ${plan.budget.maxUnpricedModelCalls})`,
+        `projected work within budget: ${projectedStatus}`,
+        ...plan.warnings.map((warning) => `warning:       ${warning}`),
+        "",
+      ].join("\n");
+    }
+    process.stdout.write(output);
+    if (!plan.fitsProjectedCallBudgets) failWith(1);
+    return;
+  }
+  const result =
+    mode === "optimize"
+      ? await optimizeExperimentCampaign(execution)
+      : await runExperimentCampaign(execution);
+  process.stdout.write(
+    options.json
+      ? `${JSON.stringify({ experimentDir: result.experimentDir, ...result.manifest }, null, 2)}\n`
+      : `${formatExperimentRun(result.experimentDir, result.manifest)}\n`,
+  );
+  if (controller.signal.aborted) failWith(SIGINT_EXIT_CODE);
+  else if (result.manifest.status !== "passed") failWith(1);
+};
+
+const runCommand = async (filters: string[], options: RunOptions): Promise<void> => {
+  if (options.interactive && options.claudeAnswerer) {
+    throw new Error("lux run: --interactive and --claude-answerer are mutually exclusive");
+  }
+  const cwd = process.cwd();
+  const config = await loadLuxConfig(cwd);
+  const profile = options.profile ? config.profiles?.[options.profile] : undefined;
+  if (options.profile && !profile) {
+    const available = Object.keys(config.profiles ?? {}).sort();
+    process.stderr.write(
+      `lux: unknown profile '${options.profile}'${available.length > 0 ? `; available: ${available.join(", ")}` : ""}\n`,
+    );
+    failWith(1);
+    return;
+  }
+  const harness = profile?.harness ?? config.harness;
+  const registry = buildRegistryFromConfig(config).withAssertion(createRubricAssertion(harness));
+  const defaultDriver = profile?.driver ?? config.defaultDriver;
+  const orchestrator = new Orchestrator({
+    registry,
+    defaultAnswerer: profile?.answerer ?? config.defaultAnswerer,
+    ...(defaultDriver ? { defaultDriver } : {}),
+    ...(harness ? { harness } : {}),
+    lifecycleFixtures: config.lifecycleFixtures,
+  });
+  const runsRoot = options.runsRoot ? resolve(options.runsRoot) : config.runsRoot;
+  const subjectRoot = options.subjectRoot ? resolve(options.subjectRoot) : config.subjectRoot;
+  // A config file is the user's opt-in to tree-wide discovery. Without one,
+  // stay scoped to the `cases/` scaffold: discovery imports every .ts it
+  // visits, and running module-level side effects across an arbitrary cwd
+  // must never be the default.
+  const root = config.casesRoot ?? (config.configPath ? config.rootDir : join(cwd, "cases"));
+  // A configured casesRoot is a path the user declared; its absence is a
+  // broken config, not an empty suite — never mask it behind "no cases
+  // matched" or a lucky explicit-file run.
+  if (config.casesRoot && !existsSync(config.casesRoot)) {
+    process.stderr.write(`lux: casesRoot does not exist: ${config.casesRoot}\n`);
+    failWith(1);
+    return;
+  }
+
+  const { explicit, patterns } = partitionFilters(filters, cwd);
+  const tags = options.tag ?? [];
+  const hasFilter = explicit.length > 0 || patterns.length > 0 || tags.length > 0;
+
+  // Explicitly named files need no discovery, so they run even without a
+  // project; anything else would need a tree walk with nowhere safe to walk.
+  if (config.configPath === null && !existsSync(root) && explicit.length === 0) {
+    process.stderr.write(
+      `lux: no lux.config.ts or cases/ directory in ${cwd} — run \`lux init\` to scaffold a project, or name a case file directly\n`,
+    );
+    failWith(1);
+    return;
+  }
+
+  const { cases, problems } = existsSync(root)
+    ? await discoverCases(root)
+    : { cases: [], problems: [] };
+  for (const problem of problems) {
+    process.stderr.write(`lux: skipped ${problem.path}: ${firstLine(problem.reason)}\n`);
+  }
+
+  const selected = selectSuite(cases, { explicit, patterns }, tags);
+
+  // An empty selection fails every mode the same way — running, `--list`, and
+  // `--list-tags` all exit 1, so a typo'd filter can never read as a clean pass.
+  if (selected.length === 0) {
+    process.stderr.write(
+      hasFilter
+        ? "lux: no cases matched the given filters\n"
+        : `lux: no cases discovered under ${root}\n`,
+    );
+    failWith(1);
+    return;
+  }
+
+  // List the tag vocabulary of the selected cases — "which --tag values exist?".
+  if (options.listTags) {
+    const counts = new Map<string, number>();
+    for (const c of selected) for (const t of c.tags) counts.set(t, (counts.get(t) ?? 0) + 1);
+    for (const [tag, n] of [...counts].sort((a, b) => a[0].localeCompare(b[0]))) {
+      process.stdout.write(`${tag}\t${n}\n`);
+    }
+    process.stderr.write(
+      `lux: ${counts.size} tag${counts.size === 1 ? "" : "s"} across ${selected.length} cases\n`,
+    );
+    return;
+  }
+
+  const base: SuiteItem[] = selected.map((c) => ({ path: c.path, id: c.id }));
+
+  // Collect-only: answer "what would run?" without spawning a single agent.
+  if (options.list) {
+    for (const item of base) process.stdout.write(`${item.id}\n`);
+    process.stderr.write(`lux: ${base.length} case${base.length === 1 ? "" : "s"} matched\n`);
+    return;
+  }
+
+  const loop = options.loop ?? 1;
+  const items = expandLoop(base, loop);
+
+  const { concurrency, note } = resolveConcurrency(options, config.concurrency);
+  if (note) process.stderr.write(note);
+
+  const reporter = new ConsoleReporter({ verbose: options.verbose ?? false });
+  const controller = abortOnInterrupt();
+  reporter.begin({
+    title: suiteTitle(patterns, tags, explicit, loop),
+    caseIds: items.map((i) => i.id),
+    runsRoot,
+    loop,
+  });
+
+  let results: SuiteCaseResult[] = [];
+  try {
+    const suite = await orchestrator.runSuite(items, {
+      runsRoot,
+      fixturesRoot: config.fixturesRoot,
+      answererOverride: resolveAnswererOverride(options),
+      observer: reporter,
+      abortSignal: controller.signal,
+      concurrency,
+      subjectRoot,
+      profileId: options.profile,
+    });
+    results = suite.results;
+    reporter.setSuiteDir(suite.suiteDir);
+  } finally {
+    reporter.finish();
+  }
+
+  if (controller.signal.aborted) failWith(SIGINT_EXIT_CODE);
+  else if (results.some(isFailure)) failWith(1);
+};
+
+// Internal exports for testing
+export {
+  expandLoop,
+  isFailure,
+  parsePositiveInt,
+  partitionFilters,
+  resolveConcurrency,
+  selectSuite,
+};