@prismatic-io/lux 0.0.2-preview.4 → 0.0.2-preview.41

package/src/cli/init-templates.ts

@@ -0,0 +1,176 @@
+type StarterDriver =
+  | "subprocess"
+  | "claude-code"
+  | "codex"
+  | "cursor"
+  | "copilot"
+  | "antigravity"
+  | "grok-build";
+
+export const luxConfigBody = (
+  withExperiment: boolean,
+  driver?: StarterDriver,
+  model?: string,
+): string => {
+  const subjectRoot = withExperiment ? '  subjectRoot: ".",\n' : "";
+  const profiles = withExperiment ? "  profiles: { starter: {} },\n" : "";
+  const selectedDriver = driver ?? (withExperiment ? undefined : "subprocess");
+  let defaultDriver: string;
+  if (selectedDriver === "subprocess") {
+    defaultDriver = `  defaultDriver: { name: "subprocess", config: {
+    command: process.execPath,
+    args: [fileURLToPath(new URL("./agent/demo.mjs", import.meta.url))],
+    inheritEnv: false,
+  } },
+`;
+  } else if (selectedDriver) {
+    defaultDriver = model
+      ? `  defaultDriver: { name: ${JSON.stringify(selectedDriver)}, config: { model: ${JSON.stringify(model)} } },\n`
+      : `  defaultDriver: ${JSON.stringify(selectedDriver)},\n`;
+  } else {
+    defaultDriver = `  // Select a driver with lux init --experiment --driver codex --force.\n`;
+  }
+  return `${selectedDriver === "subprocess" ? 'import { fileURLToPath } from "node:url";\n' : ""}import type { LuxConfig } from "@prismatic-io/lux";
+
+const config: LuxConfig = {
+  casesRoot: "./cases",
+${subjectRoot}${defaultDriver}${profiles}};
+
+export default config;
+`;
+};
+
+export const DEMO_AGENT_BODY = `import { writeFile } from "node:fs/promises";
+import { join } from "node:path";
+
+const send = (event) => process.stdout.write(JSON.stringify(event) + "\\n");
+send({ type: "ready", agentId: "addition-demo" });
+await writeFile(join(process.env.LUX_ARTIFACTS_DIR, "add.mjs"), "export const add = (a, b) => a + b;\\n");
+send({ type: "done", summary: "Wrote add.mjs" });
+`;
+
+const ADDITION_COMMAND = `import assert from 'node:assert/strict'; import { add } from './add.mjs'; assert.equal(add(2, 3), 5); assert.equal(add(-2, 2), 0); assert.equal(add(0, 0), 0);`;
+
+export const SAMPLE_CASE_BODY = `import { defineEvalCase } from "@prismatic-io/lux";
+
+export default defineEvalCase({
+  prompt: "Write add.mjs exporting add(a, b). It must add two numbers, including negative numbers and zero.",
+  assertions: [
+    { id: "completed", type: "run-succeeded" },
+    { id: "module-written", type: "file-exists", path: "add.mjs" },
+    {
+      id: "adds-correctly",
+      type: "command-exits-zero",
+      command: [process.execPath, "--input-type=module", "-e", ${JSON.stringify(ADDITION_COMMAND)}],
+    },
+  ],
+});
+`;
+
+export const EXPERIMENT_SAMPLE_CASE_BODY = `import { defineEvalCase } from "@prismatic-io/lux";
+
+/**
+ * A starter smoke case. It inherits the default driver from lux.config.ts.
+ */
+export default defineEvalCase({
+  prompt: "Say hello to the world.",
+  assertions: [{ id: "completed", type: "run-succeeded" }],
+});
+`;
+
+export const README_BODY = `# Lux evals
+
+Write tasks and assertions in \`cases/\`; choose the driver in \`lux.config.ts\`.
+Run \`npm exec -- lux run\`, then \`npm exec -- lux view\` to inspect results.
+Pass a case name to run only matching cases. Results are saved in \`.lux-runs/\`.
+
+The default starter uses a deterministic demo agent without model calls.
+Changing \`a + b\` to \`a - b\` in \`agent/demo.mjs\` demonstrates a failed check.
+Scaffolding with \`--driver\` instead uses that agent's authenticated CLI and model account.
+
+See [Lux](https://github.com/prismatic-io/lux) for setup and available assertions.
+`;
+
+export const STARTER_PROMPT_BODY = `# Starter agent instructions
+
+Complete the requested task and verify the result. Ask for clarification when
+missing information changes the correct outcome.
+`;
+
+export const experimentCaseBody = (
+  id: string,
+  split: "train" | "validation" | "test",
+  task: string,
+): string => {
+  const checks = {
+    train: "assert.equal(result(2, 3), 5); assert.equal(result(-2, 2), 0)",
+    validation:
+      "assert.equal(result(12, 0, 10), 10); assert.equal(result(-1, 0, 10), 0); assert.equal(result(5, 0, 10), 5)",
+    test: "assert.equal(result('  Lux REVIEW  '), 'lux review'); assert.equal(result(''), '')",
+  };
+  const command = `import assert from 'node:assert/strict'; import { result } from './result.mjs'; ${checks[split]};`;
+  return `import { defineEvalCase, subjectPath } from "@prismatic-io/lux";
+
+export default defineEvalCase({
+  id: ${JSON.stringify(id)},
+  prompt: [
+    \`Read \${subjectPath("prompts/agent.md")} and use it as task guidance.\`,
+    ${JSON.stringify(task)},
+  ].join("\\n\\n"),
+  assertions: [
+    { id: "completed", type: "run-succeeded" },
+    { id: "behavior", type: "command-exits-zero", command: [process.execPath, "--input-type=module", "-e", ${JSON.stringify(command)}] },
+  ],
+  meta: { tags: [${JSON.stringify(split)}] },
+});
+`;
+};
+
+export const STARTER_EXPERIMENT_BODY = `import { defineExperiment } from "@prismatic-io/lux";
+
+export default defineExperiment({
+  id: "starter-agent-improvement",
+  subject: {
+    root: "..",
+    mutable: ["prompts/agent.md"],
+  },
+  profiles: ["starter"],
+  splits: {
+    train: { tags: ["train"] },
+    validation: { tags: ["validation"] },
+    test: { tags: ["test"] },
+  },
+  repetitions: { search: 1, promotion: 2 },
+  objectives: {
+    constraints: [
+      { metric: "quality.passRate", min: 1 },
+      { metric: "quality.passRate", min: 1, split: "test" },
+      { metric: "quality.caseRegressions", max: 0 },
+      { metric: "quality.caseRegressions", max: 0, split: "test" },
+    ],
+    minimize: [
+      { metric: "agent.total" },
+      { metric: "subject.bytes" },
+    ],
+  },
+  optimizer: {
+    algorithm: "reflective",
+    config: {
+      provider: "claude-code",
+      model: "YOUR_OPTIMIZER_MODEL",
+      reasoningEffort: "low",
+    },
+  },
+  budget: {
+    maxCandidates: 4,
+    patience: 1,
+    maxMetricCalls: 40,
+    maxJudgeCalls: 40,
+    maxProposalCalls: 3,
+    maxTokens: 1_000_000,
+    maxCostUsd: 25,
+    maxUnpricedModelCalls: 25,
+    maxWallTimeMs: 1_800_000,
+  },
+});
+`;

package/src/cli/init.ts

@@ -2,14 +2,49 @@ import { randomUUID } from "node:crypto";
 import { constants, lstat, mkdir, open, realpath, rename, rm, writeFile } from "node:fs/promises";
 import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
 import { z } from "zod";
+import {
+  DEMO_AGENT_BODY,
+  EXPERIMENT_SAMPLE_CASE_BODY,
+  experimentCaseBody,
+  luxConfigBody,
+  README_BODY,
+  SAMPLE_CASE_BODY,
+  STARTER_EXPERIMENT_BODY,
+  STARTER_PROMPT_BODY,
+} from "./init-templates.js";
+
+const EXPLICIT_MODEL_DRIVERS = new Set(["cursor", "copilot", "antigravity", "grok-build"]);
 
 const InitOptionsSchema = z
   .object({
     cwd: z.string().min(1),
     force: z.boolean().default(false),
     experiment: z.boolean().default(false),
+    driver: z
+      .enum([
+        "subprocess",
+        "claude-code",
+        "codex",
+        "cursor",
+        "copilot",
+        "antigravity",
+        "grok-build",
+      ])
+      .optional(),
+    model: z.string().min(1).optional(),
   })
-  .strict();
+  .strict()
+  .refine(
+    (options) => !EXPLICIT_MODEL_DRIVERS.has(options.driver ?? "") || options.model !== undefined,
+    {
+      path: ["model"],
+      message: "Cursor, Copilot, and Antigravity require an explicit --model",
+    },
+  )
+  .refine((options) => !(options.experiment && options.driver === "subprocess"), {
+    path: ["driver"],
+    message: "The deterministic starter is for runs; choose a coding agent for experiments",
+  });
 type InitOptions = z.input<typeof InitOptionsSchema>;
 
 const InitResultSchema = z
@@ -133,181 +168,6 @@ const appendGitignoreEntry = async (
   else result.updated.push(path);
 };
 
-const luxConfigBody = (withExperiment: boolean): string => {
-  const subjectRoot = withExperiment ? '  subjectRoot: ".",\n' : "";
-  const profiles = withExperiment ? "  profiles: { starter: {} },\n" : "";
-  const defaultDriver = withExperiment
-    ? `  // Choose one driver before running the scaffold.
-  // defaultDriver: { name: "codex", config: { model: "YOUR_CODEX_MODEL", reasoningEffort: "high" } },
-  // defaultDriver: { name: "claude-code", config: { model: "YOUR_CLAUDE_MODEL", effort: "high" } },
-`
-    : `  // The subprocess command must implement Lux's lux-jsonl protocol.
-  // Replace this entire object when using the built-in Codex or Claude Code driver.
-  defaultDriver: { name: "subprocess", config: { command: "your-agent-cli" } },
-`;
-  return `import type { LuxConfig } from "@prismatic-io/lux";
-
-/**
- * Lux project configuration. Drivers, answerers, and assertions
- * register here; \`lux run\` discovers cases under \`casesRoot\`.
- */
-const config: LuxConfig = {
-  casesRoot: "./cases",
-  fixturesRoot: "./fixtures",
-  runsRoot: "./.lux-runs",
-${subjectRoot}  // Runners default to cheap models and low effort instead of inheriting CLI defaults.
-${defaultDriver}  // Persona answerers and rubric judges default to cheap models and low effort.
-  // Override both explicitly when a stronger harness is worth the added cost:
-  // harness: { provider: "codex", model: "YOUR_HARNESS_MODEL" },
-  defaultAnswerer: "persona",
-${profiles}  // drivers: [/* register drivers here */],
-  // answerers: [/* register answerers here */],
-  // assertions: [/* register custom assertions here */],
-};
-
-export default config;
-`;
-};
-
-const SAMPLE_CASE_BODY = `import { defineEvalCase } from "@prismatic-io/lux";
-
-/**
- * A starter case. It inherits defaultDriver from lux.config.ts.
- * Replace the prompt and persona to match the behavior you need.
- */
-export default defineEvalCase({
-  id: "hello-world",
-  prompt: "Say hello to the world.",
-  persona: "You are a friendly developer. Keep answers short.",
-  assertions: [
-    { id: "completed", type: "run-succeeded" },
-    // { type: "contains", text: "hello" },
-    // { type: "command-exits-zero", command: "echo done" },
-  ],
-});
-`;
-
-const EXPERIMENT_SAMPLE_CASE_BODY = `import { defineEvalCase } from "@prismatic-io/lux";
-
-/**
- * A starter smoke case. It inherits the default driver from lux.config.ts.
- */
-export default defineEvalCase({
-  id: "hello-world",
-  prompt: "Say hello to the world.",
-  assertions: [{ id: "completed", type: "run-succeeded" }],
-});
-`;
-
-const README_BODY = `# Lux evals
-
-This directory holds your evaluation cases.
-
-## Layout
-
-- \`lux.config.ts\` — project configuration: default driver/answerer and registered plugins.
-- \`cases/\` — one TypeScript file per eval case. Default-export a \`defineEvalCase({...})\`.
-- \`fixtures/\` — starting state for cases that need pre-staged input.
-- \`.lux-runs/\` — run output, gitignored.
-
-## Setup
-
-Install Lux in the project so authored TypeScript cases can resolve its API:
-
-\`\`\`bash
-npm install --save-dev @prismatic-io/lux
-\`\`\`
-
-## Run cases
-
-\`lux run\` discovers every case under \`casesRoot\` and runs it. Filter by
-name/path substring, by \`--tag\`, or repeat each matched eval with \`--loop\`:
-
-\`\`\`bash
-npm exec -- lux run                       # every discovered case
-npm exec -- lux run hello-world           # cases whose id/path contains "hello-world"
-npm exec -- lux run cases/hello-world.ts  # one file directly
-npm exec -- lux run --tag smoke           # cases tagged "smoke" (meta.tags)
-npm exec -- lux run --loop 5              # run each matched eval 5 times
-\`\`\`
-
-See the [Lux docs](https://github.com/prismatic-io/lux) for the full design.
-`;
-
-const STARTER_PROMPT_BODY = `# Starter agent instructions
-
-Complete the requested task directly. Prefer deterministic work, use the fewest
-tools needed, and ask a question only when proceeding would be unsafe.
-`;
-
-const experimentCaseBody = (
-  id: string,
-  split: "train" | "validation" | "test",
-  task: string,
-): string => `import { defineEvalCase, subjectPath } from "@prismatic-io/lux";
-
-export default defineEvalCase({
-  id: ${JSON.stringify(id)},
-  prompt: [
-    \`Read \${subjectPath("prompts/agent.md")} and follow it as the system instructions.\`,
-    ${JSON.stringify(task)},
-  ].join("\\n\\n"),
-  assertions: [{ id: "completed", type: "run-succeeded" }],
-  meta: { tags: [${JSON.stringify(split)}] },
-});
-`;
-
-const STARTER_EXPERIMENT_BODY = `import { defineExperiment } from "@prismatic-io/lux";
-
-export default defineExperiment({
-  id: "starter-agent-improvement",
-  subject: {
-    root: "..",
-    mutable: ["prompts/agent.md"],
-  },
-  profiles: ["starter"],
-  splits: {
-    train: { tags: ["train"] },
-    validation: { tags: ["validation"] },
-    test: { tags: ["test"] },
-  },
-  repetitions: { search: 1, promotion: 2 },
-  objectives: {
-    constraints: [
-      { metric: "quality.passRate", min: 1 },
-      { metric: "quality.passRate", min: 1, split: "test" },
-      { metric: "quality.caseRegressions", max: 0 },
-      { metric: "quality.caseRegressions", max: 0, split: "test" },
-    ],
-    minimize: [
-      { metric: "agent.total" },
-      { metric: "interrupts.ask" },
-      { metric: "toolCalls" },
-      { metric: "subject.bytes" },
-    ],
-  },
-  optimizer: {
-    algorithm: "reflective",
-    config: {
-      provider: "claude-code",
-      model: "YOUR_OPTIMIZER_MODEL",
-      reasoningEffort: "low",
-    },
-  },
-  budget: {
-    maxCandidates: 4,
-    patience: 1,
-    maxMetricCalls: 40,
-    maxJudgeCalls: 40,
-    maxProposalCalls: 3,
-    maxTokens: 1_000_000,
-    maxCostUsd: 25,
-    maxUnpricedModelCalls: 25,
-    maxWallTimeMs: 1_800_000,
-  },
-});
-`;
-
 export const initProject = async (options: InitOptions): Promise<InitResult> => {
   const parsedOptions = InitOptionsSchema.parse(options);
   const experiment = parsedOptions.experiment;
@@ -317,14 +177,23 @@ export const initProject = async (options: InitOptions): Promise<InitResult> =>
   await mkdir(parsedOptions.cwd, { recursive: true });
   const root = await realpath(parsedOptions.cwd);
 
-  await writeIfMissing(root, join(root, "lux.config.ts"), luxConfigBody(experiment), result, force);
   await writeIfMissing(
     root,
-    join(root, "cases", "hello-world.ts"),
+    join(root, "lux.config.ts"),
+    luxConfigBody(experiment, parsedOptions.driver, parsedOptions.model),
+    result,
+    force,
+  );
+  await writeIfMissing(
+    root,
+    join(root, "cases", experiment ? "hello-world.ts" : "addition.ts"),
     experiment ? EXPERIMENT_SAMPLE_CASE_BODY : SAMPLE_CASE_BODY,
     result,
     force,
   );
+  if (!experiment && (!parsedOptions.driver || parsedOptions.driver === "subprocess")) {
+    await writeIfMissing(root, join(root, "agent", "demo.mjs"), DEMO_AGENT_BODY, result, force);
+  }
   await writeIfMissing(root, join(root, "fixtures", ".gitkeep"), "", result, force);
   // README is the user's to evolve once it exists; `--force` refreshes only
   // the lux-owned scaffold files, never a hand-edited README.
@@ -344,7 +213,7 @@ export const initProject = async (options: InitOptions): Promise<InitResult> =>
       experimentCaseBody(
         "starter/train",
         "train",
-        "Complete the representative training task and summarize the result.",
+        "Write result.mjs exporting result(a, b), which adds two numbers including negative numbers.",
       ),
       result,
       force,
@@ -355,7 +224,7 @@ export const initProject = async (options: InitOptions): Promise<InitResult> =>
       experimentCaseBody(
         "starter/validation",
         "validation",
-        "Complete an independently designed validation task and summarize the result.",
+        "Write result.mjs exporting result(value, min, max), which clamps a number to the inclusive bounds.",
       ),
       result,
       force,
@@ -366,7 +235,7 @@ export const initProject = async (options: InitOptions): Promise<InitResult> =>
       experimentCaseBody(
         "starter/test",
         "test",
-        "Complete the final held-out task and summarize the result.",
+        "Write result.mjs exporting result(text), which trims surrounding whitespace and lowercases the text.",
       ),
       result,
       force,
@@ -397,14 +266,14 @@ export const formatInitReport = (result: InitResult, cwd: string): string => {
   lines.push("");
   lines.push("Next steps:");
   lines.push("  1. Install @prismatic-io/lux in this project if it is not already present.");
-  lines.push("  2. Edit lux.config.ts to choose a driver.");
+  lines.push("  2. Review lux.config.ts and the selected driver.");
   if (result.experiment) {
     lines.push("  3. Replace the starter prompt and split cases with representative evidence.");
     lines.push("  4. lux doctor");
     lines.push("  5. lux optimize experiments/starter.optimize.ts --plan");
   } else {
-    lines.push("  3. Edit cases/hello-world.ts with a real prompt.");
-    lines.push("  4. lux run");
+    lines.push("  3. lux run");
+    lines.push("  4. lux view");
   }
   return lines.join("\n");
 };

package/src/cli/output-schemas.ts

@@ -0,0 +1,109 @@
+import { z } from "incur";
+import {
+  ActionTimingSchema,
+  AgentActivitySchema,
+  DriverMetricsSchema,
+  GradingReportSchema,
+  PersistedCaseSchema,
+  RunEventSchema,
+  RunMetadataSchema,
+  RunUsageSchema,
+} from "../core/index.js";
+import {
+  ExperimentRunSchema,
+  SuiteComparisonSchema,
+  SuiteRunSchema,
+} from "../orchestrator/index.js";
+
+const nullableNumber = z.number().nullable();
+const listedRun = z.object({
+  caseSlug: z.string(),
+  driverName: z.string(),
+  ts: z.string(),
+  runDir: z.string(),
+  exitReason: RunMetadataSchema.shape.exitReason.nullable(),
+  driverAgentId: z.string().nullable(),
+  startedAt: z.string().nullable(),
+  endedAt: z.string().nullable(),
+  durationMs: nullableNumber,
+  timeToFirstActionMs: nullableNumber,
+  timeToFirstMutationMs: nullableNumber,
+  timeToFirstDurableMutationMs: nullableNumber,
+  driverMetrics: DriverMetricsSchema,
+  subagentCount: z.number(),
+  casePassed: z.boolean().nullable(),
+  caseScore: nullableNumber,
+  assertionPassed: z.number(),
+  assertionTotal: z.number(),
+  readErrors: z.array(z.string()).optional(),
+});
+const suite = SuiteRunSchema.extend({ suiteDir: z.string() });
+const experiment = ExperimentRunSchema.extend({ experimentDir: z.string() });
+const listing = z.object({
+  runsRoot: z.string(),
+  readErrors: z.array(z.object({ path: z.string(), message: z.string() })).optional(),
+});
+export const ViewOutputSchema = z.union([
+  z.object({
+    runDir: z.string(),
+    case: PersistedCaseSchema.nullable(),
+    metadata: RunMetadataSchema.nullable(),
+    grading: GradingReportSchema.nullable(),
+    events: z.array(RunEventSchema),
+    artifacts: z.array(z.string()),
+  }),
+  suite,
+  experiment,
+  listing.extend({ runs: z.array(listedRun), suites: z.array(suite).optional() }),
+  listing.extend({ suites: z.array(suite) }),
+  listing.extend({ experiments: z.array(experiment) }),
+  z.object({ url: z.string(), runsRoot: z.string() }),
+]);
+const interrupts = z.object({ ask: z.number(), approve: z.number(), custom: z.number() });
+const verdict = z
+  .object({ passed: z.boolean().nullable(), score: z.number().nullable() })
+  .nullable();
+const runComparison = z.object({
+  pathA: z.string(),
+  pathB: z.string(),
+  caseMismatch: z.array(z.object({ field: z.string(), a: z.unknown(), b: z.unknown() })),
+  runDelta: z
+    .object({
+      exitReason: z.object({ a: z.string(), b: z.string() }),
+      durationMsA: nullableNumber,
+      durationMsB: nullableNumber,
+      actionTiming: z.object({ a: ActionTimingSchema, b: ActionTimingSchema }),
+      driverMetrics: z.object({ a: DriverMetricsSchema, b: DriverMetricsSchema }),
+      agentActivity: z.object({ a: z.array(AgentActivitySchema), b: z.array(AgentActivitySchema) }),
+      interruptCounts: z.object({ a: interrupts, b: interrupts }),
+      usage: z.object({ a: RunUsageSchema, b: RunUsageSchema }),
+      agentUsageAvailable: z.object({ a: z.boolean(), b: z.boolean() }).optional(),
+    })
+    .nullable(),
+  grading: z.object({
+    casePassedA: z.boolean().nullable(),
+    casePassedB: z.boolean().nullable(),
+    caseScoreA: nullableNumber,
+    caseScoreB: nullableNumber,
+    caseScoreComparable: z.boolean(),
+    deltas: z.array(
+      z.object({
+        id: z.string(),
+        type: z.string(),
+        name: z.string(),
+        before: verdict,
+        after: verdict,
+        status: z.enum(["added", "removed", "improved", "regressed", "unchanged", "unavailable"]),
+      }),
+    ),
+    improvements: z.number(),
+    regressions: z.number(),
+  }),
+  artifactsDelta: z.object({
+    added: z.array(z.string()),
+    removed: z.array(z.string()),
+    changed: z.array(z.string()),
+    unchanged: z.array(z.string()),
+  }),
+});
+export const CompareOutputSchema = z.union([SuiteComparisonSchema, runComparison]);