@prismatic-io/lux 0.0.1

package/src/cli/init.ts

@@ -0,0 +1,410 @@
+import { randomUUID } from "node:crypto";
+import { constants, lstat, mkdir, open, realpath, rename, rm, writeFile } from "node:fs/promises";
+import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
+import { z } from "zod";
+
+const InitOptionsSchema = z
+  .object({
+    cwd: z.string().min(1),
+    force: z.boolean().default(false),
+    experiment: z.boolean().default(false),
+  })
+  .strict();
+type InitOptions = z.input<typeof InitOptionsSchema>;
+
+const InitResultSchema = z
+  .object({
+    created: z.array(z.string()),
+    updated: z.array(z.string()),
+    skipped: z.array(
+      z
+        .object({
+          path: z.string(),
+          reason: z.string(),
+        })
+        .strict(),
+    ),
+    experiment: z.boolean(),
+  })
+  .strict();
+type InitResult = z.infer<typeof InitResultSchema>;
+
+const pathWithin = (root: string, path: string): string | null => {
+  const fromRoot = relative(resolve(root), resolve(path));
+  if (
+    fromRoot === "" ||
+    (!isAbsolute(fromRoot) && fromRoot !== ".." && !fromRoot.startsWith(`..${sep}`))
+  ) {
+    return fromRoot;
+  }
+  return null;
+};
+
+const ensureSafeDir = async (root: string, path: string): Promise<void> => {
+  const fromRoot = pathWithin(root, path);
+  if (fromRoot === null) throw new Error(`lux init: path escapes project root: ${path}`);
+  let cursor = root;
+  for (const component of fromRoot.split(sep).filter(Boolean)) {
+    cursor = join(cursor, component);
+    try {
+      const info = await lstat(cursor);
+      if (info.isSymbolicLink() || !info.isDirectory()) {
+        throw new Error(`lux init: path component must be a real directory: ${cursor}`);
+      }
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
+      await mkdir(cursor);
+    }
+  }
+};
+
+const existingRegularFile = async (path: string): Promise<boolean> => {
+  try {
+    const info = await lstat(path);
+    if (info.isSymbolicLink() || !info.isFile()) {
+      throw new Error(`lux init: scaffold target must be a regular file: ${path}`);
+    }
+    return true;
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === "ENOENT") return false;
+    throw error;
+  }
+};
+
+const readRegularFile = async (path: string): Promise<string> => {
+  const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW);
+  try {
+    const info = await handle.stat();
+    if (!info.isFile()) throw new Error(`lux init: scaffold target must be a file: ${path}`);
+    return await handle.readFile("utf8");
+  } finally {
+    await handle.close();
+  }
+};
+
+const atomicReplace = async (path: string, body: string): Promise<void> => {
+  const temporary = join(dirname(path), `.${basename(path)}.${randomUUID()}.tmp`);
+  try {
+    await writeFile(temporary, body, { flag: "wx" });
+    await rename(temporary, path);
+  } finally {
+    await rm(temporary, { force: true });
+  }
+};
+
+const writeIfMissing = async (
+  root: string,
+  path: string,
+  body: string,
+  result: InitResult,
+  force: boolean,
+): Promise<void> => {
+  await ensureSafeDir(root, dirname(path));
+  const exists = await existingRegularFile(path);
+  if (!force && exists) {
+    result.skipped.push({ path, reason: "exists" });
+    return;
+  }
+  if (exists) await atomicReplace(path, body);
+  else await writeFile(path, body, { flag: "wx" });
+  if (exists) result.updated.push(path);
+  else result.created.push(path);
+};
+
+const appendGitignoreEntry = async (
+  cwd: string,
+  entry: string,
+  result: InitResult,
+): Promise<void> => {
+  const path = join(cwd, ".gitignore");
+  const exists = await existingRegularFile(path);
+  const current = exists ? await readRegularFile(path) : "";
+  if (current.split(/\r?\n/).includes(entry)) {
+    result.skipped.push({ path, reason: "entry already present" });
+    return;
+  }
+  // Preserve the user's exact content while replacing the directory entry
+  // atomically, so symlink/hard-link aliases are never followed.
+  const needsNewline = current !== "" && !current.endsWith("\n");
+  const updated = `${current}${needsNewline ? "\n" : ""}${entry}\n`;
+  if (exists) await atomicReplace(path, updated);
+  else await writeFile(path, updated, { flag: "wx" });
+  if (!exists) result.created.push(path);
+  else result.updated.push(path);
+};
+
+const luxConfigBody = (withExperiment: boolean): string => {
+  const subjectRoot = withExperiment ? '  subjectRoot: ".",\n' : "";
+  const profiles = withExperiment ? "  profiles: { starter: {} },\n" : "";
+  const defaultDriver = withExperiment
+    ? `  // Choose one driver before running the scaffold.
+  // defaultDriver: { name: "codex", config: { model: "YOUR_CODEX_MODEL", reasoningEffort: "high" } },
+  // defaultDriver: { name: "claude-code", config: { model: "YOUR_CLAUDE_MODEL", effort: "high" } },
+`
+    : `  // The subprocess command must implement Lux's lux-jsonl protocol.
+  // Replace this entire object when using the built-in Codex or Claude Code driver.
+  defaultDriver: { name: "subprocess", config: { command: "your-agent-cli" } },
+`;
+  return `import type { LuxConfig } from "@prismatic-io/lux";
+
+/**
+ * Lux project configuration. Drivers, answerers, and assertions
+ * register here; \`lux run\` discovers cases under \`casesRoot\`.
+ */
+const config: LuxConfig = {
+  casesRoot: "./cases",
+  fixturesRoot: "./fixtures",
+  runsRoot: "./.lux-runs",
+${subjectRoot}  // Runners default to cheap models and low effort instead of inheriting CLI defaults.
+${defaultDriver}  // Persona answerers and rubric judges default to cheap models and low effort.
+  // Override both explicitly when a stronger harness is worth the added cost:
+  // harness: { provider: "codex", model: "YOUR_HARNESS_MODEL" },
+  defaultAnswerer: "persona",
+${profiles}  // drivers: [/* register drivers here */],
+  // answerers: [/* register answerers here */],
+  // assertions: [/* register custom assertions here */],
+};
+
+export default config;
+`;
+};
+
+const SAMPLE_CASE_BODY = `import { defineEvalCase } from "@prismatic-io/lux";
+
+/**
+ * A starter case. It inherits defaultDriver from lux.config.ts.
+ * Replace the prompt and persona to match the behavior you need.
+ */
+export default defineEvalCase({
+  id: "hello-world",
+  prompt: "Say hello to the world.",
+  persona: "You are a friendly developer. Keep answers short.",
+  assertions: [
+    { id: "completed", type: "run-succeeded" },
+    // { type: "contains", text: "hello" },
+    // { type: "command-exits-zero", command: "echo done" },
+  ],
+});
+`;
+
+const EXPERIMENT_SAMPLE_CASE_BODY = `import { defineEvalCase } from "@prismatic-io/lux";
+
+/**
+ * A starter smoke case. It inherits the default driver from lux.config.ts.
+ */
+export default defineEvalCase({
+  id: "hello-world",
+  prompt: "Say hello to the world.",
+  assertions: [{ id: "completed", type: "run-succeeded" }],
+});
+`;
+
+const README_BODY = `# Lux evals
+
+This directory holds your evaluation cases.
+
+## Layout
+
+- \`lux.config.ts\` — project configuration: default driver/answerer and registered plugins.
+- \`cases/\` — one TypeScript file per eval case. Default-export a \`defineEvalCase({...})\`.
+- \`fixtures/\` — starting state for cases that need pre-staged input.
+- \`.lux-runs/\` — run output, gitignored.
+
+## Setup
+
+Install Lux in the project so authored TypeScript cases can resolve its API:
+
+\`\`\`bash
+npm install --save-dev @prismatic-io/lux
+\`\`\`
+
+## Run cases
+
+\`lux run\` discovers every case under \`casesRoot\` and runs it. Filter by
+name/path substring, by \`--tag\`, or repeat each matched eval with \`--loop\`:
+
+\`\`\`bash
+npm exec -- lux run                       # every discovered case
+npm exec -- lux run hello-world           # cases whose id/path contains "hello-world"
+npm exec -- lux run cases/hello-world.ts  # one file directly
+npm exec -- lux run --tag smoke           # cases tagged "smoke" (meta.tags)
+npm exec -- lux run --loop 5              # run each matched eval 5 times
+\`\`\`
+
+See the [Lux docs](https://github.com/prismatic-io/lux) for the full design.
+`;
+
+const STARTER_PROMPT_BODY = `# Starter agent instructions
+
+Complete the requested task directly. Prefer deterministic work, use the fewest
+tools needed, and ask a question only when proceeding would be unsafe.
+`;
+
+const experimentCaseBody = (
+  id: string,
+  split: "train" | "validation" | "test",
+  task: string,
+): string => `import { defineEvalCase, subjectPath } from "@prismatic-io/lux";
+
+export default defineEvalCase({
+  id: ${JSON.stringify(id)},
+  prompt: [
+    \`Read \${subjectPath("prompts/agent.md")} and follow it as the system instructions.\`,
+    ${JSON.stringify(task)},
+  ].join("\\n\\n"),
+  assertions: [{ id: "completed", type: "run-succeeded" }],
+  meta: { tags: [${JSON.stringify(split)}] },
+});
+`;
+
+const STARTER_EXPERIMENT_BODY = `import { defineExperiment } from "@prismatic-io/lux";
+
+export default defineExperiment({
+  id: "starter-agent-improvement",
+  subject: {
+    root: "..",
+    mutable: ["prompts/agent.md"],
+  },
+  profiles: ["starter"],
+  splits: {
+    train: { tags: ["train"] },
+    validation: { tags: ["validation"] },
+    test: { tags: ["test"] },
+  },
+  repetitions: { search: 1, promotion: 2 },
+  objectives: {
+    constraints: [
+      { metric: "quality.passRate", min: 1 },
+      { metric: "quality.passRate", min: 1, split: "test" },
+      { metric: "quality.caseRegressions", max: 0 },
+      { metric: "quality.caseRegressions", max: 0, split: "test" },
+    ],
+    minimize: [
+      { metric: "agent.total" },
+      { metric: "interrupts.ask" },
+      { metric: "toolCalls" },
+      { metric: "subject.bytes" },
+    ],
+  },
+  optimizer: {
+    algorithm: "reflective",
+    config: {
+      provider: "claude-code",
+      model: "YOUR_OPTIMIZER_MODEL",
+      reasoningEffort: "low",
+    },
+  },
+  budget: {
+    maxCandidates: 4,
+    patience: 1,
+    maxMetricCalls: 40,
+    maxJudgeCalls: 40,
+    maxProposalCalls: 3,
+    maxTokens: 1_000_000,
+    maxCostUsd: 25,
+    maxUnpricedModelCalls: 25,
+    maxWallTimeMs: 1_800_000,
+  },
+});
+`;
+
+export const initProject = async (options: InitOptions): Promise<InitResult> => {
+  const parsedOptions = InitOptionsSchema.parse(options);
+  const experiment = parsedOptions.experiment;
+  const result: InitResult = { created: [], updated: [], skipped: [], experiment };
+  const force = parsedOptions.force;
+
+  await mkdir(parsedOptions.cwd, { recursive: true });
+  const root = await realpath(parsedOptions.cwd);
+
+  await writeIfMissing(root, join(root, "lux.config.ts"), luxConfigBody(experiment), result, force);
+  await writeIfMissing(
+    root,
+    join(root, "cases", "hello-world.ts"),
+    experiment ? EXPERIMENT_SAMPLE_CASE_BODY : SAMPLE_CASE_BODY,
+    result,
+    force,
+  );
+  await writeIfMissing(root, join(root, "fixtures", ".gitkeep"), "", result, force);
+  // README is the user's to evolve once it exists; `--force` refreshes only
+  // the lux-owned scaffold files, never a hand-edited README.
+  await writeIfMissing(root, join(root, "README.md"), README_BODY, result, false);
+
+  if (experiment) {
+    await writeIfMissing(
+      root,
+      join(root, "prompts", "agent.md"),
+      STARTER_PROMPT_BODY,
+      result,
+      force,
+    );
+    await writeIfMissing(
+      root,
+      join(root, "cases", "experiment-train.ts"),
+      experimentCaseBody(
+        "starter/train",
+        "train",
+        "Complete the representative training task and summarize the result.",
+      ),
+      result,
+      force,
+    );
+    await writeIfMissing(
+      root,
+      join(root, "cases", "experiment-validation.ts"),
+      experimentCaseBody(
+        "starter/validation",
+        "validation",
+        "Complete an independently designed validation task and summarize the result.",
+      ),
+      result,
+      force,
+    );
+    await writeIfMissing(
+      root,
+      join(root, "cases", "experiment-test.ts"),
+      experimentCaseBody(
+        "starter/test",
+        "test",
+        "Complete the final held-out task and summarize the result.",
+      ),
+      result,
+      force,
+    );
+    await writeIfMissing(
+      root,
+      join(root, "experiments", "starter.optimize.ts"),
+      STARTER_EXPERIMENT_BODY,
+      result,
+      force,
+    );
+  }
+
+  await appendGitignoreEntry(root, ".lux-runs/", result);
+
+  return InitResultSchema.parse(result);
+};
+
+export const formatInitReport = (result: InitResult, cwd: string): string => {
+  const rel = (path: string): string => relative(cwd, path) || path;
+  const lines: string[] = [];
+  for (const c of result.created) lines.push(`  + ${rel(c)}`);
+  for (const u of result.updated) lines.push(`  ~ ${rel(u)} (updated)`);
+  for (const s of result.skipped) lines.push(`  · ${rel(s.path)} (${s.reason})`);
+  if (result.created.length === 0 && result.updated.length === 0) {
+    lines.push("  (nothing created — use --force to overwrite)");
+  }
+  lines.push("");
+  lines.push("Next steps:");
+  lines.push("  1. Install @prismatic-io/lux in this project if it is not already present.");
+  lines.push("  2. Edit lux.config.ts to choose a driver.");
+  if (result.experiment) {
+    lines.push("  3. Replace the starter prompt and split cases with representative evidence.");
+    lines.push("  4. lux doctor");
+    lines.push("  5. lux optimize experiments/starter.optimize.ts --plan");
+  } else {
+    lines.push("  3. Edit cases/hello-world.ts with a real prompt.");
+    lines.push("  4. lux run");
+  }
+  return lines.join("\n");
+};