@theaiteam/promptdiff 1.0.0-rc.1

package/src/cli.ts

@@ -0,0 +1,685 @@
+import { readFileSync } from "node:fs";
+import { resolve } from "node:path";
+import { CliError, parseArgs, type FlagSpecs } from "./args";
+import { loadCompareConfig, type ArmConfig, type CompareConfig, type CompareOverrides } from "./engine/config";
+import { formatCompareSummary, formatMeasureSummary, runCompare, runMeasure } from "./engine/compare";
+import { prepareSandbox } from "./engine/sandbox";
+import { installSkills, type Delivery } from "./engine/skill-install";
+import { renderStrict, resolveRenderVars, type RenderVars } from "./engine/render";
+import { appendNdjsonReport } from "./engine/report";
+import { buildCompareReceipts, buildMeasureReceipts, writeReceipts } from "./engine/receipt";
+import { formatCalibrationReport, runCalibration } from "./engine/judge";
+import { assembleSystemPrompt } from "./prompt";
+import { createRunner, RUNNER_NAMES, type RunnerName } from "./runner";
+import type { ModelPricing } from "./runner/openai-compat";
+import type { RunMode, Runner, RunnerRunOptions } from "./types";
+
+const DEFAULT_TIMEOUT_MS = 600_000;
+const DEFAULT_MAX_BUDGET_USD = 1;
+
+const runSpecs: FlagSpecs = {
+  agent: { arity: "one" },
+  skill: { arity: "one", repeat: true },
+  delivery: { arity: "one" },
+  runner: { arity: "one" },
+  "base-url": { arity: "one" },
+  model: { arity: "one" },
+  prompt: { arity: "one" },
+  "prompt-file": { arity: "one" },
+  image: { arity: "one", repeat: true },
+  var: { arity: "one", repeat: true },
+  price: { arity: "one" },
+  mode: { arity: "one" },
+  sandbox: { arity: "one" },
+  seed: { arity: "one" },
+  "add-dir": { arity: "one", repeat: true },
+  tools: { arity: "one" },
+  "timeout-ms": { arity: "one" },
+  "max-budget-usd": { arity: "one" },
+  "keep-sandbox": { arity: "none" },
+  "clean-sandbox": { arity: "none" },
+};
+
+const compareSpecs: FlagSpecs = {
+  scenario: { arity: "one" },
+  agent: { arity: "one" },
+  baseline: { arity: "one", repeat: true },
+  proposed: { arity: "one", repeat: true },
+  "baseline-skill": { arity: "one", repeat: true },
+  "proposed-skill": { arity: "one", repeat: true },
+  delivery: { arity: "one" },
+  runner: { arity: "one" },
+  "baseline-runner": { arity: "one" },
+  "proposed-runner": { arity: "one" },
+  "base-url": { arity: "one" },
+  model: { arity: "one" },
+  "baseline-model": { arity: "one" },
+  "proposed-model": { arity: "one" },
+  runs: { arity: "one" },
+  mode: { arity: "one" },
+  sandbox: { arity: "one" },
+  seed: { arity: "one" },
+  "add-dir": { arity: "one", repeat: true },
+  tools: { arity: "one" },
+  "timeout-ms": { arity: "one" },
+  "max-budget-usd": { arity: "one" },
+  "keep-sandbox": { arity: "none" },
+  report: { arity: "one" },
+  "report-out": { arity: "one" },
+  receipts: { arity: "one" },
+  cache: { arity: "none" },
+  "cache-dir": { arity: "one" },
+};
+
+export async function main(argv: string[]): Promise<number> {
+  try {
+    const [command, ...rest] = argv;
+    switch (command) {
+      case undefined:
+      case "--help":
+      case "-h":
+      case "help":
+        console.log(generalUsage());
+        return command === undefined ? 2 : 0;
+      case "--version":
+        console.log(packageVersion());
+        return 0;
+      case "run":
+        if (isHelp(rest)) {
+          console.log(runUsage());
+          return 0;
+        }
+        await cmdRun(rest);
+        return 0;
+      case "compare":
+        if (isHelp(rest)) {
+          console.log(compareUsage());
+          return 0;
+        }
+        return await cmdCompare(rest);
+      case "measure":
+        if (isHelp(rest)) {
+          console.log(measureUsage());
+          return 0;
+        }
+        await cmdMeasure(rest);
+        return 0;
+      case "calibrate":
+        if (isHelp(rest)) {
+          console.log(calibrateUsage());
+          return 0;
+        }
+        await cmdCalibrate(rest);
+        return 0;
+      default:
+        throw new CliError(generalUsage());
+    }
+  } catch (error) {
+    if (error instanceof CliError) {
+      console.error(error.message);
+      return error.exitCode;
+    }
+    console.error(error instanceof Error ? error.message : String(error));
+    return 1;
+  }
+}
+
+function isHelp(argv: string[]): boolean {
+  return argv.length === 1 && (argv[0] === "--help" || argv[0] === "-h");
+}
+
+function packageVersion(): string {
+  const pkg = JSON.parse(readFileSync(new URL("../package.json", import.meta.url), "utf8")) as { version: string };
+  return pkg.version;
+}
+
+async function cmdRun(argv: string[]): Promise<void> {
+  const args = parseArgs(argv, runSpecs);
+  const agent = args.one("agent");
+  const model = args.one("model");
+  if (!agent || !model) {
+    throw new CliError(runUsage());
+  }
+
+  const prompt = promptFromArgs(args.one("prompt"), args.one("prompt-file"));
+  const delivery = deliveryFromString(args.one("delivery") ?? "inline");
+  const mode = modeFromString(args.one("mode") ?? (delivery === "install" ? "artifact" : "text"));
+  const tools = args.one("tools") ?? (delivery === "install" ? "default" : defaultTools(mode));
+  if (delivery === "install" && tools === "") {
+    throw new CliError('--delivery install needs tools enabled: skills trigger via the Skill tool, so --tools "" can never fire them');
+  }
+  const runner = runnerFromArgs(args.one("runner"), args.one("base-url"), args.one("price"));
+  if (delivery === "install" && !runner.capabilities.skillRegistry) {
+    throw new CliError(`--delivery install needs a runner with a skill registry; runner "${runner.name}" has none (use claude-p)`);
+  }
+  if (tools !== "" && !runner.capabilities.sandboxTools) {
+    throw new CliError(`runner "${runner.name}" is text-only — artifact mode and tools need claude-p (or pass --tools "")`);
+  }
+  const images = args.many("image");
+  if (images.length > 0 && !runner.capabilities.images) {
+    throw new CliError(`runner "${runner.name}" cannot attach images — use --runner openai with a vision model`);
+  }
+  const renderVars = varsFromFlags(args.many("var"));
+  if (renderVars !== undefined && delivery === "install") {
+    throw new CliError("--var applies to inlined prompt text; --delivery install copies skill files verbatim, so placeholders cannot be bound");
+  }
+  const keepSandbox = args.has("keep-sandbox") || (mode === "artifact" && !args.has("clean-sandbox"));
+  const sandbox = prepareSandbox({
+    root: args.one("sandbox") ?? ".promptdiff/run",
+    seed: args.one("seed"),
+    prefix: "run",
+    keep: keepSandbox,
+  });
+
+  try {
+    const assembled = assembleSystemPrompt(agent, delivery === "install" ? [] : args.many("skill"));
+    const varHint = "bind them with --var name=value";
+    const systemPrompt = renderVars === undefined ? assembled : renderStrict(assembled, renderVars, "system prompt", varHint);
+    const userPrompt = renderVars === undefined ? prompt : renderStrict(prompt, renderVars, "prompt", varHint);
+    if (delivery === "install") {
+      const { installed, warnings } = installSkills(args.many("skill"), sandbox.dir);
+      console.error(
+        `[promptdiff] installed skills: ${installed.map((skill) => skill.name).join(", ") || "(none)"}`,
+      );
+      for (const warning of warnings) {
+        console.error(`[promptdiff] WARNING: ${warning}`);
+      }
+    }
+    const runOptions: RunnerRunOptions = {
+      systemPrompt,
+      systemPromptMode: delivery === "install" ? "append" : "replace",
+      userPrompt,
+      images,
+      model,
+      cwd: sandbox.dir,
+      addDirs: args.many("add-dir"),
+      tools,
+      timeoutMs: args.number("timeout-ms", DEFAULT_TIMEOUT_MS),
+      maxBudgetUsd: args.number("max-budget-usd", DEFAULT_MAX_BUDGET_USD),
+    };
+
+    console.error(
+      delivery === "install"
+        ? `[promptdiff] appended agent prompt: ${systemPrompt.length} chars (skills installed, not inlined)`
+        : `[promptdiff] system prompt: ${systemPrompt.length} chars (agent + ${args.many("skill").length} skill(s))`,
+    );
+    console.error(`[promptdiff] sandbox cwd: ${sandbox.dir}`);
+
+    const result = await runner.run(runOptions);
+    console.log("\n--- OUTPUT ---\n" + result.output);
+    console.log(
+      `\n--- $${result.costUsd.toFixed(4)} | ${result.turns} turns | ${(result.durationMs / 1000).toFixed(1)}s | models: ${result.models.join(", ")} ---`,
+    );
+    if (keepSandbox) {
+      console.log(`--- sandbox kept: ${sandbox.dir} ---`);
+    }
+  } finally {
+    sandbox.cleanup();
+  }
+}
+
+const measureSpecs: FlagSpecs = {
+  scenario: { arity: "one" },
+  agent: { arity: "one" },
+  skill: { arity: "one", repeat: true },
+  model: { arity: "one" },
+  runner: { arity: "one" },
+  "base-url": { arity: "one" },
+  runs: { arity: "one" },
+  mode: { arity: "one" },
+  sandbox: { arity: "one" },
+  seed: { arity: "one" },
+  "add-dir": { arity: "one", repeat: true },
+  tools: { arity: "one" },
+  "timeout-ms": { arity: "one" },
+  "max-budget-usd": { arity: "one" },
+  "keep-sandbox": { arity: "none" },
+  receipts: { arity: "one" },
+};
+
+async function cmdMeasure(argv: string[]): Promise<void> {
+  const args = parseArgs(argv, measureSpecs);
+  const scenario = args.one("scenario");
+  if (!scenario) {
+    throw new CliError(measureUsage());
+  }
+
+  const skills = args.many("skill");
+  const overrides: CompareOverrides = {
+    agent: args.one("agent"),
+    baselineSkills: skills.length > 0 ? skills : undefined,
+    // measure exercises the baseline arm only; mirroring keeps validation happy.
+    proposedSkills: skills.length > 0 ? skills : undefined,
+    model: args.one("model"),
+    runner: args.one("runner") ? runnerNameFromString(args.one("runner")) : undefined,
+    baseUrl: args.one("base-url"),
+    runs: args.has("runs") ? args.number("runs", 0) : undefined,
+    timeoutMs: args.has("timeout-ms") ? args.number("timeout-ms", DEFAULT_TIMEOUT_MS) : undefined,
+    maxBudgetUsd: args.has("max-budget-usd") ? args.number("max-budget-usd", DEFAULT_MAX_BUDGET_USD) : undefined,
+    mode: args.one("mode") ? modeFromString(args.one("mode")) : undefined,
+    tools: args.one("tools"),
+    addDirs: args.many("add-dir").length ? args.many("add-dir") : undefined,
+    sandboxRoot: args.one("sandbox"),
+    sandboxSeed: args.one("seed"),
+    keepSandbox: args.has("keep-sandbox") ? true : undefined,
+  };
+
+  const config = loadCompareConfig(scenario, overrides, { singleArm: true });
+  const summary = await runMeasure({
+    config,
+    runner: armRunner(config.arms.baseline, config),
+    onProgress: (message) => console.error(`[promptdiff] ${message}`),
+  });
+
+  const receiptsDir = args.one("receipts");
+  if (receiptsDir) {
+    const written = writeReceipts(receiptsDir, buildMeasureReceipts(summary, config, new Date().toISOString()));
+    console.error(`[promptdiff] wrote ${written.length} receipt(s) to ${receiptsDir}`);
+  }
+
+  console.log(formatMeasureSummary(summary));
+}
+
+const calibrateSpecs: FlagSpecs = {
+  rubric: { arity: "one" },
+  model: { arity: "one" },
+  runner: { arity: "one" },
+  "base-url": { arity: "one" },
+  "timeout-ms": { arity: "one" },
+  "max-budget-usd": { arity: "one" },
+};
+
+async function cmdCalibrate(argv: string[]): Promise<void> {
+  const args = parseArgs(argv, calibrateSpecs);
+  const rubric = args.one("rubric");
+  const model = args.one("model");
+  if (!rubric || !model) {
+    throw new CliError(calibrateUsage());
+  }
+
+  const result = await runCalibration({
+    rubric: resolve(rubric),
+    model,
+    runner: args.one("runner") ? runnerNameFromString(args.one("runner")) : "claude-p",
+    baseUrl: args.one("base-url"),
+    timeoutMs: args.number("timeout-ms", DEFAULT_TIMEOUT_MS),
+    maxBudgetUsd: args.number("max-budget-usd", DEFAULT_MAX_BUDGET_USD),
+    onProgress: (message) => console.error(`[promptdiff] ${message}`),
+  });
+
+  // Always exit 0: calibrate measures, the compare/measure gate enforces.
+  console.log(formatCalibrationReport(result));
+}
+
+async function cmdCompare(argv: string[]): Promise<number> {
+  const args = parseArgs(argv, compareSpecs);
+  const scenario = args.one("scenario");
+  if (!scenario) {
+    throw new CliError(compareUsage());
+  }
+  // Validate report flags before any paid run, not after.
+  const report = args.one("report");
+  const reportOut = args.one("report-out");
+  if (report !== undefined && report !== "ndjson") {
+    throw new CliError('--report supports only "ndjson"');
+  }
+  if (report === "ndjson" && !reportOut) {
+    throw new CliError("--report ndjson requires --report-out <file>");
+  }
+  if (reportOut && report === undefined) {
+    throw new CliError("--report-out requires --report ndjson");
+  }
+  // Caching is opt-in: a bare --cache-dir must not silently enable it.
+  if (args.one("cache-dir") !== undefined && !args.has("cache")) {
+    throw new CliError("--cache-dir requires --cache");
+  }
+  const cache = args.has("cache") ? { dir: args.one("cache-dir") ?? ".promptdiff/cache" } : undefined;
+
+  const overrides: CompareOverrides = {
+    agent: args.one("agent"),
+    baselineSkills: coalesceMany(args.many("baseline-skill"), args.many("baseline")),
+    proposedSkills: coalesceMany(args.many("proposed-skill"), args.many("proposed")),
+    delivery: args.one("delivery") ? deliveryFromString(args.one("delivery")) : undefined,
+    runner: args.one("runner") ? runnerNameFromString(args.one("runner")) : undefined,
+    baselineRunner: args.one("baseline-runner") ? runnerNameFromString(args.one("baseline-runner")) : undefined,
+    proposedRunner: args.one("proposed-runner") ? runnerNameFromString(args.one("proposed-runner")) : undefined,
+    baseUrl: args.one("base-url"),
+    model: args.one("model"),
+    baselineModel: args.one("baseline-model"),
+    proposedModel: args.one("proposed-model"),
+    runs: args.has("runs") ? args.number("runs", 0) : undefined,
+    timeoutMs: args.has("timeout-ms") ? args.number("timeout-ms", DEFAULT_TIMEOUT_MS) : undefined,
+    maxBudgetUsd: args.has("max-budget-usd") ? args.number("max-budget-usd", DEFAULT_MAX_BUDGET_USD) : undefined,
+    mode: args.one("mode") ? modeFromString(args.one("mode")) : undefined,
+    tools: args.one("tools"),
+    addDirs: args.many("add-dir").length ? args.many("add-dir") : undefined,
+    sandboxRoot: args.one("sandbox"),
+    sandboxSeed: args.one("seed"),
+    keepSandbox: args.has("keep-sandbox") ? true : undefined,
+  };
+
+  const config = loadCompareConfig(scenario, overrides);
+  const summary = await runCompare({
+    config,
+    runners: {
+      baseline: armRunner(config.arms.baseline, config),
+      proposed: armRunner(config.arms.proposed, config),
+    },
+    onProgress: (message) => console.error(`[promptdiff] ${message}`),
+    cache,
+  });
+
+  // History is appended before the exit code is decided — failed comparisons
+  // belong in the record just as much as passing ones.
+  if (report === "ndjson" && reportOut) {
+    const count = appendNdjsonReport(reportOut, summary);
+    console.error(`[promptdiff] appended ${count} record(s) to ${reportOut}`);
+  }
+  // Receipts too: a failing receipt is what stops the prompt from shipping.
+  const receiptsDir = args.one("receipts");
+  if (receiptsDir) {
+    const written = writeReceipts(receiptsDir, buildCompareReceipts(summary, config, new Date().toISOString()));
+    console.error(`[promptdiff] wrote ${written.length} receipt(s) to ${receiptsDir}`);
+  }
+
+  console.log(formatCompareSummary(summary));
+  return summary.failedAssertions.length > 0 ? 1 : 0;
+}
+
+function promptFromArgs(prompt: string | undefined, promptFile: string | undefined): string {
+  if (prompt && promptFile) {
+    throw new CliError("pass only one of --prompt or --prompt-file");
+  }
+  if (prompt) return prompt;
+  if (promptFile) return readFileSync(promptFile, "utf8");
+  throw new CliError(runUsage());
+}
+
+function modeFromString(value: string | undefined): RunMode {
+  if (value === "text" || value === "artifact") return value;
+  throw new CliError("--mode must be either text or artifact");
+}
+
+function deliveryFromString(value: string | undefined): Delivery {
+  if (value === "inline" || value === "install") return value;
+  throw new CliError("--delivery must be either inline or install");
+}
+
+export function varsFromFlags(entries: string[], baseDir = process.cwd()): RenderVars | undefined {
+  if (entries.length === 0) return undefined;
+  const raw: Record<string, unknown> = Object.create(null) as Record<string, unknown>;
+  for (const entry of entries) {
+    const eq = entry.indexOf("=");
+    if (eq <= 0) {
+      throw new CliError(`--var must be name=value (value is a file path or literal): ${entry}`);
+    }
+    const name = entry.slice(0, eq);
+    // A silently ignored duplicate is exactly the kind of surprise strict
+    // rendering exists to prevent.
+    if (Object.hasOwn(raw, name)) {
+      throw new CliError(`--var ${name} given more than once`);
+    }
+    raw[name] = entry.slice(eq + 1);
+  }
+  return resolveRenderVars(raw, baseDir, "--var");
+}
+
+function runnerNameFromString(value: string | undefined): RunnerName {
+  if (value === "claude-p" || value === "openai") return value;
+  throw new CliError(`--runner must be one of: ${RUNNER_NAMES.join(", ")}`);
+}
+
+function runnerFromArgs(name: string | undefined, baseUrl: string | undefined, price: string | undefined): Runner {
+  const runnerName = name === undefined ? "claude-p" : runnerNameFromString(name);
+  if (price !== undefined && runnerName !== "openai") {
+    throw new CliError("--price applies to the openai runner; claude-p prices itself");
+  }
+  return createRunner(runnerName, { baseUrl, pricing: price === undefined ? undefined : priceFromFlag(price) });
+}
+
+/** "--price 0.15,0.60" → USD per million input,output tokens. */
+function priceFromFlag(value: string): ModelPricing {
+  const [input, output, extra] = value.split(",").map((part) => Number(part));
+  if (extra !== undefined || !Number.isFinite(input) || !Number.isFinite(output) || input < 0 || output < 0) {
+    throw new CliError(`--price must be <input-usd-per-M>,<output-usd-per-M>: ${value}`);
+  }
+  return { input, output };
+}
+
+function armRunner(arm: ArmConfig, config: CompareConfig): Runner {
+  return createRunner(arm.runner, {
+    baseUrl: arm.baseUrl,
+    retries: config.retries,
+    pricing: config.pricing?.[arm.model],
+  });
+}
+
+function defaultTools(mode: RunMode): string {
+  return mode === "text" ? "" : "default";
+}
+
+function coalesceMany(primary: string[], fallback: string[]): string[] | undefined {
+  if (primary.length > 0) return primary;
+  if (fallback.length > 0) return fallback;
+  return undefined;
+}
+
+function runUsage(): string {
+  return [
+    "usage: promptdiff run --agent <file.md> --model <model> (--prompt <text>|--prompt-file <file>) [flags]",
+    "",
+    "Runs one bounded model invocation. Use this to inspect whether one agent +",
+    "skill set behaves roughly as expected before promoting the fixture to compare.",
+    "",
+    "flags: --skill <SKILL.md|dir>... --delivery <inline|install> --mode <text|artifact>",
+    "       --runner <claude-p|openai> --base-url <url> --image <file>...",
+    "       --var <name=value>... --sandbox <dir> --seed <dir>",
+    "       --tools <tools|default|''> --timeout-ms <ms> --max-budget-usd <usd>",
+    "",
+    "templates:",
+    "  --var draft=./fixture.md binds {{draft}} in the agent, skills, and prompt;",
+    "  values naming an existing file are read as contents, otherwise used as",
+    "  literals. With any --var set, unbound {{placeholders}} fail before the run.",
+    "",
+    "runners:",
+    "  claude-p  (default) headless Claude Code — supports tools, artifact mode,",
+    "            and install delivery",
+    "  openai    single chat completion against any OpenAI-compatible endpoint",
+    "            (text mode only); --base-url or $OPENAI_BASE_URL picks the server,",
+    "            $OPENAI_API_KEY is sent when set; --image attaches image files",
+    "            to the user message for vision models (repeatable);",
+    "            --price <in>,<out> (USD per million tokens) prices the run from",
+    "            response usage and makes --max-budget-usd enforce",
+    "",
+    "modes:",
+    "  text      disables tools with --tools '' and grades only the final output manually",
+    "  artifact  runs the agent in a fresh sandbox cwd with default tools enabled",
+    "",
+    "delivery:",
+    "  inline    (default) skill bodies are inlined into a replaced system prompt —",
+    "            controlled compliance testing of skill text the model already sees",
+    "  install   skill dirs are copied to <sandbox>/.claude/skills/<name> with",
+    "            frontmatter intact and the agent text is appended to the default",
+    "            system prompt — tests whether the registry description actually",
+    "            triggers the Skill invocation (implies --mode artifact + tools)",
+  ].join("\n");
+}
+
+function compareUsage(): string {
+  return [
+    "usage: promptdiff compare --scenario <scenario.json> [overrides]",
+    "",
+    "Runs baseline and proposed arms against the same scenarios, then grades",
+    "each run deterministically and compares pass rates. The arms may differ by",
+    "skill set, by model/runner, or both.",
+    "",
+    "overrides: --agent <file.md> --baseline <SKILL.md>... --proposed <SKILL.md>...",
+    "           --delivery <inline|install> (install: skills land in the sandbox",
+    "           registry with frontmatter intact instead of being inlined)",
+    "           --runner <claude-p|openai> --base-url <url> (openai: text-graded",
+    "           scenarios against any OpenAI-compatible endpoint)",
+    "           --model <model> --runs <n> --sandbox <dir> --keep-sandbox",
+    "           --baseline-model <m> --proposed-model <m>",
+    "           --baseline-runner <r> --proposed-runner <r>",
+    "           --mode <text|artifact> --tools <tools|default|''>",
+    "           --timeout-ms <ms> --max-budget-usd <usd>",
+    "           --report ndjson --report-out <file>",
+    "           --cache [--cache-dir <dir>]",
+    "",
+    "caching:",
+    "  --cache reuses recorded baseline-arm results (default dir .promptdiff/cache)",
+    "  when nothing that could change the outcome changed: rendered prompts, skill",
+    "  and fixture contents, model/runner, run count, tools/mode/delivery, grader.",
+    "  Opt-in only; the proposed arm always runs fresh. Delete the dir to bust.",
+    "",
+    "report:",
+    "  --report ndjson appends one record per scenario to --report-out: arms,",
+    "  pass rates, cost, sampling p, prompt hashes, productionModel. Append-only",
+    "  history that answers \"has this drifted since July?\" without hand notes.",
+    "",
+    "receipts:",
+    "  --receipts <dir> writes one <scenario>.receipt.json per scenario",
+    "  (overwritten each run): per-file sha256 of the agent and every skill,",
+    "  arm results, and a pass/fail verdict. CI in a consuming repo can then",
+    "  assert every shipped prompt has a passing receipt for its CURRENT hash —",
+    "  editing a prompt stales its receipt and names the scenario to re-run.",
+    "",
+    "model comparison:",
+    "  hold the skills constant and vary the model per arm. A top-level \"skills\"",
+    "  array is inherited by both arms, and the record form of \"baseline\"/",
+    "  \"proposed\" accepts per-arm \"model\", \"runner\", and \"baseUrl\" (each falls",
+    "  back to the shared top-level value). --baseline-model/--proposed-model and",
+    "  --baseline-runner/--proposed-runner override per arm from the CLI.",
+    "",
+    "runs:",
+    "  scenario-level \"runs\": 5 means 5 baseline runs and 5 proposed runs per case.",
+    "  --runs <n> overrides the scenario default. A case may also define its own runs.",
+    "",
+    "graders:",
+    "  text     checks the run's final output with contains, notContains, and regex arrays",
+    "  json     parses the LAST balanced JSON value in the run's output and checks",
+    "           \"assert\" path assertions: <path> <op> <literal>, e.g.",
+    "           \"findings.items.length == 0\" or \"items[*].domain contains \\\"correctness\\\"\"",
+    "           (ops: == != > >= < <= contains; [*] passes if ANY element matches;",
+    "           works with every runner)",
+    "  command  runs a shell command inside the per-run sandbox and checks exit code",
+    "           (needs a tool-capable runner: claude-p)",
+    "  judge    an explicit judge model grades the final output against a markdown",
+    "           rubric: { \"type\": \"judge\", \"rubric\": \"./rubrics/r.md\", \"model\": <m>,",
+    "           \"runner\": <claude-p|openai>, \"baseUrl\"?, \"minAccuracy\"? (default 0.9) }.",
+    "           Refuses to grade until `promptdiff calibrate` has proven the judge",
+    "           against labeled fixtures (see `promptdiff calibrate --help`). Adds",
+    "           one billed model call per graded run.",
+    "",
+    "images:",
+    "  a scenario may set \"images\": [\"photo.jpg\", ...] (paths relative to the",
+    "  scenario file) to attach images to the user message — openai runner +",
+    "  vision model only",
+    "",
+    "pricing:",
+    "  top-level \"pricing\": { \"gpt-4o-mini\": { \"input\": 0.15, \"output\": 0.60 } }",
+    "  (USD per million tokens) prices openai arms from response usage, so cost",
+    "  columns are real and maxBudgetUsd enforces. Unpriced openai arms report $0",
+    "  (true for local servers). claude-p arms price themselves.",
+    "",
+    "templates:",
+    "  top-level \"render\": { \"vars\": { \"draft\": \"./fixtures/a.md\" } } binds",
+    "  {{draft}} in the agent, inlined skills, and scenario prompts, so scenarios",
+    "  can point at production prompt files with placeholders. Values naming an",
+    "  existing file (relative to the scenario) are read as contents, otherwise",
+    "  used as literals. Scenarios may add their own \"render\" (scenario wins per",
+    "  var). Unbound placeholders fail before any paid run. Inline delivery only.",
+    "",
+    "assertions:",
+    "  target      baseline must not fully pass; proposed must beat baseline pass rate",
+    "  regression  proposed must not fall below baseline pass rate",
+    "  compare     no assertion — reports both arms' pass rates and the delta",
+    "              (for model-vs-model diffs with no directional claim)",
+    "",
+    "minimal scenario:",
+    "  {",
+    "    \"agent\": \"./agent.md\",",
+    "    \"baselineSkills\": [\"./skill.baseline.md\"],",
+    "    \"proposedSkills\": [\"./skill.proposed.md\"],",
+    "    \"model\": \"sonnet\",",
+    "    \"runs\": 5,",
+    "    \"scenarios\": [{",
+    "      \"name\": \"target-case\",",
+    "      \"kind\": \"target\",",
+    "      \"prompt\": \"Do the fixture task.\",",
+    "      \"grader\": { \"type\": \"command\", \"command\": \"bun test\" }",
+    "    }]",
+    "  }",
+  ].join("\n");
+}
+
+function measureUsage(): string {
+  return [
+    "usage: promptdiff measure --scenario <scenario.json> [overrides]",
+    "",
+    "Characterizes ONE instruction set: runs every scenario N times, grades",
+    "deterministically, and reports per-case pass rates — no delta, no",
+    "assertions. Use it to know the current survival rate before changing",
+    "anything; faking this with identical compare arms produces nonsense",
+    "verdicts from sampling noise.",
+    "",
+    "The scenario file is the compare format; measure exercises the shared",
+    "`skills` set (or `baselineSkills`) and needs no proposed arm. All compare",
+    "config applies: render vars, images, pricing, productionModel, graders.",
+    "",
+    "overrides: --agent <file.md> --skill <SKILL.md|dir>... --model <model>",
+    "           --runner <claude-p|openai> --base-url <url> --runs <n>",
+    "           --mode <text|artifact> --tools <tools|default|''>",
+    "           --sandbox <dir> --seed <dir> --keep-sandbox",
+    "           --timeout-ms <ms> --max-budget-usd <usd> --receipts <dir>",
+    "",
+    "--receipts <dir> writes one <scenario>.receipt.json per scenario with",
+    "per-file prompt hashes and the measured rates (verdict \"measured\").",
+    "",
+    "Exit code is 0 whenever the runs complete — a measurement has no pass/fail.",
+  ].join("\n");
+}
+
+function calibrateUsage(): string {
+  return [
+    "usage: promptdiff calibrate --rubric <rubric.md> --model <judge-model> [flags]",
+    "",
+    "Measures a judge (rubric + model + runner) against labeled fixtures and",
+    "writes <rubric>.calibration.json next to the rubric. compare/measure refuse",
+    "to use a judge grader until this record exists, matches the current rubric",
+    "content (sha256) and judge model/runner, and clears minAccuracy on BOTH",
+    "classes — an uncalibrated judge is worse than the regex it replaces: same",
+    "wrongness, more confidence, higher cost.",
+    "",
+    "flags: --runner <claude-p|openai> --base-url <url>",
+    "       --timeout-ms <ms> --max-budget-usd <usd>",
+    "",
+    "fixtures (sibling directory named after the rubric):",
+    "  rubrics/negate-restate.md",
+    "  rubrics/negate-restate.fixtures/pass/*.md   outputs the judge must call clean",
+    "  rubrics/negate-restate.fixtures/fail/*.md   outputs the judge must flag",
+    "",
+    "Both classes need at least one fixture. Accuracy is reported per class on",
+    "purpose: a judge that passes everything scores 100% on the pass class and",
+    "0% on the fail class — overall accuracy would hide it.",
+    "",
+    "Exit code is 0 whenever the fixtures were judged — even a failing",
+    "calibration is recorded; the compare/measure gate is what enforces the bar.",
+    "Editing the rubric invalidates the record (content hash); recalibrate after",
+    "every rubric change.",
+  ].join("\n");
+}
+
+function generalUsage(): string {
+  return [
+    "usage: promptdiff <run|compare|measure|calibrate> [flags]",
+    "",
+    "commands:",
+    "  run        one bounded model invocation with inlined skills",
+    "  compare    N-run baseline-vs-proposed scenario comparison",
+    "  measure    N-run single-arm characterization (pass rates, no assertions)",
+    "  calibrate  measure a judge grader against labeled rubric fixtures",
+    "",
+    "use `promptdiff <command> --help` for command flags",
+  ].join("\n");
+}