@theaiteam/promptdiff 1.0.0-rc.1

package/src/engine/receipt.ts

@@ -0,0 +1,150 @@
+import { createHash } from "node:crypto";
+import { mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
+import { join, relative, resolve } from "node:path";
+import type { CompareConfig } from "./config";
+import type { CompareSummary, MeasureSummary } from "./compare";
+
+/**
+ * A receipt asserts "the prompt with THIS content hash produced THIS eval
+ * result" — the content-addressed replacement for hand-maintained
+ * prompt_version strings. A consuming repo's CI can check that every prompt
+ * it ships has a passing receipt for its current hash; editing the prompt
+ * changes the hash and stales the receipt, naming the scenario to re-run.
+ */
+export interface PromptDigest {
+  path: string;
+  sha256: string;
+}
+
+export interface Receipt {
+  comparison: string;
+  scenario: string;
+  kind: string;
+  command: "compare" | "measure";
+  ranAt: string;
+  verdict: "pass" | "fail" | "none" | "measured";
+  prompts: {
+    agent: PromptDigest;
+    baselineSkills?: PromptDigest[];
+    proposedSkills?: PromptDigest[];
+    skills?: PromptDigest[];
+  };
+  /** Hash of the rendered system prompt(s) — also pins render-var fixtures. */
+  renderedPromptSha256?: unknown;
+  results: Record<string, { passes: number; totalRuns: number; passRate: number; costUsd: number }>;
+  samplingP?: number;
+  productionModel?: string;
+}
+
+export function buildCompareReceipts(
+  summary: CompareSummary,
+  config: CompareConfig,
+  timestamp: string,
+): Receipt[] {
+  const agent = digestPath(config.agent);
+  const baselineSkills = config.baselineSkills.map(digestPath);
+  const proposedSkills = config.proposedSkills.map(digestPath);
+
+  return summary.cases.map((caseSummary) => ({
+    comparison: summary.name,
+    scenario: caseSummary.name,
+    kind: caseSummary.kind,
+    command: "compare",
+    ranAt: timestamp,
+    // "compare" kind carries no directional claim, so its receipt cannot say pass.
+    verdict:
+      caseSummary.assertions.length > 0 ? "fail" : caseSummary.kind === "compare" ? "none" : "pass",
+    prompts: { agent, baselineSkills, proposedSkills },
+    renderedPromptSha256: caseSummary.promptSha256,
+    results: {
+      baseline: armResult(caseSummary.baseline),
+      proposed: armResult(caseSummary.proposed),
+    },
+    samplingP: caseSummary.samplingP,
+    productionModel: summary.productionModel,
+  }));
+}
+
+export function buildMeasureReceipts(
+  summary: MeasureSummary,
+  config: CompareConfig,
+  timestamp: string,
+): Receipt[] {
+  const agent = digestPath(config.agent);
+  const skills = config.baselineSkills.map(digestPath);
+
+  return summary.cases.map((caseSummary) => ({
+    comparison: summary.name,
+    scenario: caseSummary.name,
+    kind: caseSummary.kind,
+    command: "measure",
+    ranAt: timestamp,
+    verdict: "measured",
+    prompts: { agent, skills },
+    renderedPromptSha256: caseSummary.promptSha256,
+    results: { measured: armResult(caseSummary.result) },
+    productionModel: summary.productionModel,
+  }));
+}
+
+/**
+ * Writes one `<scenario-slug>.receipt.json` per receipt, overwriting — a
+ * receipt is current state; append-only history is `--report ndjson`'s job.
+ * Returns the written paths.
+ */
+export function writeReceipts(dir: string, receipts: Receipt[]): string[] {
+  const target = resolve(dir);
+  mkdirSync(target, { recursive: true });
+  return receipts.map((receipt) => {
+    const path = join(target, `${slug(receipt.scenario)}.receipt.json`);
+    writeFileSync(path, JSON.stringify(receipt, null, 2) + "\n", "utf8");
+    return path;
+  });
+}
+
+function armResult(arm: { passes: number; totalRuns: number; passRate: number; totalCostUsd: number }) {
+  return { passes: arm.passes, totalRuns: arm.totalRuns, passRate: arm.passRate, costUsd: arm.totalCostUsd };
+}
+
+/** Repo-relative path + content hash; install-delivery skill dirs get a deterministic tree hash. */
+function digestPath(absolutePath: string): PromptDigest {
+  return { path: relative(process.cwd(), absolutePath), sha256: hashPath(absolutePath) };
+}
+
+/** Content hash of a prompt file, or a deterministic tree hash for a skill directory. */
+export function contentHash(path: string): string {
+  return hashPath(path);
+}
+
+function hashPath(path: string): string {
+  if (!statSync(path).isDirectory()) {
+    return createHash("sha256").update(readFileSync(path)).digest("hex");
+  }
+  // Directory (install-delivery skill): hash every file, sorted by relative
+  // path, so any supporting-file edit stales the receipt too.
+  const hash = createHash("sha256");
+  for (const entry of walkSorted(path, path)) {
+    hash.update(entry);
+    hash.update("\0");
+    hash.update(readFileSync(join(path, entry)));
+    hash.update("\0");
+  }
+  return hash.digest("hex");
+}
+
+function walkSorted(root: string, dir: string): string[] {
+  const entries: string[] = [];
+  for (const name of readdirSync(dir).sort()) {
+    const full = join(dir, name);
+    if (statSync(full).isDirectory()) {
+      entries.push(...walkSorted(root, full));
+    } else {
+      entries.push(relative(root, full));
+    }
+  }
+  return entries;
+}
+
+function slug(name: string): string {
+  return name.toLowerCase().replace(/[^a-z0-9_-]+/g, "-").replace(/^-+|-+$/g, "") || "scenario";
+}

package/src/engine/render.ts

@@ -0,0 +1,59 @@
+import { existsSync, readFileSync, statSync } from "node:fs";
+import { resolve } from "node:path";
+
+const PLACEHOLDER = /\{\{\s*([A-Za-z0-9_][A-Za-z0-9_.-]*)\s*\}\}/g;
+
+export type RenderVars = Record<string, string>;
+
+/**
+ * Resolves raw `render.vars` values: a value naming an existing file (relative
+ * to baseDir) becomes that file's contents; anything else stays a literal.
+ * File reads happen at load time so a missing fixture fails before any paid run.
+ */
+export function resolveRenderVars(
+  raw: Record<string, unknown>,
+  baseDir: string,
+  label: string,
+): RenderVars {
+  // Null prototype so a var named "__proto__" binds as a normal own property
+  // instead of silently vanishing into a prototype assignment.
+  const vars: RenderVars = Object.create(null) as RenderVars;
+  for (const [name, value] of Object.entries(raw)) {
+    if (typeof value !== "string") {
+      throw new Error(`${label}.${name} must be a string (file path or literal)`);
+    }
+    const path = resolve(baseDir, value);
+    vars[name] = existsSync(path) && statSync(path).isFile() ? readFileSync(path, "utf8") : value;
+  }
+  return vars;
+}
+
+export function placeholderNames(text: string): string[] {
+  return [...new Set([...text.matchAll(PLACEHOLDER)].map((match) => match[1]))];
+}
+
+/**
+ * Substitutes {{name}} placeholders and fails on any left unbound — sending a
+ * literal `{{draft}}` to a model produces a confusing pass/fail, not an error.
+ * Unbound detection runs on the ORIGINAL text: substituted values are never
+ * re-scanned, so a fixture that itself contains braces cannot false-positive
+ * (and there is no recursive expansion). Own-property checks throughout:
+ * `in`/bracket lookups would let an unbound {{toString}} or {{constructor}}
+ * pass the strict check and inject native-function source into a paid prompt.
+ */
+export function renderStrict(
+  text: string,
+  vars: RenderVars,
+  label: string,
+  hint = "bind them in render.vars",
+): string {
+  const unbound = placeholderNames(text).filter((name) => !Object.hasOwn(vars, name));
+  if (unbound.length > 0) {
+    throw new Error(
+      `unbound template placeholder(s) in ${label}: ${unbound.map((name) => `{{${name}}}`).join(", ")} — ${hint}`,
+    );
+  }
+  return text.replace(PLACEHOLDER, (match, name: string) =>
+    Object.hasOwn(vars, name) ? vars[name] : match,
+  );
+}

package/src/engine/report.ts

@@ -0,0 +1,49 @@
+import { appendFileSync, mkdirSync } from "node:fs";
+import { dirname, resolve } from "node:path";
+import type { ArmConfig } from "./config";
+import type { ArmSummary, CompareSummary } from "./compare";
+
+/**
+ * One NDJSON record per scenario per invocation, append-only: durable,
+ * greppable history that can answer "has the catch rate drifted" without
+ * hand-transcribing summaries into READMEs.
+ */
+export function buildReportRecords(summary: CompareSummary, timestamp: string): Array<Record<string, unknown>> {
+  return summary.cases.map((caseSummary) => ({
+    ts: timestamp,
+    comparison: summary.name,
+    scenario: caseSummary.name,
+    kind: caseSummary.kind,
+    baseline: armRecord(caseSummary.baseline, summary.arms.baseline),
+    proposed: armRecord(caseSummary.proposed, summary.arms.proposed),
+    deltaPassRate: caseSummary.proposed.passRate - caseSummary.baseline.passRate,
+    samplingP: caseSummary.samplingP,
+    failedAssertions: caseSummary.assertions,
+    passed: caseSummary.assertions.length === 0,
+    promptSha256: caseSummary.promptSha256,
+    productionModel: summary.productionModel,
+  }));
+}
+
+export function appendNdjsonReport(
+  path: string,
+  summary: CompareSummary,
+  timestamp = new Date().toISOString(),
+): number {
+  const records = buildReportRecords(summary, timestamp);
+  const target = resolve(path);
+  mkdirSync(dirname(target), { recursive: true });
+  appendFileSync(target, records.map((record) => JSON.stringify(record) + "\n").join(""), "utf8");
+  return records.length;
+}
+
+function armRecord(arm: ArmSummary, config: ArmConfig): Record<string, unknown> {
+  return {
+    model: config.model,
+    runner: config.runner,
+    passes: arm.passes,
+    totalRuns: arm.totalRuns,
+    passRate: arm.passRate,
+    costUsd: arm.totalCostUsd,
+  };
+}

package/src/engine/sandbox.ts

@@ -0,0 +1,97 @@
+import {
+  cpSync,
+  existsSync,
+  mkdirSync,
+  readdirSync,
+  realpathSync,
+  rmSync,
+  statSync,
+} from "node:fs";
+import { basename, join, resolve, sep } from "node:path";
+
+export interface SandboxSpec {
+  root: string;
+  seed?: string;
+  prefix: string;
+  keep: boolean;
+}
+
+export interface PreparedSandbox {
+  dir: string;
+  cleanup(): void;
+}
+
+export function prepareSandbox(spec: SandboxSpec): PreparedSandbox {
+  const root = resolve(spec.root);
+  mkdirSync(root, { recursive: true });
+  assertSeedIsUsable(spec.seed, root);
+
+  const dir = createUniqueDir(root, spec.prefix);
+  if (spec.seed) {
+    copyDirectoryContents(resolve(spec.seed), dir);
+  }
+
+  return {
+    dir,
+    cleanup() {
+      if (!spec.keep) {
+        rmSync(dir, { recursive: true, force: true });
+      }
+    },
+  };
+}
+
+function createUniqueDir(root: string, prefix: string): string {
+  const safePrefix = prefix.replace(/[^a-zA-Z0-9._-]/g, "-").slice(0, 60) || "run";
+  let attempt = 0;
+  while (attempt < 1_000) {
+    const suffix = `${Date.now()}-${process.pid}-${attempt}`;
+    const dir = join(root, `${safePrefix}-${suffix}`);
+    try {
+      mkdirSync(dir);
+      return dir;
+    } catch (error) {
+      if (isAlreadyExists(error)) {
+        attempt += 1;
+        continue;
+      }
+      throw error;
+    }
+  }
+  throw new Error(`could not create unique sandbox in ${root}`);
+}
+
+function assertSeedIsUsable(seed: string | undefined, root: string): void {
+  if (!seed) return;
+  const resolvedSeed = resolve(seed);
+  if (!existsSync(resolvedSeed)) {
+    throw new Error(`sandbox seed does not exist: ${seed}`);
+  }
+  if (!statSync(resolvedSeed).isDirectory()) {
+    throw new Error(`sandbox seed must be a directory: ${seed}`);
+  }
+
+  const realSeed = realpathSync(resolvedSeed);
+  const realRoot = realpathSync(root);
+  if (realRoot === realSeed || realRoot.startsWith(realSeed + sep)) {
+    throw new Error(`sandbox root must not be inside the seed directory: ${root}`);
+  }
+}
+
+function copyDirectoryContents(seed: string, destination: string): void {
+  for (const entry of readdirSync(seed)) {
+    cpSync(join(seed, entry), join(destination, basename(entry)), {
+      recursive: true,
+      errorOnExist: false,
+    });
+  }
+}
+
+function isAlreadyExists(error: unknown): boolean {
+  return (
+    typeof error === "object" &&
+    error !== null &&
+    "code" in error &&
+    (error as { code?: string }).code === "EEXIST"
+  );
+}

package/src/engine/skill-install.ts

@@ -0,0 +1,112 @@
+import { cpSync, existsSync, mkdirSync, readFileSync, statSync } from "node:fs";
+import { homedir } from "node:os";
+import { basename, dirname, join, resolve } from "node:path";
+
+export type Delivery = "inline" | "install";
+
+export interface ResolvedSkill {
+  /** Directory that contains SKILL.md (and any supporting files). */
+  dir: string;
+  /** Registry name: frontmatter `name:` if present, else the directory basename. */
+  name: string;
+}
+
+export interface InstallResult {
+  installed: ResolvedSkill[];
+  warnings: string[];
+}
+
+/**
+ * Accepts either a skill directory or a path to its SKILL.md and returns the
+ * directory plus the name Claude Code will register it under.
+ */
+export function resolveSkill(path: string): ResolvedSkill {
+  const resolved = resolve(path);
+  if (!existsSync(resolved)) {
+    throw new Error(`skill path does not exist: ${path}`);
+  }
+
+  const dir = statSync(resolved).isDirectory() ? resolved : dirname(resolved);
+  const skillFile = join(dir, "SKILL.md");
+  if (!existsSync(skillFile)) {
+    throw new Error(`skill directory has no SKILL.md: ${dir}`);
+  }
+
+  return { dir, name: skillNameFrom(skillFile) ?? basename(dir) };
+}
+
+/**
+ * Copies skill directories into the sandbox's project skills path
+ * (<sandbox>/.claude/skills/<name>) with frontmatter intact, so headless Claude
+ * discovers them through the normal registry instead of inlined prompt text.
+ */
+export function installSkills(
+  skillPaths: string[],
+  sandboxDir: string,
+  userSkillsDir = defaultUserSkillsDir(),
+): InstallResult {
+  const installed: ResolvedSkill[] = [];
+  const warnings: string[] = [];
+  const seen = new Set<string>();
+
+  for (const path of skillPaths) {
+    const skill = resolveSkill(path);
+    if (seen.has(skill.name)) {
+      throw new Error(`duplicate skill name in one arm: ${skill.name}`);
+    }
+    seen.add(skill.name);
+
+    const destination = join(sandboxDir, ".claude", "skills", skill.name);
+    mkdirSync(dirname(destination), { recursive: true });
+    cpSync(skill.dir, destination, { recursive: true });
+    installed.push(skill);
+
+    // A user-level skill with the same name loads in every run of every arm,
+    // contaminating the comparison. Detection only; unlinking is the operator's call.
+    if (existsSync(join(userSkillsDir, skill.name))) {
+      warnings.push(
+        `user-level skill "${skill.name}" exists at ${join(userSkillsDir, skill.name)} and will also load in both arms — remove or rename it for a clean comparison`,
+      );
+    }
+
+    // Project skills with over-limit descriptions are silently dropped from the
+    // registry — the arm then runs skill-less and scores 0 for the wrong reason.
+    const descriptionLength = skillDescriptionLength(join(skill.dir, "SKILL.md"));
+    if (descriptionLength > MAX_DESCRIPTION_CHARS) {
+      warnings.push(
+        `skill "${skill.name}" has a ${descriptionLength}-char description (limit ${MAX_DESCRIPTION_CHARS}) — Claude Code drops over-limit project skills silently, so this arm would run without it`,
+      );
+    }
+  }
+
+  return { installed, warnings };
+}
+
+export function deliveryValue(value: unknown, fallback: Delivery): Delivery {
+  if (value === undefined) return fallback;
+  if (value === "inline" || value === "install") return value;
+  throw new Error(`delivery must be "inline" or "install"`);
+}
+
+const MAX_DESCRIPTION_CHARS = 1024;
+
+function skillDescriptionLength(skillFile: string): number {
+  const content = readFileSync(skillFile, "utf8");
+  const frontmatter = content.match(/^---[ \t]*(?:\r?\n)([\s\S]*?)(?:\r?\n)---/);
+  if (!frontmatter) return 0;
+  const description = frontmatter[1].match(/^description:\s*(.+)$/m);
+  return description ? description[1].length : 0;
+}
+
+function skillNameFrom(skillFile: string): string | undefined {
+  const content = readFileSync(skillFile, "utf8");
+  const frontmatter = content.match(/^---[ \t]*(?:\r?\n)([\s\S]*?)(?:\r?\n)---/);
+  if (!frontmatter) return undefined;
+  const name = frontmatter[1].match(/^name:\s*["']?([A-Za-z0-9][A-Za-z0-9_-]*)["']?\s*$/m);
+  return name?.[1];
+}
+
+function defaultUserSkillsDir(): string {
+  const configDir = process.env.CLAUDE_CONFIG_DIR ?? join(homedir(), ".claude");
+  return join(configDir, "skills");
+}

package/src/engine/stats.ts

@@ -0,0 +1,41 @@
+/**
+ * Two-tailed Fisher's exact test on a 2x2 pass/fail table. Exact enumeration
+ * is cheap at eval-sized n and avoids normal-approximation lies at n=3, which
+ * is exactly where the noise warning matters.
+ */
+export function fisherExactTwoTailedP(
+  baselinePasses: number,
+  baselineTotal: number,
+  proposedPasses: number,
+  proposedTotal: number,
+): number {
+  const passColumn = baselinePasses + proposedPasses;
+  const n = baselineTotal + proposedTotal;
+  const kMin = Math.max(0, passColumn - proposedTotal);
+  const kMax = Math.min(baselineTotal, passColumn);
+
+  const logTableProb = (k: number) =>
+    logChoose(baselineTotal, k) + logChoose(proposedTotal, passColumn - k) - logChoose(n, passColumn);
+
+  const observed = logTableProb(baselinePasses);
+  let total = 0;
+  for (let k = kMin; k <= kMax; k += 1) {
+    const logP = logTableProb(k);
+    // Two-tailed by summing every table at most as probable as the observed one.
+    if (logP <= observed + 1e-9) total += Math.exp(logP);
+  }
+  return Math.min(1, total);
+}
+
+function logChoose(n: number, k: number): number {
+  return logFactorial(n) - logFactorial(k) - logFactorial(n - k);
+}
+
+const logFactorialCache: number[] = [0];
+
+function logFactorial(n: number): number {
+  for (let i = logFactorialCache.length; i <= n; i += 1) {
+    logFactorialCache[i] = logFactorialCache[i - 1] + Math.log(i);
+  }
+  return logFactorialCache[n];
+}

package/src/prompt.ts

@@ -0,0 +1,16 @@
+import { readFileSync } from "node:fs";
+
+export function stripFrontmatter(markdown: string): string {
+  const match = markdown.match(/^---[ \t]*(?:\r?\n)[\s\S]*?(?:\r?\n)---[ \t]*(?:\r?\n|$)/);
+  if (!match) return markdown;
+  return markdown.slice(match[0].length);
+}
+
+export function assembleSystemPrompt(agentPath: string, skillPaths: string[]): string {
+  const agentBody = stripFrontmatter(readFileSync(agentPath, "utf8"));
+  const skills = skillPaths.map((path) => {
+    const body = stripFrontmatter(readFileSync(path, "utf8"));
+    return `\n\n===== SKILL (inlined for eval): ${path} =====\n${body}`;
+  });
+  return agentBody + skills.join("");
+}

package/src/runner/claude-p.ts

@@ -0,0 +1,156 @@
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import type { RunResult, Runner, RunnerRunOptions } from "../types";
+
+interface ClaudeJsonResult {
+  result?: unknown;
+  subtype?: unknown;
+  total_cost_usd?: unknown;
+  num_turns?: unknown;
+  duration_ms?: unknown;
+  modelUsage?: unknown;
+  errors?: unknown;
+}
+
+export function buildClaudeArgs(options: RunnerRunOptions & { systemPromptFile: string }): string[] {
+  const systemPromptArgs =
+    options.systemPromptMode === "append"
+      ? options.systemPrompt.trim().length > 0
+        ? ["--append-system-prompt", options.systemPrompt]
+        : []
+      : ["--system-prompt-file", options.systemPromptFile];
+
+  // "--tools" controls availability only; an explicit tool list still hits
+  // permission denials in headless mode (Bash especially). Granting the same
+  // list via --allowedTools makes list-mode evals actually able to act.
+  const toolArgs =
+    options.tools === "" || options.tools === "default"
+      ? ["--tools", options.tools]
+      : ["--tools", options.tools, "--allowedTools", options.tools];
+
+  const args = [
+    "-p",
+    options.userPrompt,
+    ...systemPromptArgs,
+    "--output-format",
+    "json",
+    "--model",
+    options.model,
+    ...toolArgs,
+    "--max-budget-usd",
+    String(options.maxBudgetUsd),
+    "--no-session-persistence",
+    // Headless denies file edits without an explicit permission mode, which breaks
+    // artifact-mode agents that must write outputs (e.g. findings.json) into the
+    // sandbox. acceptEdits is safe: text mode passes --tools "" so nothing can write,
+    // and artifact mode's cwd IS the disposable sandbox.
+    "--permission-mode",
+    "acceptEdits",
+  ];
+
+  if (options.addDirs.length > 0) {
+    args.push("--add-dir", ...options.addDirs);
+  }
+
+  return args;
+}
+
+export class ClaudePrintRunner implements Runner {
+  readonly name = "claude-p";
+  readonly capabilities = { sandboxTools: true, skillRegistry: true, images: false };
+
+  constructor(private readonly claudeBin = "claude") {}
+
+  async run(options: RunnerRunOptions): Promise<RunResult> {
+    const promptDir = mkdtempSync(join(tmpdir(), "promptdiff-prompt-"));
+    const systemPromptFile = join(promptDir, "system-prompt.md");
+    writeFileSync(systemPromptFile, options.systemPrompt, "utf8");
+
+    try {
+      const args = buildClaudeArgs({ ...options, systemPromptFile });
+      const proc = Bun.spawn([this.claudeBin, ...args], {
+        cwd: options.cwd,
+        stdout: "pipe",
+        stderr: "pipe",
+      });
+
+      let timedOut = false;
+      const timeout = setTimeout(() => {
+        timedOut = true;
+        proc.kill("SIGTERM");
+        setTimeout(() => proc.kill("SIGKILL"), 2_000);
+      }, options.timeoutMs);
+
+      const [stdout, stderr, code] = await Promise.all([
+        new Response(proc.stdout).text(),
+        new Response(proc.stderr).text(),
+        proc.exited,
+      ]);
+      clearTimeout(timeout);
+
+      if (timedOut) {
+        throw new Error(`claude timed out after ${options.timeoutMs}ms`);
+      }
+      if (code !== 0) {
+        throw new Error(describeClaudeFailure(code, stdout, stderr, options.maxBudgetUsd));
+      }
+
+      let parsed: ClaudeJsonResult;
+      try {
+        parsed = JSON.parse(stdout) as ClaudeJsonResult;
+      } catch (error) {
+        const reason = error instanceof Error ? error.message : String(error);
+        throw new Error(`claude returned invalid JSON: ${reason}\n${stdout.slice(0, 1_500)}`);
+      }
+
+      return normalizeClaudeResult(parsed);
+    } finally {
+      rmSync(promptDir, { recursive: true, force: true });
+    }
+  }
+}
+
+/**
+ * A budget abort exits 1 with EMPTY stderr but a full JSON result on stdout
+ * ("subtype": "error_max_budget_usd") — without decoding it, the failure is
+ * indistinguishable from a crash and has cost real debug cycles.
+ */
+export function describeClaudeFailure(
+  code: number,
+  stdout: string,
+  stderr: string,
+  maxBudgetUsd: number,
+): string {
+  let parsed: ClaudeJsonResult | undefined;
+  try {
+    parsed = JSON.parse(stdout) as ClaudeJsonResult;
+  } catch {
+    // Not JSON — fall through to the raw-stream message.
+  }
+
+  if (parsed && typeof parsed === "object") {
+    if (parsed.subtype === "error_max_budget_usd") {
+      const spent = typeof parsed.total_cost_usd === "number" ? ` after spending $${parsed.total_cost_usd.toFixed(4)}` : "";
+      return `claude hit the $${maxBudgetUsd} max budget${spent} — raise --max-budget-usd / maxBudgetUsd (the cap is checked between turns, so artifact-mode runs need headroom)`;
+    }
+    if (Array.isArray(parsed.errors) && parsed.errors.length > 0) {
+      return `claude exited ${code}: ${parsed.errors.map(String).join("; ").slice(0, 1_500)}`;
+    }
+  }
+
+  const detail = stderr.trim().length > 0 ? stderr : stdout;
+  return `claude exited ${code}: ${detail.slice(0, 1_500)}`;
+}
+
+function normalizeClaudeResult(result: ClaudeJsonResult): RunResult {
+  const modelUsage = result.modelUsage;
+  return {
+    output: typeof result.result === "string" ? result.result : "",
+    costUsd: typeof result.total_cost_usd === "number" ? result.total_cost_usd : 0,
+    turns: typeof result.num_turns === "number" ? result.num_turns : 0,
+    durationMs: typeof result.duration_ms === "number" ? result.duration_ms : 0,
+    models: modelUsage && typeof modelUsage === "object" ? Object.keys(modelUsage) : [],
+    raw: result,
+  };
+}