@pareto4/pp-core 1.21.1 → 1.23.1

package/scripts/live-runs/run.cjs

@@ -0,0 +1,906 @@
+'use strict';
+
+/**
+ * run.cjs - Phase 44 REACH-02 live-run CLI (D-01 .. D-12).
+ *
+ * `node scripts/live-runs/run.cjs --scenario s1|s2|s3` is the only way a live run
+ * starts. It is never required by `npm test` or CI: a live run costs money and needs
+ * a login, so it runs only when someone asks for it by name (D-02). Inside the test
+ * suite it refuses to spawn a model unless PP_LIVE_CLAUDE_BIN points at a fake.
+ *
+ * What it ties together:
+ *   - scripts/live-runs/scratch-project.cjs and scenarios.cjs: a throwaway project that
+ *     holds this build, and the fixed user turns of the scenario;
+ *   - scripts/live-runs/driver.cjs: one `claude -p` turn at a time, with caps;
+ *   - scripts/live-runs/scrub.cjs: the transcript is scrubbed before the first write (D-11);
+ *   - tests/helpers/live-run-check.cjs: the verdict (D-09, D-10), loaded lazily because
+ *     tests/ does not ship in the npm package.
+ *
+ * Outcomes (D-03): a run is pass, fail or invalid. "Cannot run here" (no CLI, not
+ * logged in, rate limited, nested launch refused, cost cap reached) is a recorded
+ * `<scenario>-r<round>.not-run.json` and exit 3, never a crash and never a pass.
+ * Runs are numbered per round (D-06): a fix re-runs all of a scenario's runs as round
+ * r+1 while round r stays on disk as evidence.
+ *
+ * Modes: run (default), --preflight, --status, --recheck DIR, --fixture-check.
+ * Exit codes: 0 ok, 1 failed (a fail verdict, an unsettled round, a re-check mismatch),
+ * 2 usage, 3 not run.
+ */
+
+const { execFileSync } = require('node:child_process');
+const crypto = require('node:crypto');
+const fs = require('node:fs');
+const os = require('node:os');
+const path = require('node:path');
+
+const driver = require('./driver.cjs');
+const scrub = require('./scrub.cjs');
+const { createScratchProject, REPO_ROOT } = require('./scratch-project.cjs');
+const { SCENARIOS } = require('./scenarios.cjs');
+const { ExitError, runMain } = require('../lib/cli-exit.cjs');
+
+const DEFAULTS = Object.freeze({
+  runs: 3,
+  round: 1,
+  maxAttempts: 6,
+  turnWallMs: 600000,
+  turnBudgetUsd: 3,
+  ledgerCapUsd: 40,
+  out: path.join(REPO_ROOT, '.planning', 'phases', '44-proof-live-runs-walks', 'runs'),
+});
+
+const EXIT = Object.freeze({ OK: 0, FAILED: 1, USAGE: 2, NOT_RUN: 3 });
+
+/** Base name of one attempt's files: scenario, round 1-9, run 1-5, attempt 1-99. */
+const RESULT_NAME_RE = /^s[1-3]-r[1-9]-run[1-5]-a(?:[1-9]|[1-9][0-9])$/;
+
+const OUTPUT_NAME_RES = Object.freeze([
+  /^ledger\.json$/,
+  /^preflight\.json$/,
+  /^s[1-3]-r[1-9]\.summary\.json$/,
+  /^s[1-3]-r[1-9]\.not-run\.json$/,
+  /^s[1-3]-r[1-9]-run[1-5]-a(?:[1-9]|[1-9][0-9])\.(?:jsonl|result\.json)$/,
+]);
+
+const SCENARIO_IDS = Object.freeze(Object.keys(SCENARIOS));
+const SCHEMA_VERSION = 1;
+const DETAIL_MAX = 300;
+const HARNESS_INVALID = Object.freeze(['timeout', 'crashed', 'no_session_id']);
+const NOT_RUN_KINDS = Object.freeze(['auth', 'rate_limited', 'launch_refused', 'cli_missing']);
+/** The not-run reason for a CLI that refused the harness's arguments: recorded, but never an OK outcome. */
+const HARNESS_REJECTED = 'cli_rejected';
+const MAX_CORPUS_FILE_BYTES = 1024 * 1024;
+
+const USAGE_TEXT = [
+  'usage: node scripts/live-runs/run.cjs --scenario s1|s2|s3 [options]',
+  '       node scripts/live-runs/run.cjs --preflight | --status --scenario <s> | --fixture-check --scenario <s> | --recheck DIR',
+  '',
+  'options:',
+  '  --scenario s1|s2|s3        the scenario (required except for --recheck and --preflight)',
+  '  --runs N                   runs per round, 1-5 (default 3)',
+  '  --only N                   run only run number N of the round',
+  '  --round R                  round number, 1-9 (default 1); a fix re-runs everything as round R+1',
+  '  --max-attempts N           attempts per round, from --runs up to 15 (default 6)',
+  '  --turn-wall-ms N           wall time per turn, 1000-1800000 (default 600000)',
+  '  --turn-budget-usd X        dollar cap per turn, 0.05-10 (default 3)',
+  '  --ledger-cap-usd X         dollar cap across the phase, 0.5-200 (default 40)',
+  '  --out DIR                  evidence directory (default the phase runs/ directory)',
+  '  --force                    run again even when a run already has a pass or fail verdict',
+  '  --keep-scratch             keep the scratch project and print where it is',
+  '  --preflight                check the CLI and the login with one cheap turn',
+  '  --status                   offline: is the round settled? (needs --scenario and --round, and --runs N',
+  '                             unless the round has a summary file to take its run count from)',
+  '  --fixture-check            offline: is the scenario fixture the project the scenario needs?',
+  '  --recheck DIR              offline: re-run the checker over kept transcripts',
+  '  --help                     this text',
+  '',
+  'exit codes: 0 ok, 1 failed or unsettled, 2 usage, 3 not run (reason in <scenario>-r<round>.not-run.json)',
+].join('\n');
+
+// ── Checker (loaded lazily: tests/ does not ship in the npm package) ─────────
+
+function loadChecker() {
+  const file = path.join(REPO_ROOT, 'tests', 'helpers', 'live-run-check.cjs');
+  try {
+    return require(file);
+  } catch (err) {
+    if (err && err.code === 'MODULE_NOT_FOUND' && String(err.message).includes('live-run-check.cjs')) {
+      throw new ExitError(EXIT.USAGE, 'live runs need a repository checkout: tests/helpers/live-run-check.cjs is missing (run from a repository checkout)');
+    }
+    throw err;
+  }
+}
+
+// ── Arguments ────────────────────────────────────────────────────────────────
+
+const usageError = (message) => new ExitError(EXIT.USAGE, message);
+
+/** A number flag, fail closed: set but empty, non-numeric or out of range is a usage error. */
+function numberFlag(name, raw, { min, max, integer = false }) {
+  if (typeof raw !== 'string' || raw.trim() === '') throw usageError(`${name} needs a value`);
+  const n = Number(raw);
+  if (!Number.isFinite(n) || (integer && !Number.isInteger(n)) || n < min || n > max) {
+    throw usageError(`${name} must be ${integer ? 'an integer' : 'a number'} from ${min} to ${max}, got "${raw}"`);
+  }
+  return n;
+}
+
+const VALUE_FLAGS = Object.freeze(['--scenario', '--runs', '--only', '--round', '--max-attempts', '--turn-wall-ms', '--turn-budget-usd', '--ledger-cap-usd', '--out', '--recheck']);
+const MODE_FLAGS = Object.freeze({ '--preflight': 'preflight', '--status': 'status', '--fixture-check': 'fixture-check', '--recheck': 'recheck' });
+const BOOL_FLAGS = Object.freeze(['--force', '--keep-scratch', '--help', '--preflight', '--status', '--fixture-check']);
+
+/** Parse argv into one options object. Throws ExitError(2) for anything it does not understand. */
+function parseArgs(argv) {
+  const raw = {};
+  const flags = new Set();
+  for (let i = 0; i < argv.length; i++) {
+    const flag = argv[i];
+    if (VALUE_FLAGS.includes(flag)) {
+      const value = argv[i + 1];
+      if (value === undefined || (value.startsWith('--') && value !== '')) throw usageError(`${flag} needs a value`);
+      raw[flag] = value;
+      i++;
+    } else if (BOOL_FLAGS.includes(flag)) {
+      flags.add(flag);
+    } else {
+      throw usageError(`unknown option: ${flag}`);
+    }
+  }
+
+  const modes = Object.keys(MODE_FLAGS).filter((f) => flags.has(f) || raw[f] !== undefined);
+  if (modes.length > 1) throw usageError(`pick one mode, got ${modes.join(' and ')}`);
+  const mode = modes.length === 1 ? MODE_FLAGS[modes[0]] : 'run';
+
+  const opts = {
+    mode,
+    help: flags.has('--help'),
+    force: flags.has('--force'),
+    keepScratch: flags.has('--keep-scratch'),
+    scenario: null,
+    runs: DEFAULTS.runs,
+    runsGiven: raw['--runs'] !== undefined,
+    only: null,
+    round: DEFAULTS.round,
+    maxAttempts: DEFAULTS.maxAttempts,
+    turnWallMs: DEFAULTS.turnWallMs,
+    turnBudgetUsd: DEFAULTS.turnBudgetUsd,
+    ledgerCapUsd: DEFAULTS.ledgerCapUsd,
+    out: DEFAULTS.out,
+    recheckDir: null,
+  };
+  if (opts.help) return opts;
+
+  if (raw['--scenario'] !== undefined) {
+    if (!SCENARIO_IDS.includes(raw['--scenario'])) throw usageError(`--scenario must be one of ${SCENARIO_IDS.join(', ')}, got "${raw['--scenario']}"`);
+    opts.scenario = raw['--scenario'];
+  }
+  if (raw['--runs'] !== undefined) opts.runs = numberFlag('--runs', raw['--runs'], { min: 1, max: 5, integer: true });
+  if (raw['--round'] !== undefined) opts.round = numberFlag('--round', raw['--round'], { min: 1, max: 9, integer: true });
+  if (raw['--only'] !== undefined) opts.only = numberFlag('--only', raw['--only'], { min: 1, max: opts.runs, integer: true });
+  if (raw['--max-attempts'] !== undefined) opts.maxAttempts = numberFlag('--max-attempts', raw['--max-attempts'], { min: opts.runs, max: 15, integer: true });
+  else if (opts.maxAttempts < opts.runs) opts.maxAttempts = opts.runs;
+  if (raw['--turn-wall-ms'] !== undefined) opts.turnWallMs = numberFlag('--turn-wall-ms', raw['--turn-wall-ms'], { min: 1000, max: 1800000, integer: true });
+  if (raw['--turn-budget-usd'] !== undefined) opts.turnBudgetUsd = numberFlag('--turn-budget-usd', raw['--turn-budget-usd'], { min: 0.05, max: 10 });
+  if (raw['--ledger-cap-usd'] !== undefined) opts.ledgerCapUsd = numberFlag('--ledger-cap-usd', raw['--ledger-cap-usd'], { min: 0.5, max: 200 });
+  if (raw['--out'] !== undefined) opts.out = path.resolve(raw['--out']);
+  if (raw['--recheck'] !== undefined) {
+    if (raw['--recheck'] === '') throw usageError('--recheck needs a directory');
+    opts.recheckDir = path.resolve(raw['--recheck']);
+  }
+
+  const needsScenario = mode === 'run' || mode === 'status' || mode === 'fixture-check';
+  if (needsScenario && !opts.scenario) throw usageError(`--scenario is required (${SCENARIO_IDS.join(', ')})`);
+  return opts;
+}
+
+// ── Output paths ─────────────────────────────────────────────────────────────
+
+/** A path for one of the harness's own files: the name must be a known shape and the path must stay under `out`. */
+function outputPath(out, name) {
+  if (!OUTPUT_NAME_RES.some((re) => re.test(name))) throw new Error(`run.cjs: refusing to write unexpected file name ${JSON.stringify(name)}`);
+  const root = path.resolve(out);
+  const full = path.join(root, name);
+  if (!full.startsWith(root + path.sep)) throw new Error(`run.cjs: ${name} resolves outside ${root}`);
+  return full;
+}
+
+const attemptBase = (scenario, round, run, attempt) => `${scenario}-r${round}-run${run}-a${attempt}`;
+
+function writeJson(out, name, value) {
+  fs.writeFileSync(outputPath(out, name), `${JSON.stringify(value, null, 2)}\n`);
+}
+
+function readJsonIfPresent(file) {
+  if (!fs.existsSync(file)) return null;
+  try {
+    return JSON.parse(fs.readFileSync(file, 'utf8'));
+  } catch {
+    return undefined;
+  }
+}
+
+const cut = (s) => String(s).trim().slice(0, DETAIL_MAX);
+
+// ── Shipped markdown (the provenance corpus) ─────────────────────────────────
+
+const corpusCache = new Map();
+
+/** The text of every .md file under commands/, agents/ and pp-core/ (regular files, at most 1 MB each). */
+function shippedCorpus(repoRoot) {
+  const root = path.resolve(repoRoot);
+  if (corpusCache.has(root)) return corpusCache.get(root);
+  const texts = [];
+  const walk = (dir) => {
+    let names;
+    try { names = fs.readdirSync(dir); } catch { return; }
+    for (const name of names.sort()) {
+      if (name === 'node_modules') continue;
+      const full = path.join(dir, name);
+      const st = fs.lstatSync(full);
+      if (st.isDirectory()) walk(full);
+      else if (st.isFile() && name.endsWith('.md') && st.size <= MAX_CORPUS_FILE_BYTES) texts.push(fs.readFileSync(full, 'utf8'));
+    }
+  };
+  for (const top of ['commands', 'agents', 'pp-core']) walk(path.join(root, top));
+  corpusCache.set(root, texts);
+  return texts;
+}
+
+// ── Ledger ───────────────────────────────────────────────────────────────────
+
+/** The phase-wide cost ledger. An unreadable ledger stops the run: it must never reset the spend. */
+function readLedger(out, capUsd) {
+  const file = path.join(path.resolve(out), 'ledger.json');
+  const data = readJsonIfPresent(file);
+  if (data === undefined) throw new ExitError(EXIT.FAILED, `ledger.json in ${out} cannot be read; fix or remove it before another run spends money`);
+  const spent = data && Number.isFinite(data.spent_usd) ? data.spent_usd : 0;
+  const entries = data && Array.isArray(data.entries) ? data.entries : [];
+  return { cap_usd: capUsd, spent_usd: spent, entries };
+}
+
+// ── Evidence on disk ─────────────────────────────────────────────────────────
+
+const ATTEMPT_FILE_RE = /^(s[1-3])-r([1-9])-run([1-5])-a([1-9][0-9]?)\.(jsonl|result\.json)$/;
+
+/** Every attempt of one round found on disk: { run, attempt, base, result } sorted by run then attempt. */
+function readAttempts(out, scenario, round) {
+  const root = path.resolve(out);
+  let names = [];
+  try { names = fs.readdirSync(root); } catch { return []; }
+  const byBase = new Map();
+  for (const name of names) {
+    const m = ATTEMPT_FILE_RE.exec(name);
+    if (!m || m[1] !== scenario || Number(m[2]) !== round) continue;
+    const base = name.slice(0, name.indexOf('.'));
+    if (!byBase.has(base)) byBase.set(base, { run: Number(m[3]), attempt: Number(m[4]), base, result: null, resultFile: null });
+    if (m[5] === 'result.json') {
+      const data = readJsonIfPresent(path.join(root, name));
+      byBase.get(base).resultFile = name;
+      byBase.get(base).result = data === undefined ? { verdict: 'invalid', invalid_reason: 'unreadable_result' } : data;
+    }
+  }
+  return [...byBase.values()].sort((a, b) => a.run - b.run || a.attempt - b.attempt);
+}
+
+const hasVerdict = (a) => Boolean(a.result) && (a.result.verdict === 'pass' || a.result.verdict === 'fail');
+
+/**
+ * One round's state. Per run: the highest attempt with a pass or fail verdict, else the
+ * highest attempt. `runs` is how many runs the round has (1..runs).
+ */
+function summarise(out, scenario, round, runs) {
+  const attempts = readAttempts(out, scenario, round);
+  const rows = [];
+  let passed = 0;
+  let failed = 0;
+  let invalid = 0;
+  let missing = 0;
+  for (let run = 1; run <= runs; run++) {
+    const mine = attempts.filter((a) => a.run === run);
+    if (mine.length === 0) {
+      missing++;
+      rows.push({ run, attempt: null, verdict: 'missing', invalid_reason: null, file: null });
+      continue;
+    }
+    const decided = [...mine].reverse().find(hasVerdict);
+    const chosen = decided || mine[mine.length - 1];
+    const verdict = chosen.result ? chosen.result.verdict : 'invalid';
+    if (verdict === 'pass') passed++;
+    else if (verdict === 'fail') failed++;
+    else invalid++;
+    rows.push({
+      run,
+      attempt: chosen.attempt,
+      verdict: chosen.result ? verdict : 'invalid',
+      invalid_reason: chosen.result ? chosen.result.invalid_reason || null : 'no_result_file',
+      file: chosen.resultFile,
+    });
+  }
+  const costOf = (r) => {
+    if (!r) return 0;
+    if (Number.isFinite(r.charged_usd)) return r.charged_usd;
+    return Number.isFinite(r.cost_usd) ? r.cost_usd : 0;
+  };
+  const cost = attempts.reduce((sum, a) => sum + costOf(a.result), 0);
+  return {
+    scenario,
+    round,
+    target_runs: runs,
+    runs: rows,
+    passed,
+    failed,
+    invalid,
+    missing,
+    settled: passed + failed === runs,
+    cost_usd: Math.round(cost * 1e6) / 1e6,
+  };
+}
+
+function readNotRun(out, scenario, round) {
+  const name = `${scenario}-r${round}.not-run.json`;
+  const data = readJsonIfPresent(outputPath(out, name));
+  if (data === null) return null;
+  return data === undefined ? { reason: 'unreadable', detail: `${name} cannot be read` } : data;
+}
+
+// ── Offline modes ────────────────────────────────────────────────────────────
+
+const defaultIo = () => ({
+  out: (t) => process.stdout.write(t),
+  err: (t) => process.stderr.write(t),
+});
+const say = (io, text) => (io.out || defaultIo().out)(`${text}\n`);
+const warn = (io, text) => (io.err || defaultIo().err)(`${text}\n`);
+
+/**
+ * How many runs the round has: --runs when given, else the largest of the target_runs its
+ * summary file and its not-run record recorded (a round that stopped as not run has no
+ * summary) and the highest run number on disk. A count is never smaller than the runs
+ * the folder holds, so a summary left from an earlier, smaller invocation cannot hide a
+ * later failed run. With no recorded target_runs, the count is unknown and --status
+ * refuses to guess (a round run with --runs 1 or 5 would be judged against the default 3).
+ */
+function statusRuns(opts) {
+  if (opts.runsGiven) return opts.runs;
+  const validTarget = (t) => Number.isInteger(t) && t >= 1 && t <= 5;
+  const summary = readJsonIfPresent(outputPath(opts.out, `${opts.scenario}-r${opts.round}.summary.json`));
+  const notRun = readJsonIfPresent(outputPath(opts.out, `${opts.scenario}-r${opts.round}.not-run.json`));
+  const recorded = [summary && summary.target_runs, notRun && notRun.target_runs].filter(validTarget);
+  if (recorded.length === 0) {
+    throw usageError(`--status needs --runs N: ${opts.scenario} round ${opts.round} has no readable summary or not-run file to take the run count from`);
+  }
+  const seen = readAttempts(opts.out, opts.scenario, opts.round).reduce((m, a) => Math.max(m, a.run), 0);
+  return Math.max(seen, ...recorded);
+}
+
+/**
+ * Exit 0 when the round is settled (every run has a pass or fail verdict; the counts
+ * show any fail), or when a not-run record explains the missing or invalid runs of a
+ * round with no fail. A fail next to a not-run record exits 1: the record must never
+ * make a round with a failed run look settled. A `cli_rejected` record exits 1 as well:
+ * the CLI refused the harness's own arguments, which is a harness bug to fix, not a
+ * reason the round cannot run here (D-03).
+ */
+function status(opts) {
+  const io = opts.io || defaultIo();
+  const s = summarise(opts.out, opts.scenario, opts.round, statusRuns(opts));
+  say(io, `${opts.scenario} round ${opts.round}: ${s.passed} passed, ${s.failed} failed, ${s.invalid} invalid, ${s.missing} missing`);
+  if (s.settled) return EXIT.OK;
+  if (s.failed > 0) return EXIT.FAILED;
+  const notRun = readNotRun(opts.out, opts.scenario, opts.round);
+  if (notRun) {
+    say(io, `${opts.scenario} round ${opts.round}: not run (${notRun.reason}: ${notRun.detail || 'no detail'})`);
+    return notRun.reason === HARNESS_REJECTED ? EXIT.FAILED : EXIT.OK;
+  }
+  return EXIT.FAILED;
+}
+
+const sortedUnique = (list) => [...new Set(list)].sort();
+const sameList = (a, b) => JSON.stringify(sortedUnique(a)) === JSON.stringify(sortedUnique(b));
+
+/** Re-run the checker over every kept transcript in `dir` and compare with what each result file recorded. */
+function recheck(dir, opts = {}) {
+  const io = opts.io || defaultIo();
+  const root = path.resolve(dir);
+  let names = [];
+  try { names = fs.readdirSync(root).sort(); } catch { names = []; }
+  const resultFiles = names.filter((n) => n.endsWith('.result.json') && RESULT_NAME_RE.test(n.slice(0, -'.result.json'.length)));
+
+  if (resultFiles.length === 0) {
+    // a cli_rejected record is a harness bug, so it explains nothing (see status)
+    const notRun = names
+      .filter((n) => n.endsWith('.not-run.json'))
+      .some((n) => {
+        const record = readJsonIfPresent(path.join(root, n));
+        return !record || record.reason !== HARNESS_REJECTED;
+      });
+    const pre = readJsonIfPresent(path.join(root, 'preflight.json'));
+    const preflightExplains = Boolean(pre) && pre.auth !== 'ok';
+    if (notRun || preflightExplains) {
+      say(io, `no results in ${root}; a not-run or failed preflight record explains it`);
+      return EXIT.OK;
+    }
+    warn(io, `nothing to re-check in ${root}: no result files and no not-run record`);
+    return EXIT.FAILED;
+  }
+
+  const checker = loadChecker();
+  const corpus = shippedCorpus(REPO_ROOT);
+  const mismatches = [];
+  for (const file of resultFiles) {
+    const base = file.slice(0, -'.result.json'.length);
+    const recorded = readJsonIfPresent(path.join(root, file));
+    if (!recorded) {
+      mismatches.push(`${file}: the result file cannot be read`);
+      continue;
+    }
+    if (recorded.transcript !== `${base}.jsonl`) {
+      mismatches.push(`${file}: transcript field ${JSON.stringify(recorded.transcript)} is not ${base}.jsonl`);
+      continue;
+    }
+    const scenario = SCENARIOS[recorded.scenario];
+    if (!scenario || !base.startsWith(`${recorded.scenario}-`)) {
+      mismatches.push(`${file}: unknown scenario ${JSON.stringify(recorded.scenario)}`);
+      continue;
+    }
+    const transcriptFile = path.join(root, `${base}.jsonl`);
+    if (!fs.existsSync(transcriptFile)) {
+      mismatches.push(`${file}: ${base}.jsonl is missing`);
+      continue;
+    }
+    if (HARNESS_INVALID.includes(recorded.invalid_reason)) {
+      if (recorded.verdict !== 'invalid') mismatches.push(`${file}: recorded ${recorded.invalid_reason} but verdict ${recorded.verdict}`);
+      continue;
+    }
+    const { events } = checker.parseTranscript(fs.readFileSync(transcriptFile, 'utf8'));
+    const again = checker.checkRun({ scenario: recorded.scenario, events, turns: scenario.turns, corpus });
+    if (again.verdict !== recorded.verdict) mismatches.push(`${file}: recorded verdict ${recorded.verdict}, checker now says ${again.verdict}`);
+    if (!sameList(again.jargon, recorded.jargon || [])) mismatches.push(`${file}: jargon hits differ (recorded ${JSON.stringify(recorded.jargon || [])}, now ${JSON.stringify(again.jargon)})`);
+    const nowTokens = again.provenance_failures.map((f) => f.token);
+    const thenTokens = (recorded.provenance_failures || []).map((f) => f.token);
+    if (!sameList(nowTokens, thenTokens)) mismatches.push(`${file}: provenance failures differ (recorded ${JSON.stringify(thenTokens)}, now ${JSON.stringify(nowTokens)})`);
+  }
+
+  if (mismatches.length > 0) {
+    for (const m of mismatches) warn(io, `MISMATCH ${m}`);
+    say(io, `re-check: ${mismatches.length} mismatch(es) in ${resultFiles.length} result file(s)`);
+    return EXIT.FAILED;
+  }
+  say(io, `re-check: ${resultFiles.length} result file(s) agree with the checker`);
+  return EXIT.OK;
+}
+
+/** Build the scenario's starting project, ask the scenario whether it is what it needs, then remove it. */
+function fixtureCheck(opts) {
+  const io = opts.io || defaultIo();
+  const scenario = SCENARIOS[opts.scenario];
+  let project;
+  try {
+    project = createScratchProject({
+      label: scenario.id,
+      audience: scenario.audience,
+      fixture: scenario.fixture,
+      initialCommit: scenario.initialCommit,
+      commands: [scenario.command],
+      tmpRoot: opts.tmpRoot,
+    });
+    const res = scenario.fixtureCheck(project.projectDir, { personHome: project.personHome });
+    if (res.ok) {
+      say(io, `fixture ${scenario.id}: ok (${String(res.detail).split('\n')[0]})`);
+      return EXIT.OK;
+    }
+    say(io, `fixture ${scenario.id}: FAILED ${res.detail}`);
+    return EXIT.FAILED;
+  } catch (err) {
+    say(io, `fixture ${scenario.id}: FAILED ${cut(err.message)}`);
+    return EXIT.FAILED;
+  } finally {
+    if (project) project.cleanup();
+  }
+}
+
+// ── Preflight and the run loop ───────────────────────────────────────────────
+
+/**
+ * The paths that make up the shipped build: package.json and its `files` list. The
+ * dirty check looks only here, so uncommitted planning files and run evidence (the
+ * default --out sits inside the repo) do not mark every run build-dirty.
+ */
+function shippedPaths(repoRoot = REPO_ROOT) {
+  try {
+    const pkg = JSON.parse(fs.readFileSync(path.join(repoRoot, 'package.json'), 'utf8'));
+    if (Array.isArray(pkg.files) && pkg.files.length > 0) {
+      return ['package.json', ...pkg.files.filter((f) => typeof f === 'string' && f !== '' && !f.startsWith('!'))];
+    }
+  } catch {
+    // no readable package.json: fall back to the whole repository
+  }
+  return null;
+}
+
+/** What the harness can record about this build and machine without a model. */
+function preflight(opts) {
+  const env = opts.env || process.env;
+  const git = (args) => {
+    try {
+      return execFileSync('git', args, { cwd: REPO_ROOT, encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'], timeout: 30000 });
+    } catch {
+      return null;
+    }
+  };
+  const head = git(['rev-parse', 'HEAD']);
+  const shipped = shippedPaths();
+  const porcelain = git(shipped ? ['status', '--porcelain', '--', ...shipped] : ['status', '--porcelain']);
+  let userClaudeMd = null;
+  try {
+    const file = path.join(env.HOME || os.homedir(), '.claude', 'CLAUDE.md');
+    if (fs.existsSync(file)) userClaudeMd = crypto.createHash('sha256').update(fs.readFileSync(file)).digest('hex').slice(0, 12);
+  } catch {
+    userClaudeMd = null;
+  }
+  return {
+    cliVersion: driver.cliVersion({ env }),
+    buildCommit: head === null ? null : head.trim(),
+    buildDirty: porcelain === null ? true : porcelain.trim() !== '',
+    userClaudeMd,
+  };
+}
+
+const UNTRUSTED_RE = /not (?:been )?trusted|trust this (?:folder|workspace|directory)|workspace trust/i;
+const PROBE_PROMPT = 'Reply with the single word ready.';
+const PROBE_BUDGET_USD = 0.25;
+const PROBE_WALL_MS = 120000;
+
+const nowIso = () => new Date().toISOString();
+const roundCost = (n) => Math.round(n * 1e6) / 1e6;
+
+/** Record "cannot run here" (D-03). The detail is scrubbed and cut to 300 characters (T-44-16). */
+function recordNotRun(opts, pre, reason, detail, secrets) {
+  const scenario = opts.scenario;
+  const safe = cut(scrub.scrubText(String(detail || ''), secrets || scrub.hostSecrets({ env: opts.env })));
+  writeJson(opts.out, `${scenario}-r${opts.round}.not-run.json`, {
+    schema: SCHEMA_VERSION,
+    scenario,
+    round: opts.round,
+    // --status reads the run count from here when the round stopped before its summary
+    target_runs: opts.runs,
+    reason,
+    detail: safe,
+    at: nowIso(),
+    cli_version: pre.cliVersion,
+    build_commit: pre.buildCommit,
+    build_dirty: pre.buildDirty,
+  });
+  say(opts.io, `${scenario} round ${opts.round}: not run (${reason}: ${safe || 'no detail'})`);
+  return EXIT.NOT_RUN;
+}
+
+/** The limits of one attempt that a reader of the evidence must know about (Pitfalls 4 and 5, A4). */
+function limitsOf({ facts, events, pre, stderr }) {
+  const limits = [];
+  const hasInit = events.some((e) => e && e.type === 'system' && e.subtype === 'init');
+  if (hasInit && !facts.tools.includes('AskUserQuestion')) limits.push('headless-no-ask-tool');
+  if (pre.userClaudeMd) limits.push(`user-claude-md:${pre.userClaudeMd}`);
+  if (UNTRUSTED_RE.test(stderr)) limits.push('untrusted-workspace');
+  if (facts.permissionDenials.length > 0) limits.push(`permission-denials:${facts.permissionDenials.length}`);
+  if (pre.buildDirty) limits.push('build-dirty');
+  return limits;
+}
+
+/** Turn outcomes that end before any model spend: nothing is charged for them. */
+const NO_SPEND_KINDS = Object.freeze(['cli_missing', 'auth', 'launch_refused', 'cli_rejected']);
+
+/**
+ * A crash that never reached the model: the CLI exited on its own (no wall-time kill, no
+ * signal) and printed no `system/init` event. In stream-json mode the CLI prints init
+ * before any API call, so this is the CLI refusing its arguments or failing to start.
+ */
+function cliRejected(turn, events, kind) {
+  if (kind !== 'crashed' || turn.timedOut || turn.spawnError || !Number.isInteger(turn.status)) return false;
+  return !events.some((e) => e && e.type === 'system' && e.subtype === 'init');
+}
+
+/**
+ * What one turn is charged on the ledger: the largest total_cost_usd its result events
+ * report, else (a timeout, a crash, a rate limit, a result with no cost) the whole turn
+ * budget, because the turn may have spent up to it. A turn that never reached the model
+ * (NO_SPEND_KINDS, including a `cliRejected` crash) costs nothing. The charges of an
+ * attempt's turns are summed: if the CLI reports a resumed session's cost cumulatively
+ * this over-counts, which is the safe side of a cap.
+ */
+function turnCharge(events, kind, budgetUsd) {
+  const costs = events
+    .filter((e) => e && e.type === 'result' && typeof e.total_cost_usd === 'number' && Number.isFinite(e.total_cost_usd))
+    .map((e) => e.total_cost_usd);
+  if (costs.length > 0) return Math.max(...costs);
+  if (NO_SPEND_KINDS.includes(kind)) return 0;
+  return budgetUsd;
+}
+
+const ledgerCapDetail = (ledger, pending, budgetUsd) =>
+  `the phase cost ledger is at $${(ledger.spent_usd + pending).toFixed(2)} of its $${ledger.cap_usd.toFixed(2)} cap; one more turn may spend up to $${budgetUsd.toFixed(2)}`;
+
+/**
+ * One attempt: a fresh scratch project, the scenario's turns, a scrubbed transcript and
+ * a result. Returns { notRun: { reason, detail, secrets }, base, costUsd } when the model
+ * cannot be reached (or the next turn would pass the ledger cap), else
+ * { verdict, base, line, costUsd }. `costUsd` is always what the attempt is charged,
+ * turns that ran before a not-run included. `reserve(usd)`, when given, is called just
+ * before each turn with the attempt's charge so far plus the whole turn budget.
+ */
+function runAttempt({ opts, scenario, checker, corpus, pre, run, attempt, ledger, reserve }) {
+  const env = opts.env || process.env;
+  const base = attemptBase(scenario.id, opts.round, run, attempt);
+  const project = createScratchProject({
+    label: scenario.id,
+    audience: scenario.audience,
+    fixture: scenario.fixture,
+    initialCommit: scenario.initialCommit,
+    commands: [scenario.command],
+    tmpRoot: opts.tmpRoot,
+  });
+  try {
+    const secrets = scrub.hostSecrets({ projectDir: project.projectDir, personHome: project.personHome, env });
+    const raw = [];
+    const stderrAll = [];
+    let durationMs = 0;
+    let sessionId = null;
+    let harnessInvalid = null;
+    let charged = 0;
+
+    for (let i = 0; i < scenario.turns.length; i++) {
+      if (ledger && ledger.spent_usd + charged + opts.turnBudgetUsd > ledger.cap_usd) {
+        return { notRun: { reason: 'ledger_cap', detail: ledgerCapDetail(ledger, charged, opts.turnBudgetUsd), secrets }, base, costUsd: charged };
+      }
+      if (reserve) reserve(roundCost(charged + opts.turnBudgetUsd));
+      const turn = driver.runTurn({
+        cwd: project.projectDir,
+        personHome: project.personHome,
+        prompt: scenario.turns[i],
+        sessionId,
+        wallMs: opts.turnWallMs,
+        budgetUsd: opts.turnBudgetUsd,
+        env,
+        audience: 'business',
+      });
+      const { events } = driver.parseLines(turn.stdout);
+      const cls = driver.classifyTurn(turn, events);
+      const rejected = cliRejected(turn, events, cls.kind);
+      charged = roundCost(charged + turnCharge(events, rejected ? 'cli_rejected' : cls.kind, opts.turnBudgetUsd));
+      if (NOT_RUN_KINDS.includes(cls.kind)) return { notRun: { reason: cls.kind, detail: cls.detail, secrets }, base, costUsd: charged };
+      // A first turn the CLI refused is not retried: every retry would be refused the same
+      // way. A refused later turn stays an invalid crashed attempt (turn 1's output is kept).
+      if (rejected && i === 0) return { notRun: { reason: 'cli_rejected', detail: cls.detail, secrets }, base, costUsd: charged };
+      raw.push(turn.stdout);
+      stderrAll.push(turn.stderr);
+      durationMs += turn.durationMs;
+      if (cls.kind === 'timeout' || cls.kind === 'crashed') {
+        harnessInvalid = cls.kind;
+        break;
+      }
+      sessionId = driver.sessionIdOf(events);
+      if (!sessionId && i < scenario.turns.length - 1) {
+        harnessInvalid = 'no_session_id';
+        break;
+      }
+    }
+
+    const scrubbed = scrub.scrubJsonl(raw.join('\n'), secrets);
+    const { events } = checker.parseTranscript(scrubbed);
+    const { facts, ...checked } = checker.checkRun({ scenario: scenario.id, events, turns: scenario.turns, corpus });
+    const verdict = harnessInvalid ? 'invalid' : checked.verdict;
+    const result = {
+      ...checked,
+      schema: SCHEMA_VERSION,
+      scenario: scenario.id,
+      round: opts.round,
+      run,
+      attempt,
+      verdict,
+      invalid_reason: harnessInvalid || checked.invalid_reason,
+      model: facts.model,
+      cli_version: facts.cliVersion || pre.cliVersion,
+      build_commit: pre.buildCommit,
+      build_dirty: pre.buildDirty,
+      models_used: facts.modelsUsed,
+      cost_usd: facts.costUsd,
+      charged_usd: charged,
+      num_turns: facts.numTurns,
+      duration_ms: durationMs,
+      permission_denials: facts.permissionDenials,
+      subagent_stats: facts.subagentStats,
+      limits: limitsOf({ facts, events, pre, stderr: stderrAll.join('\n') }),
+      transcript: `${base}.jsonl`,
+      at: nowIso(),
+    };
+    fs.writeFileSync(outputPath(opts.out, `${base}.jsonl`), scrubbed);
+    writeJson(opts.out, `${base}.result.json`, scrub.scrubEvent(result, secrets));
+    const reason = result.invalid_reason || result.fail_reason;
+    const chargedNote = charged === roundCost(facts.costUsd) ? '' : ` (charged $${charged.toFixed(2)})`;
+    return {
+      verdict,
+      base,
+      costUsd: charged,
+      line: `${base}: ${verdict}${reason ? ` (${reason})` : ''} $${facts.costUsd.toFixed(2)}${chargedNote}`,
+    };
+  } finally {
+    if (opts.keepScratch) say(opts.io, `scratch kept: ${project.projectDir} (person home ${project.personHome})`);
+    else project.cleanup();
+  }
+}
+
+/** Run one scenario's runs for one round: resumable, capped, and every outcome recorded. */
+function runScenario(opts) {
+  const io = opts.io || defaultIo();
+  const scenario = SCENARIOS[opts.scenario];
+  const checker = loadChecker();
+  const pre = preflight(opts);
+  fs.mkdirSync(opts.out, { recursive: true });
+  // The round's records from an earlier invocation are stale: this one ends by writing
+  // a fresh summary or a fresh not-run record. A stale summary left beside an early stop
+  // would give --status an old, smaller run count.
+  for (const name of [`${scenario.id}-r${opts.round}.not-run.json`, `${scenario.id}-r${opts.round}.summary.json`]) {
+    const stale = outputPath(opts.out, name);
+    if (fs.existsSync(stale)) fs.unlinkSync(stale);
+  }
+  const ledger = readLedger(opts.out, opts.ledgerCapUsd);
+  const corpus = shippedCorpus(REPO_ROOT);
+
+  if (pre.cliVersion === null) return recordNotRun(opts, pre, 'cli_missing', 'the claude binary could not be run (claude --version failed)');
+
+  // Write-ahead charge: before each turn, ledger.json on disk already holds the attempt's
+  // charge so far plus the whole turn budget, as a `pending` entry. Ctrl-C (or any kill)
+  // ends run.cjs inside spawnSync before it can record the turn, and a JS signal handler
+  // cannot run until the synchronous loop ends, so the reservation is what keeps an
+  // interrupted turn on the permanent ledger, on the safe side of the cap. After the
+  // attempt, the pending entry is replaced by the real charge (or dropped when it is 0).
+  const ledgerFile = outputPath(opts.out, 'ledger.json');
+  const reserveFor = (base) => (pendingUsd) => writeJson(opts.out, 'ledger.json', {
+    ...ledger,
+    spent_usd: roundCost(ledger.spent_usd + pendingUsd),
+    entries: [...ledger.entries, { file: base, cost_usd: pendingUsd, at: nowIso(), pending: true }],
+  });
+
+  const targets = opts.only ? [opts.only] : Array.from({ length: opts.runs }, (_, i) => i + 1);
+  for (const run of targets) {
+    const highest = (list) => list.filter((a) => a.run === run).pop();
+    // Settled the way summarise sees it: any attempt of this run has a pass or fail
+    // verdict, even when a later forced attempt ended invalid.
+    let settled = !opts.force && readAttempts(opts.out, scenario.id, opts.round).some((a) => a.run === run && hasVerdict(a));
+    while (!settled) {
+      const all = readAttempts(opts.out, scenario.id, opts.round);
+      if (all.length >= opts.maxAttempts) break;
+      if (ledger.spent_usd + opts.turnBudgetUsd > ledger.cap_usd) {
+        return recordNotRun(opts, pre, 'ledger_cap', ledgerCapDetail(ledger, 0, opts.turnBudgetUsd));
+      }
+      const prior = highest(all);
+      const attempt = prior ? prior.attempt + 1 : 1;
+      if (attempt > 99) break;
+      const hadLedger = fs.existsSync(ledgerFile);
+      const outcome = runAttempt({ opts, scenario, checker, corpus, pre, run, attempt, ledger, reserve: reserveFor(attemptBase(scenario.id, opts.round, run, attempt)) });
+      if (!outcome.notRun || outcome.costUsd > 0) {
+        ledger.spent_usd = roundCost(ledger.spent_usd + outcome.costUsd);
+        const entry = { file: outcome.base, cost_usd: outcome.costUsd, at: nowIso() };
+        if (outcome.notRun) entry.not_run = outcome.notRun.reason;
+        ledger.entries.push(entry);
+        writeJson(opts.out, 'ledger.json', ledger);
+      } else if (hadLedger) {
+        writeJson(opts.out, 'ledger.json', ledger);
+      } else if (fs.existsSync(ledgerFile)) {
+        fs.unlinkSync(ledgerFile);
+      }
+      if (outcome.notRun) return recordNotRun(opts, pre, outcome.notRun.reason, outcome.notRun.detail, outcome.notRun.secrets);
+      say(io, outcome.line);
+      settled = outcome.verdict === 'pass' || outcome.verdict === 'fail';
+    }
+  }
+
+  const summary = summarise(opts.out, scenario.id, opts.round, opts.runs);
+  writeJson(opts.out, `${scenario.id}-r${opts.round}.summary.json`, { schema: SCHEMA_VERSION, at: nowIso(), ...summary });
+  say(io, `${scenario.id} round ${opts.round}: ${summary.passed} passed, ${summary.failed} failed, ${summary.invalid} invalid, ${summary.missing} missing`);
+  return targets.every((n) => summary.runs[n - 1].verdict === 'pass') ? EXIT.OK : EXIT.FAILED;
+}
+
+/** One cheap probe turn: is the CLI there and logged in? Writes preflight.json; exit 3 unless auth is ok. */
+function runPreflight(opts) {
+  const io = opts.io || defaultIo();
+  const env = opts.env || process.env;
+  const pre = preflight(opts);
+  fs.mkdirSync(opts.out, { recursive: true });
+  let auth;
+  let detail = '';
+  if (pre.cliVersion === null) {
+    auth = 'cli_missing';
+    detail = 'the claude binary could not be run (claude --version failed)';
+  } else {
+    const root = opts.tmpRoot || os.tmpdir();
+    const cwd = fs.mkdtempSync(path.join(root, 'pp-live-probe-'));
+    const personHome = fs.mkdtempSync(path.join(root, 'pp-live-home-'));
+    try {
+      const turn = driver.runTurn({ cwd, personHome, prompt: PROBE_PROMPT, sessionId: null, wallMs: PROBE_WALL_MS, budgetUsd: PROBE_BUDGET_USD, env, audience: 'business' });
+      const cls = driver.classifyTurn(turn, driver.parseLines(turn.stdout).events);
+      auth = cls.kind === 'ok' ? 'ok' : cls.kind;
+      detail = cut(scrub.scrubText(cls.detail, scrub.hostSecrets({ projectDir: cwd, personHome, env })));
+    } finally {
+      fs.rmSync(cwd, { recursive: true, force: true });
+      fs.rmSync(personHome, { recursive: true, force: true });
+    }
+  }
+  writeJson(opts.out, 'preflight.json', {
+    schema: SCHEMA_VERSION,
+    at: nowIso(),
+    cli_version: pre.cliVersion,
+    build_commit: pre.buildCommit,
+    build_dirty: pre.buildDirty,
+    user_claude_md: pre.userClaudeMd,
+    auth,
+    detail,
+  });
+  say(io, `preflight: auth ${auth}${detail ? ` (${detail})` : ''}; claude ${pre.cliVersion || 'not found'}; build ${pre.buildCommit ? pre.buildCommit.slice(0, 8) : 'unknown'}${pre.buildDirty ? ' (dirty)' : ''}`);
+  return auth === 'ok' ? EXIT.OK : EXIT.NOT_RUN;
+}
+
+// ── Entry ────────────────────────────────────────────────────────────────────
+
+const GUARD_MESSAGE = 'live runs never run inside the test suite (D-02); point PP_LIVE_CLAUDE_BIN at a fake to test the harness';
+
+/**
+ * Run the CLI. `io` is `{ out, err, tmpRoot }`: the two writers default to stdout and
+ * stderr, `tmpRoot` is where scratch projects go (tests pass a tracked temp dir).
+ */
+function main(argv = process.argv.slice(2), env = process.env, io = {}) {
+  const writers = { ...defaultIo(), ...io };
+  let parsed;
+  try {
+    parsed = parseArgs(argv);
+  } catch (err) {
+    if (!(err instanceof ExitError)) throw err;
+    warn(writers, err.message);
+    warn(writers, USAGE_TEXT);
+    return err.code;
+  }
+  if (parsed.help) {
+    say(writers, USAGE_TEXT);
+    return EXIT.OK;
+  }
+  const opts = { ...parsed, env, io: writers, tmpRoot: io.tmpRoot };
+
+  const spawnsModel = opts.mode === 'run' || opts.mode === 'preflight';
+  if (spawnsModel && env.NODE_TEST_CONTEXT && !env.PP_LIVE_CLAUDE_BIN) {
+    warn(writers, GUARD_MESSAGE);
+    return EXIT.USAGE;
+  }
+
+  try {
+    switch (opts.mode) {
+      case 'status': return status(opts);
+      case 'recheck': return recheck(opts.recheckDir, opts);
+      case 'fixture-check': return fixtureCheck(opts);
+      case 'preflight': return runPreflight(opts);
+      default: return runScenario(opts);
+    }
+  } catch (err) {
+    if (!(err instanceof ExitError)) throw err;
+    if (err.hasUserMessage && err.code !== 0) warn(writers, err.message);
+    return err.code;
+  }
+}
+
+module.exports = {
+  DEFAULTS,
+  EXIT,
+  RESULT_NAME_RE,
+  parseArgs,
+  main,
+  runScenario,
+  preflight,
+  recheck,
+  status,
+  fixtureCheck,
+  summarise,
+  shippedCorpus,
+  shippedPaths,
+  readLedger,
+};
+
+if (require.main === module) runMain(() => main(process.argv.slice(2), process.env));