crewforth 3.2.0-rc.2 → 3.2.0-rc.4

package/kit/eval/lib/review-bench/bench.mjs

@@ -0,0 +1,273 @@
+#!/usr/bin/env node
+// bench.mjs — how right the review is, in numbers: run labelled cases, match what was found to where the defect
+// is, and say the precision, the recall and every false alarm by name.
+//
+//   node bench.mjs --cases DIR [--arm prepass|agent] [--split all|train|holdout] [--only id,id]
+//                  [--model sonnet] [--advisor opus] [--case-budget 0.75] [--max-cost 12] [--floors FILE] [--out FILE]
+//                  [--no-tools]
+//       --no-tools  the pre-pass runs without shellcheck, so its numbers do not depend on what the machine has
+//                   installed (the gate in CI uses it; shellcheck's own precision is not part of the floors)
+//   rc 0 = ran (and, with --floors, at or above every floor) · 1 = below a floor, or the agent arm could not run a
+//   case · 2 = usage
+//
+// A CASE is a directory: case.json, before/ and after/ (the touched files only, with their modes in case.json).
+// It is built as a git repository with `before` committed and `after` staged, so a reviewer meets it the way it
+// meets real work: `git diff --cached`. case.json says what the change claims to do (`intent`, the only thing the
+// reviewer is told) and where the defect is (`expect`: file and line range in the after side; a null range is a
+// file-level defect, a mode bit). A clean case expects nothing.
+//
+// SCORING. A finding matches an expected location when it cites the same file within three lines of the range
+// (any line, for a file-level defect). A defect case is CAUGHT when any finding matches any of its locations:
+// one defect is one defect, however many hunks its fix took. Recall = caught / defect cases. Precision =
+// matching findings / all findings of severity low or above (info is a note, not a claim). Every finding that
+// matches nothing is listed as a false alarm with its case, line and title.
+//
+// A defect case made by undoing a fix can show the fix's own explanation among the removed lines. That is what a
+// reviewer of a real revert would see too, but it is a hint, so such cases are marked and recall is reported for
+// both halves.
+//
+// ARMS. prepass: kit/eval/lib/review-prepass, in process, no model. agent: Claude Code headless, the session run
+// AS crew-review-agent (--agent) with Crewforth's review agent, skills and eval/lib copied into the case's .claude/,
+// on --model, each case capped by --case-budget and the run by --max-cost (it stops before a case that could pass
+// it). The agent arm scores what it wrote to .crewforth/review.json together with the pre-pass it was told to carry
+// over; when it wrote no valid file, its final answer's `file:line · severity` lines are read instead, and the
+// result says so. It runs one review pass: the refuter is not installed in the case, so nothing is --verified.
+//
+// --advisor MODEL runs the agent arm with that advisor (the `advisorModel` setting, passed with --settings; the user's
+// own settings are not read). What a run cost is the result's own total, which counts the advisor's calls; each row
+// also carries the cost and tokens by model (the result's modelUsage) and how many advisor calls were made.
+//
+// A TASK is a case of `kind: "task"`: an implementation, not a review. before/ is the repository, `intent` is what
+// the session is asked to do, check/ is copied in only after the session has ended and `check` (argv, run in the
+// repository) grades it: rc 0 is done. The session runs as itself, not as the review agent.
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { spawnSync } from 'node:child_process';
+import { fileURLToPath } from 'node:url';
+import { prepass } from '../review-prepass/prepass.mjs';
+import { validate } from '../findings/findings.mjs';
+
+const HERE = path.dirname(fileURLToPath(import.meta.url));
+const KIT = path.resolve(HERE, '..', '..', '..');   // kit/ (source) or .claude/ (installed)
+const NEAR = 3;
+const RANK = { info: 0, low: 1, medium: 2, high: 3, critical: 4 };
+const TOKEN = `ghp_${'R3v1'.repeat(9)}`;
+
+function args(argv) {
+  const o = { arm: 'prepass', split: 'all', model: 'sonnet', advisor: null, caseBudget: 0.75, maxCost: 12 };
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i]; const v = () => { const x = argv[++i]; if (x === undefined) throw new Error(`${a} needs a value`); return x; };
+    if (a === '--cases') o.cases = v(); else if (a === '--arm') o.arm = v(); else if (a === '--split') o.split = v();
+    else if (a === '--only') o.only = v().split(','); else if (a === '--model') o.model = v(); else if (a === '--advisor') o.advisor = v();
+    else if (a === '--case-budget') o.caseBudget = Number(v()); else if (a === '--max-cost') o.maxCost = Number(v());
+    else if (a === '--floors') o.floors = v(); else if (a === '--out') o.out = v(); else if (a === '--no-tools') o.noTools = true;
+    else throw new Error(`unknown argument ${a}`);
+  }
+  if (!o.cases || !['prepass', 'agent'].includes(o.arm) || !['all', 'train', 'holdout'].includes(o.split)) throw new Error('--cases DIR is required; --arm prepass|agent; --split all|train|holdout');
+  return o;
+}
+
+export function loadCases(dir, { split = 'all', only = null } = {}) {
+  return fs.readdirSync(dir).sort().filter((d) => fs.existsSync(path.join(dir, d, 'case.json')))
+    .map((d) => ({ ...JSON.parse(fs.readFileSync(path.join(dir, d, 'case.json'), 'utf8')), dir: path.join(dir, d) }))
+    .filter((c) => (split === 'all' || (split === 'holdout') === !!c.holdout) && (!only || only.includes(c.id)));
+}
+
+const ENV = (tmp) => ({ ...process.env, GIT_CONFIG_GLOBAL: path.join(tmp, 'gitconfig'), GIT_CONFIG_NOSYSTEM: '1' });
+function git(root, env, ...a) { const r = spawnSync('git', ['-C', root, ...a], { encoding: 'utf8', env }); if (r.status !== 0) throw new Error(`git ${a.join(' ')}: ${r.stderr}`); return r.stdout; }
+// A folder a build writes (bin, dist, …) is stored with a leading '=' inside a case, so a repository's own commit
+// gates do not read a case's past source file as a build artifact; the case is built under the real name.
+const BUILD_DIRS = new Set(['node_modules', 'bower_components', 'dist', 'build', 'out', 'target', 'bin', 'obj', 'vendor', 'coverage']);
+export const storedPath = (p) => p.split('/').map((s, i, a) => (i < a.length - 1 && BUILD_DIRS.has(s) ? `=${s}` : s)).join('/');
+export const realPath = (p) => p.split('/').map((s) => (s.startsWith('=') && BUILD_DIRS.has(s.slice(1)) ? s.slice(1) : s)).join('/');
+function tree(dir) { const out = []; const walk = (d) => { if (!fs.existsSync(d)) return; for (const e of fs.readdirSync(d, { withFileTypes: true })) { const p = path.join(d, e.name); if (e.isDirectory()) walk(p); else out.push(path.relative(dir, p).split(path.sep).join('/')); } }; walk(dir); return out.sort(); }
+
+// The case as a repository: before committed, after staged. Returns { root, env, diff }.
+export function build(c, tmp) {
+  const root = path.join(tmp, c.id); fs.mkdirSync(root, { recursive: true });
+  fs.writeFileSync(path.join(tmp, 'gitconfig'), '[user]\n\tname = bench\n\temail = bench@bench.invalid\n[core]\n\tautocrlf = false\n\tfileMode = false\n[init]\n\tdefaultBranch = main\n');
+  const env = ENV(tmp);
+  const put = (side) => {
+    for (const s of tree(path.join(c.dir, side))) {
+      const f = realPath(s);
+      const out = path.join(root, f); fs.mkdirSync(path.dirname(out), { recursive: true });
+      fs.writeFileSync(out, fs.readFileSync(path.join(c.dir, side, s), 'utf8').replaceAll('{{TOKEN}}', TOKEN));
+      fs.chmodSync(out, c.modes?.[side]?.[f] === '100755' ? 0o755 : 0o644);
+    }
+  };
+  // A file's mode is written into the index, not read from the disk: Windows has no executable bit, so a case whose
+  // whole diff is a mode (mode-bit) built as no diff at all there. `git init` probes the disk and writes its own
+  // core.filemode into the repository (true on macOS and Linux, false on Windows), so it is set after init, to false,
+  // and every platform takes this one path.
+  const modes = (side) => { for (const [f, m] of Object.entries(c.modes?.[side] ?? {})) git(root, env, 'update-index', `--chmod=${m === '100755' ? '+x' : '-x'}`, '--', f); };
+  git(root, env, 'init', '-q', '.'); git(root, env, 'config', 'core.fileMode', 'false');
+  put('before'); git(root, env, 'add', '-A'); modes('before'); git(root, env, 'commit', '-qm', 'base', '--allow-empty');
+  if (c.kind !== 'task') {
+    for (const s of tree(path.join(c.dir, 'before'))) if (!fs.existsSync(path.join(c.dir, 'after', s))) fs.rmSync(path.join(root, realPath(s)));
+    put('after'); git(root, env, 'add', '-A'); modes('after');
+  }
+  fs.appendFileSync(path.join(root, '.git', 'info', 'exclude'), '.claude/\n.crewforth/\n');
+  const diff = git(root, env, 'diff', '--cached', '--no-ext-diff', '-U3');
+  return { root, env, diff };
+}
+
+// Whether the removed lines of the diff carry a comment: the fix's own explanation, visible to the reviewer.
+export const hintInDiff = (diff) => diff.split('\n').some((l) => /^-(?!--)\s*(#|\/\/|\/\*|\*)\s*\S/.test(l) && !/^-\s*#!/.test(l));
+
+export function score(c, findings) {
+  const claims = findings.filter((f) => (RANK[f.severity] ?? 1) >= RANK.low);
+  const hits = (f) => (c.expect ?? []).some((e) => e.file === f.file && (e.from == null || (f.line >= e.from - NEAR && f.line <= e.to + NEAR)));
+  const matched = claims.filter(hits);
+  return { caught: c.kind === 'defect' ? findings.some(hits) : null, claims: claims.length, matched: matched.length,
+    falseAlarms: claims.filter((f) => !hits(f)).map((f) => ({ case: c.id, file: f.file, line: f.line, severity: f.severity, title: f.title })) };
+}
+
+const flat = (doc) => (doc?.findings ?? []).map((f) => ({ file: f.evidence?.[0]?.file, line: f.evidence?.[0]?.line, severity: f.severity ?? 'low', title: f.title, rule: f.rule }));
+// `file:line · severity · …` lines of a final answer, for a review that wrote no file.
+export const fromText = (text) => [...String(text).matchAll(/([\w./-]+\.[\w]+):(\d+)\s*[·|-]\s*(critical|high|medium|low|info)\b[^\n]*/gi)]
+  .map((m) => ({ file: m[1].replace(/^\.\//, ''), line: Number(m[2]), severity: m[3].toLowerCase(), title: m[0].slice(0, 160) }));
+
+function copyKit(root) {
+  const dst = path.join(root, '.claude');
+  const parts = [['agents/crew-review-agent.md'], ['skills/crew-code-review'], ['skills/docs-writer'], ['eval/lib/review-prepass'], ['eval/lib/findings'], ['eval/lib/crew-map'], ['hooks/secret-blocklist.txt']];
+  for (const [p] of parts) { const s = path.join(KIT, p); if (fs.existsSync(s)) fs.cpSync(s, path.join(dst, p), { recursive: true }); }
+}
+
+// The flags every headless run shares. The bench measures the model and Crewforth, not the machine it runs on: no
+// user settings (an advisor model, hooks), no MCP server of the user's, no session written. Seen without them: a run
+// took the user's advisor model, and the process stayed alive after its answer. An advisor is the arm's own choice.
+function claudeArgs(prompt, o, tools) {
+  return ['-p', prompt, '--model', o.model, '--output-format', 'stream-json', '--verbose',
+    '--permission-mode', 'bypassPermissions', '--allowedTools', tools, '--max-budget-usd', String(o.caseBudget),
+    '--setting-sources', 'project,local', '--strict-mcp-config', '--no-session-persistence',
+    ...(o.advisor ? ['--settings', JSON.stringify({ advisorModel: o.advisor })] : [])];
+}
+
+// The result event, what each model cost (modelUsage counts the advisor's calls under the advisor's model), and the
+// advisor calls themselves. Measured on a headless run: an assistant event's usage carries no `iterations`; a call is
+// a `server_tool_use` block named `advisor` in the message's content, so those are counted, once per block id.
+export function readRun(stdout) {
+  let res = null; const seen = new Set();
+  for (const l of String(stdout || '').split('\n')) {
+    let e; try { e = JSON.parse(l); } catch { continue; }
+    if (e.type === 'result') res = e;
+    if (e.type === 'assistant' && Array.isArray(e.message?.content)) {
+      for (const c of e.message.content) if (c?.type === 'server_tool_use' && c.name === 'advisor' && c.id) seen.add(c.id);
+    }
+  }
+  const advisorCalls = seen.size;
+  const byModel = Object.fromEntries(Object.entries(res?.modelUsage ?? {}).map(([k, u]) => [k, { cost: Number(u.costUSD ?? 0),
+    input: u.inputTokens ?? 0, output: u.outputTokens ?? 0, cacheRead: u.cacheReadInputTokens ?? 0, cacheWrite: u.cacheCreationInputTokens ?? 0 }]));
+  return { res, byModel, advisorCalls };
+}
+
+const runFields = ({ res, byModel, advisorCalls }) => ({
+  cost: Number(res?.total_cost_usd ?? 0), turns: res?.num_turns ?? null, ms: res?.duration_ms ?? null, byModel, advisorCalls,
+  tokens: res?.usage ? { input: res.usage.input_tokens, output: res.usage.output_tokens, cacheRead: res.usage.cache_read_input_tokens, cacheWrite: res.usage.cache_creation_input_tokens } : null });
+const runError = (res, r) => (!res ? `no result event (rc ${r.status}${r.error ? `, ${r.error.code}` : ''})` : (res.is_error ? (res.subtype || 'error') : null));
+// The CLI. CREW_BENCH_CLAUDE names a node script that stands in for it (the selfcheck's fake, on every platform: a
+// script called `claude` cannot be spawned on Windows without a shell).
+const claude = (args, opts) => (process.env.CREW_BENCH_CLAUDE
+  ? spawnSync(process.execPath, [process.env.CREW_BENCH_CLAUDE, ...args], opts) : spawnSync('claude', args, opts));
+const SPAWN = (b) => ({ cwd: b.root, encoding: 'utf8', env: b.env, maxBuffer: 256 << 20, timeout: 12 * 60 * 1000, killSignal: 'SIGKILL', input: '' });
+
+// A task: the session does the work, then the check it never saw grades it.
+function runTask(c, b, o) {
+  const r = claude(claudeArgs(c.intent, o, 'Bash Read Grep Glob Edit Write'), SPAWN(b));
+  const run = readRun(r.stdout);
+  if (fs.existsSync(path.join(c.dir, 'check'))) fs.cpSync(path.join(c.dir, 'check'), b.root, { recursive: true });
+  const [cmd, ...rest] = c.check ?? [];
+  const k = cmd ? spawnSync(cmd, rest, { cwd: b.root, encoding: 'utf8', env: b.env, timeout: 120000 }) : null;
+  const tail = `${k?.stdout ?? ''}${k?.stderr ?? ''}`.trim().split('\n').slice(-3).join(' | ');
+  return { findings: [], source: 'check', done: !!k && k.status === 0, check: k ? `rc ${k.status}${tail ? ` · ${tail.slice(0, 200)}` : ''}` : 'no check',
+    ...runFields(run), error: runError(run.res, r) };
+}
+
+function runAgent(c, b, o) {
+  copyKit(b.root);
+  const prompt = `files: the staged change (git diff --cached)\nchange: audit\nverify: none\n\n`
+    + `Review the staged change. Its author says: "${c.intent}". Follow crew-code-review: run the pre-pass first `
+    + `(bash .claude/eval/lib/review-prepass/prepass.sh), carry its findings over, then write your own findings to `
+    + `.crewforth/review.json with bash .claude/eval/lib/findings/findings.sh write review <file> (the refuter is `
+    + `not installed here, so write without --verified). Change no other file. End with your findings in the format `
+    + `file:line · severity · category · observation · what to do instead.`;
+  const r = claude(['--agent', 'crew-review-agent', ...claudeArgs(prompt, o, 'Bash Read Grep Glob Write')], SPAWN(b));
+  const run = readRun(r.stdout); const { res } = run;
+  const read = (n) => { try { return JSON.parse(fs.readFileSync(path.join(b.root, '.crewforth', `${n}.json`), 'utf8')); } catch { return null; } };
+  const review = read('review'); const pre = read('review-prepass');
+  const reviewOk = !!review && validate(review, b.root).ok;
+  const own = reviewOk ? flat(review) : fromText(res?.result ?? '');
+  const seen = new Set(); const all = [];
+  for (const f of [...own, ...flat(pre)]) { const k = `${f.file}:${f.line}:${f.rule ?? f.title}`; if (f.file && f.line && !seen.has(k)) { seen.add(k); all.push(f); } }
+  return { findings: all, source: reviewOk ? 'review.json' : 'final answer', prepassRan: !!pre, ...runFields(run), error: runError(res, r) };
+}
+
+export function run(o) {
+  const cases = loadCases(o.cases, o);
+  const tmp = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'crew-bench-')));
+  const rows = []; let spent = 0;
+  try {
+    for (const c of cases) {
+      if (o.arm === 'agent' && spent + o.caseBudget > o.maxCost) { rows.push({ id: c.id, skipped: `the run's cap ($${o.maxCost}) would be passed` }); continue; }
+      const b = build(c, tmp);
+      let out;
+      if (c.kind === 'task') {
+        if (o.arm !== 'agent') { rows.push({ id: c.id, kind: c.kind, skipped: 'a task needs the agent arm' }); continue; }
+        out = runTask(c, b, o); spent += out.cost;
+      }
+      else if (o.arm === 'prepass') {
+        const keep = process.env.CREW_SHELLCHECK; if (o.noTools) process.env.CREW_SHELLCHECK = path.join(tmp, 'no-shellcheck');
+        try { out = { findings: flat(prepass(b.root)), source: 'prepass' }; } finally { if (o.noTools) { if (keep === undefined) delete process.env.CREW_SHELLCHECK; else process.env.CREW_SHELLCHECK = keep; } }
+      }
+      else { out = runAgent(c, b, o); spent += out.cost; }
+      rows.push({ id: c.id, kind: c.kind, stratum: c.stratum, holdout: !!c.holdout, hint: c.kind === 'defect' ? hintInDiff(b.diff) : null, ...out, ...score(c, out.findings) });
+    }
+  } finally { fs.rmSync(tmp, { recursive: true, force: true }); }
+  return summarise(rows, o, spent);
+}
+
+export function summarise(rows, o, spent = 0) {
+  const done = rows.filter((r) => !r.skipped);
+  const defects = done.filter((r) => r.kind === 'defect'); const tasks = done.filter((r) => r.kind === 'task');
+  const pct = (a, b) => (b ? Math.round((1000 * a) / b) / 10 : null);
+  const rec = (list) => ({ caught: list.filter((r) => r.caught).length, of: list.length, pct: pct(list.filter((r) => r.caught).length, list.length) });
+  const claims = done.reduce((s, r) => s + r.claims, 0); const matched = done.reduce((s, r) => s + r.matched, 0);
+  const sum = (k) => done.reduce((n, r) => n + (Number(r[k]) || 0), 0);
+  return { arm: o.arm, model: o.arm === 'agent' ? o.model : null, advisor: o.arm === 'agent' ? o.advisor ?? null : null, split: o.split, cases: rows.length, ran: done.length,
+    tasks: { done: tasks.filter((r) => r.done).length, of: tasks.length }, ms: sum('ms'), advisorCalls: sum('advisorCalls'),
+    recall: rec(defects), recallNoHint: rec(defects.filter((r) => !r.hint)), recallHint: rec(defects.filter((r) => r.hint)),
+    byStratum: Object.fromEntries(['pattern', 'judgement'].map((s) => [s, rec(defects.filter((r) => r.stratum === s))])),
+    precision: { matched, of: claims, pct: pct(matched, claims) },
+    falseAlarms: done.flatMap((r) => r.falseAlarms), cost: Math.round(spent * 100) / 100, errors: done.filter((r) => r.error).map((r) => `${r.id}: ${r.error}`), rows };
+}
+
+function main(argv) {
+  let o; try { o = args(argv); } catch (e) { console.error(`review-bench: ${e.message}`); return 2; }
+  if (o.arm === 'agent' && claude(['--version'], { encoding: 'utf8' }).status !== 0) { console.error('review-bench: the claude CLI does not run here; the agent arm needs it'); return 2; }
+  const s = run(o);
+  const line = (n, x) => `${n} ${x.caught}/${x.of}${x.pct === null ? '' : ` (${x.pct}%)`}`;
+  console.log(`review-bench · arm ${s.arm}${s.model ? ` (${s.model}${s.advisor ? ` + ${s.advisor} advisor` : ''})` : ''} · split ${s.split} · ${s.ran}/${s.cases} cases`);
+  console.log(`  ${line('recall', s.recall)} · ${line('without a hint', s.recallNoHint)} · ${line('with one', s.recallHint)}`);
+  console.log(`  ${line('pattern', s.byStratum.pattern)} · ${line('judgement', s.byStratum.judgement)}`);
+  console.log(`  precision ${s.precision.matched}/${s.precision.of}${s.precision.pct === null ? ' (no claim made)' : ` (${s.precision.pct}%)`} · ${s.falseAlarms.length} false alarm(s)${s.arm === 'agent' ? ` · cost $${s.cost}` : ''}`);
+  if (s.tasks.of) console.log(`  tasks done ${s.tasks.done}/${s.tasks.of}`);
+  if (s.arm === 'agent') console.log(`  time ${Math.round(s.ms / 1000)} s · advisor calls ${s.advisorCalls}`);
+  for (const f of s.falseAlarms) console.log(`    FALSE ALARM ${f.case} ${f.file}:${f.line} ${f.severity} ${String(f.title).slice(0, 100)}`);
+  for (const e of s.errors) console.log(`    ERROR ${e}`);
+  for (const r of s.rows.filter((x) => x.skipped)) console.log(`    NOT RUN ${r.id}: ${r.skipped}`);
+  if (o.out) fs.writeFileSync(o.out, JSON.stringify(s, null, 2) + '\n');
+  let rc = s.errors.length && o.arm === 'agent' ? 1 : 0;
+  if (o.floors) {
+    const fl = JSON.parse(fs.readFileSync(o.floors, 'utf8'))[o.arm] ?? {};
+    for (const [k, v] of Object.entries(fl)) {
+      const got = k === 'precision' ? s.precision.pct : k === 'falseAlarms' ? s.falseAlarms.length : (k === 'recall' ? s.recall : s.byStratum[k.replace(/^recall-/, '')])?.pct;
+      const ok = k === 'falseAlarms' ? got <= v : got !== null && got !== undefined && got >= v;
+      console.log(`  floor ${k}: ${got} ${k === 'falseAlarms' ? '≤' : '≥'} ${v} ${ok ? 'ok' : 'BELOW THE FLOOR'}`);
+      if (!ok) rc = 1;
+    }
+  }
+  return rc;
+}
+
+if (process.argv[1] && fs.realpathSync(process.argv[1]) === fileURLToPath(import.meta.url)) process.exitCode = main(process.argv.slice(2));

package/kit/eval/lib/review-prepass/prepass.mjs

@@ -0,0 +1,203 @@
+#!/usr/bin/env node
+// prepass.mjs — the review's deterministic first pass: cheap checks over the lines a change ADDS, run before the
+// review agent reads the diff, written as a findings document (crewforth.findings/1) the agent starts from.
+//
+//   node prepass.mjs [--root DIR] [--base REF] [--stdout]
+//       default   the staged change (git diff --cached), what the next commit will hold
+//       --base    the working tree against REF (a branch under review: --base origin/next)
+//       --stdout  print the document, write nothing; otherwise .crewforth/review-prepass.json, through findings.mjs
+//   rc 0 = written (or printed), whatever it found · 1 = the document did not verify, nothing written ·
+//   2 = usage, or not a git repository
+//
+// The checks are chosen by the file, and kept narrow on purpose: a first pass that cries wolf trains the reader to
+// skip it. Each one either reads the added line alone (proven) or one fact next to it (inferred):
+//
+//   prepass-secret        a secret-blocklist.txt pattern on an added line (the pre-commit gate's own list and
+//                         .secret-allowlist.txt). Cited by sha256 only: the value is never quoted.
+//   prepass-debug-call    console.log / debugger (JS, TS), breakpoint() / pdb (Python), print( (Dart), outside a
+//                         test and outside a command-line script (a shebang, or a bin/, scripts/ or tool/ folder).
+//   prepass-todo          a TODO, FIXME, XXX or HACK opening a comment added in code, outside tests
+//                         (Markdown is prose, not a promise; the word in a string or a pattern is not a marker).
+//   prepass-shellcheck    shellcheck's own findings on a shell file, kept to the added lines. Without shellcheck
+//                         the shell files are listed as not measured: nothing is installed.
+//   prepass-lockfile      a lock file changed and the manifest beside it did not (inferred: a regenerated lock
+//                         can be on purpose).
+//   prepass-large-change  one file gains more than 500 lines (generated files, locks and snapshots aside).
+//
+// What it does not do: judge. A finding here says a line is there; whether it matters is the reviewer's call, and
+// the reviewer writes its own findings beside these (.crewforth/review.json), in the same shape.
+// Host-independent like findings.mjs: git, Node's standard library, and shellcheck when it is there.
+
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import crypto from 'node:crypto';
+import { spawnSync } from 'node:child_process';
+import { fileURLToPath } from 'node:url';
+import { loadSecretPatterns, isTestPath } from '../crew-map/project.mjs';
+import { SCHEMA_ID, validate, write, serialize } from '../findings/findings.mjs';
+
+const HERE = path.dirname(fileURLToPath(import.meta.url));
+export const NAME = 'review-prepass';
+const LARGE = 500;
+const QUOTE = 200;
+
+const DEBUG = [
+  { files: /\.(m?[jt]sx?|cjs|cts|vue|svelte)$/, re: /\bconsole\.(log|debug|trace)\(|^\s*debugger\s*;?\s*$/, what: 'a console call or debugger statement' },
+  { files: /\.py$/, re: /\bbreakpoint\(\)|\bpdb\.set_trace\(|^\s*import\s+pdb\b/, what: 'a debugger breakpoint' },
+  { files: /\.dart$/, re: /^\s*print\(/, what: 'a print call' },
+];
+const SCRIPT_DIR = /(^|\/)(bin|scripts?|tools?|tool)\//;
+// A marker opens a comment; the word inside a string, a pattern or prose that names it is not one.
+const TODO = /(?:\/\/|#|\/\*|^\s*\*|--|<!--)\s*(TODO|FIXME|XXX|HACK)\b/;
+const PROSE = /\.(md|markdown|txt|rst|adoc)$/i;
+const SHELL = /\.(sh|bash)$/;
+const SECRET_LISTS = /(^|\/)secret-(block|allow)list\.txt$/;
+// A lock file and the manifest that declares what it pins, in the same folder.
+const LOCKS = {
+  'package-lock.json': 'package.json', 'npm-shrinkwrap.json': 'package.json', 'yarn.lock': 'package.json',
+  'pnpm-lock.yaml': 'package.json', 'bun.lockb': 'package.json', 'pubspec.lock': 'pubspec.yaml',
+  'poetry.lock': 'pyproject.toml', 'uv.lock': 'pyproject.toml', 'Pipfile.lock': 'Pipfile', 'go.sum': 'go.mod',
+  'Cargo.lock': 'Cargo.toml', 'composer.lock': 'composer.json', 'Gemfile.lock': 'Gemfile',
+};
+const GENERATED = /(^|\/)(dist|build|vendor|node_modules|__snapshots__)\/|\.(g|freezed|gr|pb)\.dart$|\.min\.(js|css)$|\.snap$|\.lock$|-lock\.(json|yaml)$/;
+
+const sha = (s) => crypto.createHash('sha256').update(s).digest('hex');
+const git = (root, args) => spawnSync('git', ['-C', root, '-c', 'core.quotePath=false', ...args], { encoding: 'utf8', maxBuffer: 256 * 1024 * 1024 });
+
+// The added lines of a unified diff with no context: { file, line, text }[] and the files it touches.
+export function addedLines(diff) {
+  const added = []; const files = new Set(); const binary = new Set();
+  let file = null; let n = 0;
+  for (const raw of diff.split('\n')) {
+    if (raw.startsWith('diff --git ')) { file = null; continue; }
+    if (raw.startsWith('+++ ')) { file = raw === '+++ /dev/null' ? null : raw.slice(6).replace(/\t$/, ''); if (file) files.add(file); continue; }
+    if (raw.startsWith('Binary files ')) { const b = / and b\/(.*) differ$/.exec(raw); if (b) binary.add(b[1]); continue; }
+    const h = /^@@ -\d+(?:,\d+)? \+(\d+)(?:,\d+)? @@/.exec(raw);
+    if (h) { n = Number(h[1]); continue; }
+    if (file && raw.startsWith('+')) { added.push({ file, line: n, text: raw.slice(1).replace(/\r$/, '') }); n++; }
+  }
+  return { added, files: [...files].sort(), binary: [...binary].sort() };
+}
+
+// A finding id from a rule, a file and a line: lower case, the schema's characters only, unique in the document.
+function idOf(seen, rule, file, line) {
+  let id = `${rule}:${file.toLowerCase().replace(/[^a-z0-9._-]+/g, '-')}:${line}`.slice(0, 190);
+  if (seen.has(id)) { let k = 2; while (seen.has(`${id}.${k}`)) k++; id = `${id}.${k}`; }
+  seen.add(id); return id;
+}
+
+export function prepass(root = '.', { base = null } = {}) {
+  const rootReal = fs.realpathSync(root);
+  const top = git(rootReal, ['rev-parse', '--show-toplevel']);
+  if (top.status !== 0) throw new Error('not a git repository');
+  const diffArgs = ['diff', '--no-ext-diff', '--no-textconv', '--no-color', '-U0', '--diff-filter=AMR'];
+  const d = git(rootReal, base ? [...diffArgs, base, '--'] : [...diffArgs, '--cached', '--']);
+  if (d.status !== 0) throw new Error(`git diff failed: ${(d.stderr || '').trim().split('\n')[0]}`);
+  const { added, files, binary } = addedLines(d.stdout);
+
+  const doc = { schema: SCHEMA_ID, producer: { name: NAME, version: '1' }, findings: [], notMeasured: [] };
+  const seen = new Set();
+  const machine = [rootReal, os.homedir()].filter((p) => p && p.length > 1);
+  const add = (rule, severity, confidence, title, at, detail, quote = true) => {
+    // A line that carries this machine's directory, or must not be quoted, is cited by its hash.
+    const ev = { file: at.file, line: at.line };
+    if (quote && !machine.some((m) => at.text.includes(m)) && at.text.trim()) ev.text = at.text.trim().slice(0, QUOTE);
+    else ev.sha256 = sha(at.text);
+    doc.findings.push({ id: idOf(seen, rule, at.file, at.line), rule, title, confidence, severity, ...(detail ? { detail } : {}), evidence: [ev] });
+  };
+
+  // A staged line the working tree no longer has cannot be cited (the citation is checked against the tree).
+  const tree = new Map();
+  const linesOf = (f) => { if (!tree.has(f)) { let t = null; try { t = fs.readFileSync(path.join(rootReal, f), 'utf8').split('\n').map((l) => l.replace(/\r$/, '')); } catch { /* gone */ } tree.set(f, t); } return tree.get(f); };
+  const stale = new Set();
+  const lines = added.filter((a) => { const t = linesOf(a.file); if (t && t[a.line - 1] === a.text) return true; stale.add(a.file); return false; });
+  for (const f of [...stale].sort()) doc.notMeasured.push({ what: f, reason: 'edited again after it was staged: its staged lines are not the ones on disk, so they were not read' });
+  for (const f of binary) doc.notMeasured.push({ what: f, reason: 'a binary file: no line to read' });
+
+  // ---- secrets: the pre-commit gate's list, minus the project's allowlist ----
+  let allow = new Set();
+  try { allow = new Set(fs.readFileSync(path.join(rootReal, '.secret-allowlist.txt'), 'utf8').split('\n').map((l) => l.replace(/\r$/, '')).filter(Boolean)); } catch { /* none */ }
+  const patterns = loadSecretPatterns(path.join(HERE, '..', 'crew-map'), allow);
+  if (!patterns) doc.notMeasured.push({ what: 'prepass-secret', reason: 'secret-blocklist.txt was not found beside the hooks, so no line was checked for a secret' });
+  const byFile = new Map();
+  for (const a of lines) { if (!byFile.has(a.file)) byFile.set(a.file, []); byFile.get(a.file).push(a); }
+
+  for (const [f, list] of [...byFile].sort((x, y) => (x[0] < y[0] ? -1 : 1))) {
+    const test = isTestPath(f);
+    const first = linesOf(f)?.[0] ?? '';
+    const script = first.startsWith('#!') || SCRIPT_DIR.test(f);
+    const debug = DEBUG.find((x) => x.files.test(f));
+    for (const a of list) {
+      if (patterns && !SECRET_LISTS.test(f) && patterns.some((re) => re.test(a.text))) {
+        add('prepass-secret', 'high', 'proven', 'A secret pattern on an added line', a, 'It matches a pattern of secret-blocklist.txt, the list the pre-commit gate refuses; the value is not quoted here.', false);
+        continue;
+      }
+      if (debug && !test && !script && debug.re.test(a.text)) add('prepass-debug-call', 'low', 'proven', `An added ${debug.what}`, a);
+      if (!PROSE.test(f) && !test && TODO.test(a.text)) add('prepass-todo', 'info', 'proven', `An added ${TODO.exec(a.text)[1]}`, a);
+    }
+    if (list.length > LARGE && !GENERATED.test(f)) add('prepass-large-change', 'low', 'proven', `${list.length} lines added to one file`, list[0], 'A change this size is hard to review in one pass; split it, or say in the description what to read first.');
+  }
+
+  // ---- lock files: the manifest beside them should have changed too ----
+  const changed = new Set(files);
+  for (const f of files) {
+    const want = LOCKS[path.posix.basename(f)]; if (!want) continue;
+    const dir = path.posix.dirname(f); const manifest = dir === '.' ? want : `${dir}/${want}`;
+    if (changed.has(manifest) || !fs.existsSync(path.join(rootReal, manifest))) continue;
+    const at = byFile.get(f)?.[0]; if (!at) continue;
+    add('prepass-lockfile', 'low', 'inferred', `${path.posix.basename(f)} changed and ${want} did not`, at, 'A lock file that moves on its own pins different versions than the manifest asks for; regenerate it from the manifest, or say why it moved.');
+  }
+
+  // ---- shell: shellcheck's findings on the added lines ----
+  const shells = [...byFile.keys()].filter((f) => SHELL.test(f) || /^#!.*\b(ba|z|da)?sh\b/.test(linesOf(f)?.[0] ?? '')).sort();
+  if (shells.length) {
+    // CREW_SHELLCHECK names another shellcheck. One ending in .mjs is run with this Node: the self-check's stand-in,
+    // which a Windows runner could not start as a `#!/bin/sh` script (spawnSync runs no interpreter line there).
+    const sc = process.env.CREW_SHELLCHECK || 'shellcheck';
+    const run = (args, opts) => (sc.endsWith('.mjs') ? spawnSync(process.execPath, [sc, ...args], opts) : spawnSync(sc, args, opts));
+    const v = run(['--version'], { encoding: 'utf8' });
+    if (v.status !== 0) doc.notMeasured.push({ what: `prepass-shellcheck (${shells.length} shell file${shells.length === 1 ? '' : 's'})`, reason: 'shellcheck is not installed here, so the shell files were not linted; nothing was installed' });
+    else {
+      for (const f of shells) {
+        const r = run(['-f', 'json1', '-x', f], { cwd: rootReal, encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 });
+        let out; try { out = JSON.parse(r.stdout || '{}'); } catch { doc.notMeasured.push({ what: f, reason: 'shellcheck gave no output this pass can read' }); continue; }
+        const at = new Map(byFile.get(f).map((a) => [a.line, a]));
+        for (const c of out.comments ?? []) {
+          const a = at.get(c.line); if (!a) continue;
+          const sev = { error: 'medium', warning: 'low' }[c.level] ?? 'info';
+          add('prepass-shellcheck', sev, 'proven', `SC${c.code}: ${String(c.message).slice(0, 180)}`, a);
+        }
+      }
+    }
+  }
+  doc.findings.sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
+  return doc;
+}
+
+function main(argv) {
+  let root = '.'; let base = null; let stdout = false;
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i];
+    if (a === '--root') root = argv[++i];
+    else if (a === '--base') base = argv[++i];
+    else if (a === '--stdout') stdout = true;
+    else { console.error(`prepass: unknown argument ${a}\nusage: prepass.mjs [--root DIR] [--base REF] [--stdout]`); return 2; }
+    if (root === undefined || base === undefined) { console.error('prepass: --root and --base need a value'); return 2; }
+  }
+  let doc;
+  try { doc = prepass(root, { base }); } catch (e) { console.error(`prepass: ${e.message}`); return 2; }
+  const count = (s) => doc.findings.filter((f) => f.severity === s).length;
+  const line = `review-prepass: ${doc.findings.length} finding(s) · high ${count('high')} · medium ${count('medium')} · low ${count('low')} · info ${count('info')} · ${doc.notMeasured.length} not measured`;
+  if (stdout) {
+    const r = validate(doc, root);
+    if (!r.ok) { for (const p of r.problems) console.error(`prepass: ${p.at} ${p.code}: ${p.message}`); return 1; }
+    process.stdout.write(serialize(doc)); console.error(line); return 0;
+  }
+  const r = write(doc, root, NAME);
+  if (!r.ok) { for (const p of r.problems) console.error(`prepass: ${p.at} ${p.code}: ${p.message}`); return 1; }
+  console.log(`${line} → ${r.file}`);
+  return 0;
+}
+
+if (process.argv[1] && fs.realpathSync(process.argv[1]) === fileURLToPath(import.meta.url)) process.exitCode = main(process.argv.slice(2));

package/kit/eval/lib/review-prepass/prepass.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+# prepass.sh — the shell entry to prepass.mjs, the review's deterministic first pass. It finds a Node that RUNS
+# and hands over; the arguments and exit codes are prepass.mjs's (0, 1, 2).
+#
+#   bash prepass.sh [--root DIR] [--base REF] [--stdout]
+#       rc 3 = no usable Node here, so nothing was checked. Not a pass: the review then reads the diff without it.
+#
+# The Node is found the way findings.sh finds it (one rule): asked for its version, never trusted by name.
+set -u
+d="${BASH_SOURCE%[/\\]*}"; [ "$d" = "${BASH_SOURCE}" ] && d=.
+FLOOR=18
+usable(){ local v; v="$("$1" -p 'process.versions.node.split(".")[0]' 2>/dev/null)" || return 1
+  case "$v" in ''|*[!0-9]*) return 1 ;; esac; [ "$v" -ge "$FLOOR" ]; }
+NODE=""
+if usable node; then NODE=node
+elif [ -f "$d/../../../studio/ensure-node.sh" ]; then
+  n="$(bash "$d/../../../studio/ensure-node.sh" 2>/dev/null)" && [ -n "$n" ] && usable "$n" && NODE="$n"
+fi
+if [ -z "$NODE" ]; then
+  echo "review-prepass: no Node $FLOOR+ that runs was found, so nothing was checked (this is not a pass)" >&2
+  exit 3
+fi
+exec "$NODE" "$d/prepass.mjs" "$@"

package/kit/eval/lib/settings-json.awk

@@ -7,6 +7,8 @@
 #   awk -v op=gets -v path='a.b|c.d' -f settings-json.awk FILE   several values in one read, tab-separated, "" if absent
 #   awk -v op=strings -v path=permissions.deny -f settings-json.awk FILE   each string element of that array, one per
 #       line, quotes off, escapes as written (rc 1 = not an array)
+#   awk -v op=keys [-v path=P] -f settings-json.awk FILE   the member names of the object there, one per line, quotes
+#       off, in file order (rc 1 = not an object)
 #       path = dot-separated keys, a number indexes an array (0-based), empty = the root. rc 1 = no such path
 #       (or, for len, a scalar there), 10 = FILE is not a JSON object.
 #   awk -v op=get -v path=P -v setk=M -v setv=RAWJSON -f settings-json.awk FILE   the object at P with member M set
@@ -142,6 +144,8 @@ END{
   if(op=="strings"){ if(!(r=doc(1))) exit 10; if(!(r=at(r,path))) exit 1; if(T[r]!="a") exit 1
     for(i=1;i<=N[r];i++){ c=C[r,i]; if(T[c]=="s" && substr(V[c],1,1)=="\"") print substr(V[c],2,length(V[c])-2) }
     exit 0 }
+  if(op=="keys"){ if(!(r=doc(1))) exit 10; if(!(r=at(r,path))) exit 1; if(T[r]!="o") exit 1
+    for(i=1;i<=N[r];i++){ k=K[r,i]; print substr(k,2,length(k)-2) } exit 0 }
   if(op=="overlay"){ if(!(t=doc(1))) exit 10; if(!(q=doc(2))) exit 11; kk="\"" key "\""
     qv=get(q,kk); if(T[qv]!="o") exit 11; tv=get(t,kk); if(tv && T[tv]!="o") exit 10
     nv=mk("o"); for(i=1;i<=N[tv];i++){ x=get(qv,K[tv,i]); add(nv,K[tv,i], x ? x : C[tv,i]) }

package/kit/eval/lib/statusline.sh

@@ -52,7 +52,7 @@ if [ -n "$SID" ] && [ -d .claude ]; then
 fi
 
 # 🛡 The gate's last word in this session.
-GL="${CREW_GATE_LOG:-.claude/gate-log.tsv}"
+GL=""; [ -f "$_here/gate-log.sh" ] && { . "$_here/gate-log.sh"; crew_gate_log_read; GL="$_CREW_GL"; }   # one rule (gate-log.sh)
 if [ -n "$BASE" ] && [ -f "$GL" ] && [ "$GL" -nt "$BASE" ]; then
   last="$(tail -n 1 "$GL" 2>/dev/null)"
   IFS=$'\t' read -r verdict section _ <<< "$last"

package/kit/eval/lib/verify-cmd.sh

@@ -0,0 +1,17 @@
+# Sourced, not run. How a card's verify command is read: one definition, for the gate that runs it
+# (hooks/guard-agent-model.sh) and for doctor, which offers the line that would allow it.
+#
+# A verify command in a sub-folder: `cd apps/mobile && flutter test`, the usual shape in a repository of several
+# projects (measured on one: 41 of 71 refused verify commands had it). Split into the folder and the one command,
+# so the command is judged alone and run in that folder with no chain; a `verify <command>` rule names the command
+# alone. Only a relative folder inside the project, with no .. step, that exists; anything else is left whole, and
+# is refused as a chain.
+crew_verify_split(){  # $1 = the command, $2 = the project root -> _AMVD the folder ("" = the root), _AMVC the command
+  local c="$1" d r root="${2:-.}"
+  _AMVD=""; _AMVC="$c"
+  case "$c" in 'cd '*' && '*) ;; *) return 0 ;; esac
+  d="${c#cd }"; d="${d%% && *}"; r="${c#cd "$d" && }"
+  case "$d" in ''|/*|~*|*[!A-Za-z0-9._/-]*|..|../*|*/..|*/../*) return 0 ;; esac
+  [ -n "$r" ] && [ -d "$root/$d" ] || return 0
+  _AMVD="$d"; _AMVC="$r"
+}

package/kit/eval/review-bench.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+# review-bench.sh — how right the review is, on labelled cases: precision, recall and every false alarm by name.
+# The shell entry to eval/lib/review-bench/bench.mjs; it finds a Node that RUNS and hands over.
+#
+#   bash review-bench.sh --cases DIR [--arm prepass|agent] [--split all|train|holdout] [--only id,id]
+#                        [--model sonnet] [--advisor opus] [--case-budget 0.75] [--max-cost 12] [--floors FILE]
+#                        [--out FILE] [--no-tools]
+#
+# --arm prepass (the default) runs no model and costs nothing. --arm agent runs Claude Code headless as
+# crew-review-agent on every case and SPENDS TOKENS: each case is capped by --case-budget (US dollars) and the run
+# by --max-cost; ask the person paying before running it. rc 3 = no usable Node here (not a pass).
+set -u
+d="${BASH_SOURCE%[/\\]*}"; [ "$d" = "${BASH_SOURCE}" ] && d=.
+FLOOR=18
+usable(){ local v; v="$("$1" -p 'process.versions.node.split(".")[0]' 2>/dev/null)" || return 1
+  case "$v" in ''|*[!0-9]*) return 1 ;; esac; [ "$v" -ge "$FLOOR" ]; }
+NODE=""
+if usable node; then NODE=node
+elif [ -f "$d/../studio/ensure-node.sh" ]; then
+  n="$(bash "$d/../studio/ensure-node.sh" 2>/dev/null)" && [ -n "$n" ] && usable "$n" && NODE="$n"
+fi
+if [ -z "$NODE" ]; then
+  echo "review-bench: no Node $FLOOR+ that runs was found, so nothing was measured (this is not a pass)" >&2
+  exit 3
+fi
+exec "$NODE" "$d/lib/review-bench/bench.mjs" "$@"