crewforth 3.1.1 → 3.2.0-rc.10

package/kit/eval/lib/preset.sh

@@ -0,0 +1,109 @@
+#!/usr/bin/env bash
+# preset.sh — an organisation's settings for Crewforth in one file, applied to a project.
+#
+#   bash preset.sh --apply <name|file.json>    from the project root; a name is one of eval/lib/presets/
+#   bash preset.sh --check <name|file.json>    is it a valid preset? Nothing is written (the installers ask first)
+#   bash preset.sh --show                      the preset in force here, or "none"
+#   bash preset.sh --list                      the presets that ship
+#
+# A preset is plain JSON (see presets/default.json for every key):
+#   critical.add / critical.remove   paths made critical, or not (globs, as `+ glob` / `- glob`)
+#   floors                           [agent, change, risk, model]: a model a class of work runs on at least
+#   verify                           commands the gate may run as a verify command
+#   lang                             en | tr, the language of what Crewforth says (only when the project has none)
+#
+# Where it goes: .claude/crew-model-rules, the file every reader of these rules already reads (the model gate,
+# agent-outcome's scan, doctor, Studio), as one marked block at the top. The project's own lines stay below it, so
+# the rule is the file's own: for a critical path the last line that matches wins, so a project's `- glob` undoes a
+# preset's `+ glob`; a floor only ever raises, so a preset's floor is a minimum a project can raise and not lower.
+# Applying again replaces the block and nothing else; the same preset twice gives the same bytes.
+#
+# rc 0 = applied (or shown) · 1 = the preset is not valid, or names nothing; nothing was written · 2 = usage
+# Every JSON read is Crewforth's own reader (settings-json.awk): no jq, no python.
+set -u
+_here="${BASH_SOURCE%[/\\]*}"; [ "$_here" = "${BASH_SOURCE}" ] && _here=.
+SJ="$_here/settings-json.awk"; DIR="$_here/presets"
+RULES=.claude/crew-model-rules; CONF=.claude/kit.conf
+BEGIN_MARK='# ---- preset:'; END_MARK='# ---- end of preset (lines below are this project'"'"'s own and win over it, see preset.sh) ----'
+
+die(){ echo "preset: $*" >&2; exit 1; }
+jget(){ awk -v op=get -v path="$1" -f "$SJ" "$2" 2>/dev/null; }
+jstrings(){ awk -v op=strings -v path="$1" -f "$SJ" "$2" 2>/dev/null; }
+jlen(){ local n; n="$(awk -v op=len -v path="$1" -f "$SJ" "$2" 2>/dev/null)"; printf '%s' "${n:-0}"; }
+
+case "${1:-}" in
+  --list) for f in "$DIR"/*.json; do [ -f "$f" ] || continue; n="${f##*/}"; printf '%s\n' "${n%.json}"; done; exit 0 ;;
+  --show)
+    [ -f "$RULES" ] || { echo none; exit 0; }
+    while IFS= read -r l || [ -n "$l" ]; do case "$l" in "$BEGIN_MARK "*) l="${l#"$BEGIN_MARK "}"; echo "${l%% *}"; exit 0 ;; esac; done < "$RULES"
+    echo none; exit 0 ;;
+  --apply|--check) [ -n "${2:-}" ] || { echo "usage: preset.sh --apply|--check <name|file.json> | --show | --list" >&2; exit 2; } ;;
+  *) echo "usage: preset.sh --apply|--check <name|file.json> | --show | --list" >&2; exit 2 ;;
+esac
+MODE="$1"
+[ "$MODE" = --check ] || [ -d .claude ] || die "no .claude/ here: run it from the root of a project Crewforth is installed in"
+
+# ---- which file ----
+P="$2"
+case "$P" in *.json|*/*) ;; *) P="$DIR/$P.json" ;; esac
+[ -f "$P" ] || die "no preset at $P (the ones that ship: $(bash "$0" --list 2>/dev/null | tr '\n' ' '))"
+awk -v op=validate -f "$SJ" "$P" 2>/dev/null || die "$P is not a JSON object"
+NAME="$(jget name "$P")"; NAME="${NAME#\"}"; NAME="${NAME%\"}"
+case "$NAME" in ''|*[!a-z0-9-]*) die "$P: name must be lower case letters, digits and -" ;; esac
+
+# ---- what it says, checked before anything is written ----
+LINES=()
+for k in add remove; do
+  sign='+'; [ "$k" = remove ] && sign='-'
+  while IFS= read -r g; do
+    [ -n "$g" ] || continue
+    case "$g" in *[[:space:]\#]*) die "$P: critical.$k holds '$g', which has a space or a #" ;; esac
+    LINES+=("$sign $g")
+  done < <(jstrings "critical.$k" "$P")
+done
+nf="$(jlen floors "$P")"; i=0
+while [ "$i" -lt "$nf" ]; do
+  f=(); while IFS= read -r w; do f+=("$w"); done < <(jstrings "floors.$i" "$P")
+  [ "${#f[@]}" = 4 ] || die "$P: floors[$i] is not [agent, change, risk, model]"
+  case "${f[3]}" in haiku|sonnet|opus) ;; *) die "$P: floors[$i] names the model '${f[3]}', not haiku, sonnet or opus" ;; esac
+  for w in "${f[@]}"; do case "$w" in ''|*[[:space:]\#]*) die "$P: floors[$i] has an empty word, a space or a #" ;; esac; done
+  LINES+=("floor ${f[0]} ${f[1]} ${f[2]} ${f[3]}"); i=$((i+1))
+done
+while IFS= read -r c; do
+  [ -n "$c" ] || continue
+  case "$c" in *[\;\&\|\<\>\`\$\#]*) die "$P: verify '$c' has a shell operator or a #; the gate runs a verify command as one command" ;; esac
+  LINES+=("verify $c")
+done < <(jstrings verify "$P")
+LANGV="$(jget lang "$P")"; LANGV="${LANGV#\"}"; LANGV="${LANGV%\"}"
+case "$LANGV" in ''|null|en|tr) ;; *) die "$P: lang is '$LANGV', not en or tr" ;; esac
+for k in $(awk -v op=keys -f "$SJ" "$P" 2>/dev/null); do
+  case "$k" in about|name|critical|floors|verify|lang) ;; *) die "$P: unknown key '$k' (about, name, critical, floors, verify, lang)" ;; esac
+done
+
+[ "$MODE" = --check ] && { echo "preset: $NAME is valid — ${#LINES[@]} line(s)"; exit 0; }
+
+# ---- the block, at the top of the rules file; the project's own lines kept below it ----
+OWN=""
+if [ -f "$RULES" ]; then
+  skip=0
+  while IFS= read -r l || [ -n "$l" ]; do
+    case "$l" in "$BEGIN_MARK "*) skip=1; continue ;; "$END_MARK") skip=0; continue ;; esac
+    [ "$skip" = 1 ] || OWN+="$l"$'\n'
+  done < "$RULES"
+fi
+T="$RULES.preset.$$"
+{
+  printf '%s %s (written by preset.sh; apply again to change it, edit your own lines below) ----\n' "$BEGIN_MARK" "$NAME"
+  for l in ${LINES[@]+"${LINES[@]}"}; do printf '%s\n' "$l"; done
+  printf '%s\n' "$END_MARK"
+  printf '%s' "$OWN"
+} > "$T" && mv -f "$T" "$RULES" || { rm -f "$T"; die "could not write $RULES"; }
+
+# ---- kit.conf: the preset's language, only where the project has none. Which preset is in force is said once, by
+# the block's own first line (--show reads it): kit.conf is rewritten by every update, the rules file is not.
+if [ -f "$CONF" ] && case "$LANGV" in en|tr) true ;; *) false ;; esac; then
+  hadlang=0; while IFS= read -r l || [ -n "$l" ]; do case "$l" in lang=*) hadlang=1 ;; esac; done < "$CONF"
+  [ "$hadlang" = 0 ] && printf 'lang=%s\n' "$LANGV" >> "$CONF"
+fi
+echo "preset: $NAME applied — ${#LINES[@]} line(s) in $RULES; your own lines below the block were kept"
+exit 0

package/kit/eval/lib/presets/default.json

@@ -0,0 +1,7 @@
+{
+  "about": "Crewforth as it ships: no line of its own. Every key a preset may hold is here, empty. critical.add and critical.remove are globs made critical or not (a critical path runs on opus and only an agent on opus writes there); floors are [agent, change, risk, model] with * for any, a model a class of work runs on at least; verify are commands the model gate may run as a card's verify command; lang (en or tr) is used only where the project has no language yet.",
+  "name": "default",
+  "critical": { "add": [], "remove": [] },
+  "floors": [],
+  "verify": []
+}

package/kit/eval/lib/presets/solo.json

@@ -0,0 +1,7 @@
+{
+  "about": "For one person on their own project: the common test commands may run as a card's verify command, so cheap work can be checked and run on haiku; no path is added.",
+  "name": "solo",
+  "critical": { "add": [], "remove": [] },
+  "floors": [],
+  "verify": ["npm test", "pnpm test", "yarn test", "pytest", "go test ./...", "cargo test", "dotnet test", "flutter test"]
+}

package/kit/eval/lib/presets/strict.json

@@ -0,0 +1,11 @@
+{
+  "about": "For a team that ships to production from this repository: what deploys it is critical too, and the referees of the work run on opus whatever the risk.",
+  "name": "strict",
+  "critical": { "add": ["infra", "deploy", "terraform", "k8s", "helm", ".github/workflows"], "remove": [] },
+  "floors": [
+    ["crew-review-agent", "*", "*", "opus"],
+    ["*", "test-write", "*", "opus"],
+    ["*", "audit", "*", "opus"]
+  ],
+  "verify": []
+}

package/kit/eval/lib/preview.sh

@@ -0,0 +1,150 @@
+# Sourced, not run. What a risky command would have touched, in one line, for the gate that stops or asks about it:
+# the user reads "it would delete 7 uncommitted changes", not only "git reset --hard".
+#
+#   crew_preview <kind> <command>   -> _CREW_PV ("" when there is nothing to say)
+#     reset    git reset --hard: the uncommitted changes to tracked files it discards, and the commits it drops
+#              from the branch when it names a target (git reset --hard HEAD~3)
+#     clean    git clean -f…: the same command as a dry run (-n, with its -d / -x / -X), and how many paths it lists
+#     push     git push --force: the commits on the remote branch the local one does not have, as of the last fetch
+#     rm       rm -r…: the files under each named target inside the project; a target outside it, or a pattern, is
+#              not counted and the line says so
+#     commit   an approved commit: the files and lines it takes (the staged ones, the tracked changes too for -a)
+#     push-ask an approved push: the commits it sends, the first three by subject
+#
+# READ ONLY. Every command here only reads (status, diff --stat, rev-list, log, clean -n, find), nothing fetches,
+# and a count of files stops at 10 000. The caller runs this in a subshell and prints what comes back: a preview
+# that fails, or ends the shell, leaves the gate's decision as it was. It never decides anything.
+
+_PV_MAX=10000
+
+# Is this a repository, without a process where the gate log's helper is loaded (it is, in the gates).
+_pv_repo(){ if declare -F crew_git_dir >/dev/null 2>&1; then crew_git_dir .; else git rev-parse --is-inside-work-tree >/dev/null 2>&1; fi; }
+
+# The current branch's upstream as remote/branch, read from HEAD and the config with builtins: _PV_UP ("" = none).
+# A push is asked about on the hot path of a gate whose cost is pinned per call, so the common case (no upstream
+# yet) opens no process at all. A linked worktree keeps HEAD in its own git dir and the config in the common one.
+_pv_upstream(){
+  _PV_UP=""; local gd="" cd l br="" sec="" rem="" mer=""
+  declare -F crew_git_dir >/dev/null 2>&1 && crew_git_dir . && gd="$_CREW_GD"
+  [ -n "$gd" ] && [ -f "$gd/HEAD" ] || return 1
+  IFS= read -r l < "$gd/HEAD" || return 1; l="${l%$'\r'}"
+  case "$l" in "ref: refs/heads/"*) br="${l#ref: refs/heads/}" ;; *) return 1 ;; esac
+  cd="$gd"; if [ -f "$gd/commondir" ]; then IFS= read -r l < "$gd/commondir"; l="${l%$'\r'}"; case "$l" in /*|[A-Za-z]:*) cd="$l" ;; *) cd="$gd/$l" ;; esac; fi
+  [ -f "$cd/config" ] || return 1
+  while IFS= read -r l || [ -n "$l" ]; do
+    l="${l%$'\r'}"; l="${l#"${l%%[![:space:]]*}"}"
+    case "$l" in
+      "[branch \"$br\"]") sec=1 ;;
+      "["*) sec="" ;;
+      remote*=*) [ -n "$sec" ] && { rem="${l#*=}"; rem="${rem#"${rem%%[![:space:]]*}"}"; } ;;
+      merge*=*) [ -n "$sec" ] && { mer="${l#*=}"; mer="${mer#"${mer%%[![:space:]]*}"}"; } ;;
+    esac
+  done < "$cd/config"
+  [ -n "$rem" ] && [ -n "$mer" ] && [ "$rem" != . ] && _PV_UP="$rem/${mer#refs/heads/}"
+  return 0
+}
+
+_pv_words(){  # $1 = command -> _PV_W (the words after the git subcommand $2, quotes dropped)
+  local s="$1" sub="$2"
+  s="${s#*git}"; s="${s#*"$sub"}"; s="${s%%[;&|]*}"; s="${s//\"/}"; s="${s//\'/}"
+  _PV_W="$s"
+}
+
+# With pathname expansion off for the whole preview: the words of the command are split, never expanded here, so
+# `rm -rf build/*` is said as a pattern instead of each file the hook's own shell would find (measured on Windows).
+crew_preview(){
+  local _pvf=0; case "$-" in *f*) _pvf=1 ;; esac
+  set -f; _crew_preview "$@"; [ "$_pvf" = 1 ] || set +f
+  return 0
+}
+_crew_preview(){
+  _CREW_PV=""
+  local kind="$1" cmd="$2" n t w a flags line
+  case "$kind" in
+    reset|clean|push|commit|push-ask)
+      _pv_repo || { _CREW_PV="Preview: not a git work tree here, nothing counted."; return 0; } ;;
+  esac
+  case "$kind" in
+    reset)
+      n="$(git status --porcelain --untracked-files=no 2>/dev/null | wc -l)"; n="${n//[!0-9]/}"
+      _pv_words "$cmd" reset; t=""
+      for w in $_PV_W; do case "$w" in -*) ;; *) t="$w"; break ;; esac; done
+      a=""
+      if [ -n "$t" ] && git rev-parse -q --verify "$t^{commit}" >/dev/null 2>&1; then
+        a="$(git rev-list --count "$t..HEAD" 2>/dev/null)"; a="${a//[!0-9]/}"
+      fi
+      _CREW_PV="Preview: it would discard ${n:-0} uncommitted change(s) to tracked files${a:+ and drop $a commit(s) from the branch (back to $t)}."
+      ;;
+    clean)
+      _pv_words "$cmd" clean; flags="-n"
+      for w in $_PV_W; do case "$w" in
+        --) break ;;
+        -*) case "$w" in --*) ;; *) case "$w" in *d*) flags="$flags -d" ;; esac; case "$w" in *x*) flags="$flags -x" ;; *X*) flags="$flags -X" ;; esac ;; esac ;;
+      esac; done
+      # shellcheck disable=SC2086 # the flags are words of our own
+      n="$(git clean $flags 2>/dev/null | wc -l)"; n="${n//[!0-9]/}"
+      _CREW_PV="Preview: a dry run (git clean $flags) lists ${n:-0} path(s) it would delete."
+      ;;
+    push)
+      _pv_words "$cmd" push; local rem="" ref="" up=""
+      for w in $_PV_W; do case "$w" in -*) ;; *) if [ -z "$rem" ]; then rem="$w"; elif [ -z "$ref" ]; then ref="$w"; fi ;; esac; done
+      ref="${ref#+}"; ref="${ref##*:}"
+      if [ -n "$rem" ] && [ -n "$ref" ]; then up="refs/remotes/$rem/$ref"
+      else up="$(git rev-parse --abbrev-ref --symbolic-full-name '@{u}' 2>/dev/null)"; fi
+      if [ -n "$up" ] && git rev-parse -q --verify "$up^{commit}" >/dev/null 2>&1; then
+        n="$(git rev-list --count "HEAD..$up" 2>/dev/null)"; n="${n//[!0-9]/}"; up="${up#refs/remotes/}"
+        if [ "${n:-0}" -gt 0 ]; then _CREW_PV="Preview: $up has $n commit(s) this branch does not, as of the last fetch; a forced push would overwrite them."
+        else _CREW_PV="Preview: as of the last fetch, $up has no commit this branch lacks; a fetch now could show otherwise."; fi
+      else
+        _CREW_PV="Preview: no remote branch known for it here (nothing fetched), so what it would overwrite is not counted."
+      fi
+      ;;
+    rm)
+      local root out="" one c
+      root="$(pwd -P 2>/dev/null)"
+      _pv_words "x $cmd" rm
+      for w in $_PV_W; do
+        case "$w" in -*|'') continue ;; *'*'*|*'?'*|'~'*|'$'*) out="${out:+$out; }$w: a pattern, not counted"; continue ;; esac
+        # A drive-letter path (D:/x, as Git Bash passes it) is absolute too: measured on Windows, it read as relative
+        # and the line said "not there" for a folder that was.
+        one="$w"; case "$one" in /*|[A-Za-z]:[/\\]*) ;; *) one="$root/$one" ;; esac
+        if [ ! -e "$one" ]; then out="${out:+$out; }$w: not there"; continue; fi
+        c="$(cd "$one" 2>/dev/null && pwd -P)" || c="$(cd "$(dirname "$one")" 2>/dev/null && pwd -P)/${one##*/}"
+        case "$c/" in "$root"/*) ;; *) out="${out:+$out; }$w: outside the project, not counted"; continue ;; esac
+        n="$(find "$one" -type f 2>/dev/null | head -n $((_PV_MAX + 1)) | wc -l)"; n="${n//[!0-9]/}"
+        [ "${n:-0}" -gt "$_PV_MAX" ] && n="more than $_PV_MAX"
+        out="${out:+$out; }$w: ${n:-0} file(s)"
+      done
+      [ -n "$out" ] && _CREW_PV="Preview: $out."
+      ;;
+    commit)
+      case " $cmd " in *' -a'*|*' --all'*|*' -[A-Za-z]*a'*) line="$(git diff HEAD --shortstat 2>/dev/null)" ;; *) line="$(git diff --cached --shortstat 2>/dev/null)" ;; esac
+      line="${line# }"
+      _CREW_PV="Preview: ${line:-nothing staged}."
+      ;;
+    push-ask)
+      local subj="" k=0 rem="" ref="" src="" tip=HEAD
+      # A push that names its remote and branch (git push origin feat/x) sends to that branch, not the upstream; and
+      # what it sends is the named LOCAL branch's commits (src of src:dst, or the name itself), not the current one's.
+      _pv_words "$cmd" push
+      for w in $_PV_W; do case "$w" in -*) ;; *) if [ -z "$rem" ]; then rem="$w"; elif [ -z "$ref" ]; then ref="$w"; fi ;; esac; done
+      ref="${ref#+}"; src="${ref%%:*}"; ref="${ref##*:}"
+      if [ -n "$src" ] && [ "$src" != HEAD ]; then
+        if [ -f "${_CREW_GD:-.git}/refs/heads/$src" ] || git rev-parse -q --verify "refs/heads/$src" >/dev/null 2>&1; then tip="refs/heads/$src"; fi
+      fi
+      if [ -n "$rem" ] && [ -n "$ref" ] && [ "$ref" != HEAD ]; then
+        if [ -f "${_CREW_GD:-.git}/refs/remotes/$rem/$ref" ] || git rev-parse -q --verify "refs/remotes/$rem/$ref" >/dev/null 2>&1; then _PV_UP="$rem/$ref"
+        else _CREW_PV="Preview: $rem/$ref is not known here; the push would create it on the remote."; return 0; fi
+      else _pv_upstream; fi
+      if [ -n "$_PV_UP" ]; then
+        # One process: the subjects of what it sends, counted here and the first three named.
+        while IFS= read -r line; do k=$((k+1)); [ "$k" -le 3 ] && subj="${subj:+$subj; }${line:0:72}"; done < <(git log --format=%s "$_PV_UP..$tip" 2>/dev/null)
+        [ "$k" -gt 3 ] && subj="$subj …"
+        _CREW_PV="Preview: it sends $k commit(s) to $_PV_UP${subj:+ — $subj}."
+      else
+        _CREW_PV="Preview: the branch has no upstream set here; what it sends is not counted."
+      fi
+      ;;
+  esac
+  return 0
+}

package/kit/eval/lib/review-bench/bench.mjs

@@ -0,0 +1,273 @@
+#!/usr/bin/env node
+// bench.mjs — how right the review is, in numbers: run labelled cases, match what was found to where the defect
+// is, and say the precision, the recall and every false alarm by name.
+//
+//   node bench.mjs --cases DIR [--arm prepass|agent] [--split all|train|holdout] [--only id,id]
+//                  [--model sonnet] [--advisor opus] [--case-budget 0.75] [--max-cost 12] [--floors FILE] [--out FILE]
+//                  [--no-tools]
+//       --no-tools  the pre-pass runs without shellcheck, so its numbers do not depend on what the machine has
+//                   installed (the gate in CI uses it; shellcheck's own precision is not part of the floors)
+//   rc 0 = ran (and, with --floors, at or above every floor) · 1 = below a floor, or the agent arm could not run a
+//   case · 2 = usage
+//
+// A CASE is a directory: case.json, before/ and after/ (the touched files only, with their modes in case.json).
+// It is built as a git repository with `before` committed and `after` staged, so a reviewer meets it the way it
+// meets real work: `git diff --cached`. case.json says what the change claims to do (`intent`, the only thing the
+// reviewer is told) and where the defect is (`expect`: file and line range in the after side; a null range is a
+// file-level defect, a mode bit). A clean case expects nothing.
+//
+// SCORING. A finding matches an expected location when it cites the same file within three lines of the range
+// (any line, for a file-level defect). A defect case is CAUGHT when any finding matches any of its locations:
+// one defect is one defect, however many hunks its fix took. Recall = caught / defect cases. Precision =
+// matching findings / all findings of severity low or above (info is a note, not a claim). Every finding that
+// matches nothing is listed as a false alarm with its case, line and title.
+//
+// A defect case made by undoing a fix can show the fix's own explanation among the removed lines. That is what a
+// reviewer of a real revert would see too, but it is a hint, so such cases are marked and recall is reported for
+// both halves.
+//
+// ARMS. prepass: kit/eval/lib/review-prepass, in process, no model. agent: Claude Code headless, the session run
+// AS crew-review-agent (--agent) with Crewforth's review agent, skills and eval/lib copied into the case's .claude/,
+// on --model, each case capped by --case-budget and the run by --max-cost (it stops before a case that could pass
+// it). The agent arm scores what it wrote to .crewforth/review.json together with the pre-pass it was told to carry
+// over; when it wrote no valid file, its final answer's `file:line · severity` lines are read instead, and the
+// result says so. It runs one review pass: the refuter is not installed in the case, so nothing is --verified.
+//
+// --advisor MODEL runs the agent arm with that advisor (the `advisorModel` setting, passed with --settings; the user's
+// own settings are not read). What a run cost is the result's own total, which counts the advisor's calls; each row
+// also carries the cost and tokens by model (the result's modelUsage) and how many advisor calls were made.
+//
+// A TASK is a case of `kind: "task"`: an implementation, not a review. before/ is the repository, `intent` is what
+// the session is asked to do, check/ is copied in only after the session has ended and `check` (argv, run in the
+// repository) grades it: rc 0 is done. The session runs as itself, not as the review agent.
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { spawnSync } from 'node:child_process';
+import { fileURLToPath } from 'node:url';
+import { prepass } from '../review-prepass/prepass.mjs';
+import { validate } from '../findings/findings.mjs';
+
+const HERE = path.dirname(fileURLToPath(import.meta.url));
+const KIT = path.resolve(HERE, '..', '..', '..');   // kit/ (source) or .claude/ (installed)
+const NEAR = 3;
+const RANK = { info: 0, low: 1, medium: 2, high: 3, critical: 4 };
+const TOKEN = `ghp_${'R3v1'.repeat(9)}`;
+
+function args(argv) {
+  const o = { arm: 'prepass', split: 'all', model: 'sonnet', advisor: null, caseBudget: 0.75, maxCost: 12 };
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i]; const v = () => { const x = argv[++i]; if (x === undefined) throw new Error(`${a} needs a value`); return x; };
+    if (a === '--cases') o.cases = v(); else if (a === '--arm') o.arm = v(); else if (a === '--split') o.split = v();
+    else if (a === '--only') o.only = v().split(','); else if (a === '--model') o.model = v(); else if (a === '--advisor') o.advisor = v();
+    else if (a === '--case-budget') o.caseBudget = Number(v()); else if (a === '--max-cost') o.maxCost = Number(v());
+    else if (a === '--floors') o.floors = v(); else if (a === '--out') o.out = v(); else if (a === '--no-tools') o.noTools = true;
+    else throw new Error(`unknown argument ${a}`);
+  }
+  if (!o.cases || !['prepass', 'agent'].includes(o.arm) || !['all', 'train', 'holdout'].includes(o.split)) throw new Error('--cases DIR is required; --arm prepass|agent; --split all|train|holdout');
+  return o;
+}
+
+export function loadCases(dir, { split = 'all', only = null } = {}) {
+  return fs.readdirSync(dir).sort().filter((d) => fs.existsSync(path.join(dir, d, 'case.json')))
+    .map((d) => ({ ...JSON.parse(fs.readFileSync(path.join(dir, d, 'case.json'), 'utf8')), dir: path.join(dir, d) }))
+    .filter((c) => (split === 'all' || (split === 'holdout') === !!c.holdout) && (!only || only.includes(c.id)));
+}
+
+const ENV = (tmp) => ({ ...process.env, GIT_CONFIG_GLOBAL: path.join(tmp, 'gitconfig'), GIT_CONFIG_NOSYSTEM: '1' });
+function git(root, env, ...a) { const r = spawnSync('git', ['-C', root, ...a], { encoding: 'utf8', env }); if (r.status !== 0) throw new Error(`git ${a.join(' ')}: ${r.stderr}`); return r.stdout; }
+// A folder a build writes (bin, dist, …) is stored with a leading '=' inside a case, so a repository's own commit
+// gates do not read a case's past source file as a build artifact; the case is built under the real name.
+const BUILD_DIRS = new Set(['node_modules', 'bower_components', 'dist', 'build', 'out', 'target', 'bin', 'obj', 'vendor', 'coverage']);
+export const storedPath = (p) => p.split('/').map((s, i, a) => (i < a.length - 1 && BUILD_DIRS.has(s) ? `=${s}` : s)).join('/');
+export const realPath = (p) => p.split('/').map((s) => (s.startsWith('=') && BUILD_DIRS.has(s.slice(1)) ? s.slice(1) : s)).join('/');
+function tree(dir) { const out = []; const walk = (d) => { if (!fs.existsSync(d)) return; for (const e of fs.readdirSync(d, { withFileTypes: true })) { const p = path.join(d, e.name); if (e.isDirectory()) walk(p); else out.push(path.relative(dir, p).split(path.sep).join('/')); } }; walk(dir); return out.sort(); }
+
+// The case as a repository: before committed, after staged. Returns { root, env, diff }.
+export function build(c, tmp) {
+  const root = path.join(tmp, c.id); fs.mkdirSync(root, { recursive: true });
+  fs.writeFileSync(path.join(tmp, 'gitconfig'), '[user]\n\tname = bench\n\temail = bench@bench.invalid\n[core]\n\tautocrlf = false\n\tfileMode = false\n[init]\n\tdefaultBranch = main\n');
+  const env = ENV(tmp);
+  const put = (side) => {
+    for (const s of tree(path.join(c.dir, side))) {
+      const f = realPath(s);
+      const out = path.join(root, f); fs.mkdirSync(path.dirname(out), { recursive: true });
+      fs.writeFileSync(out, fs.readFileSync(path.join(c.dir, side, s), 'utf8').replaceAll('{{TOKEN}}', TOKEN));
+      fs.chmodSync(out, c.modes?.[side]?.[f] === '100755' ? 0o755 : 0o644);
+    }
+  };
+  // A file's mode is written into the index, not read from the disk: Windows has no executable bit, so a case whose
+  // whole diff is a mode (mode-bit) built as no diff at all there. `git init` probes the disk and writes its own
+  // core.filemode into the repository (true on macOS and Linux, false on Windows), so it is set after init, to false,
+  // and every platform takes this one path.
+  const modes = (side) => { for (const [f, m] of Object.entries(c.modes?.[side] ?? {})) git(root, env, 'update-index', `--chmod=${m === '100755' ? '+x' : '-x'}`, '--', f); };
+  git(root, env, 'init', '-q', '.'); git(root, env, 'config', 'core.fileMode', 'false');
+  put('before'); git(root, env, 'add', '-A'); modes('before'); git(root, env, 'commit', '-qm', 'base', '--allow-empty');
+  if (c.kind !== 'task') {
+    for (const s of tree(path.join(c.dir, 'before'))) if (!fs.existsSync(path.join(c.dir, 'after', s))) fs.rmSync(path.join(root, realPath(s)));
+    put('after'); git(root, env, 'add', '-A'); modes('after');
+  }
+  fs.appendFileSync(path.join(root, '.git', 'info', 'exclude'), '.claude/\n.crewforth/\n');
+  const diff = git(root, env, 'diff', '--cached', '--no-ext-diff', '-U3');
+  return { root, env, diff };
+}
+
+// Whether the removed lines of the diff carry a comment: the fix's own explanation, visible to the reviewer.
+export const hintInDiff = (diff) => diff.split('\n').some((l) => /^-(?!--)\s*(#|\/\/|\/\*|\*)\s*\S/.test(l) && !/^-\s*#!/.test(l));
+
+export function score(c, findings) {
+  const claims = findings.filter((f) => (RANK[f.severity] ?? 1) >= RANK.low);
+  const hits = (f) => (c.expect ?? []).some((e) => e.file === f.file && (e.from == null || (f.line >= e.from - NEAR && f.line <= e.to + NEAR)));
+  const matched = claims.filter(hits);
+  return { caught: c.kind === 'defect' ? findings.some(hits) : null, claims: claims.length, matched: matched.length,
+    falseAlarms: claims.filter((f) => !hits(f)).map((f) => ({ case: c.id, file: f.file, line: f.line, severity: f.severity, title: f.title })) };
+}
+
+const flat = (doc) => (doc?.findings ?? []).map((f) => ({ file: f.evidence?.[0]?.file, line: f.evidence?.[0]?.line, severity: f.severity ?? 'low', title: f.title, rule: f.rule }));
+// `file:line · severity · …` lines of a final answer, for a review that wrote no file.
+export const fromText = (text) => [...String(text).matchAll(/([\w./-]+\.[\w]+):(\d+)\s*[·|-]\s*(critical|high|medium|low|info)\b[^\n]*/gi)]
+  .map((m) => ({ file: m[1].replace(/^\.\//, ''), line: Number(m[2]), severity: m[3].toLowerCase(), title: m[0].slice(0, 160) }));
+
+function copyKit(root) {
+  const dst = path.join(root, '.claude');
+  const parts = [['agents/crew-review-agent.md'], ['skills/crew-code-review'], ['skills/docs-writer'], ['eval/lib/review-prepass'], ['eval/lib/findings'], ['eval/lib/crew-map'], ['hooks/secret-blocklist.txt']];
+  for (const [p] of parts) { const s = path.join(KIT, p); if (fs.existsSync(s)) fs.cpSync(s, path.join(dst, p), { recursive: true }); }
+}
+
+// The flags every headless run shares. The bench measures the model and Crewforth, not the machine it runs on: no
+// user settings (an advisor model, hooks), no MCP server of the user's, no session written. Seen without them: a run
+// took the user's advisor model, and the process stayed alive after its answer. An advisor is the arm's own choice.
+function claudeArgs(prompt, o, tools) {
+  return ['-p', prompt, '--model', o.model, '--output-format', 'stream-json', '--verbose',
+    '--permission-mode', 'bypassPermissions', '--allowedTools', tools, '--max-budget-usd', String(o.caseBudget),
+    '--setting-sources', 'project,local', '--strict-mcp-config', '--no-session-persistence',
+    ...(o.advisor ? ['--settings', JSON.stringify({ advisorModel: o.advisor })] : [])];
+}
+
+// The result event, what each model cost (modelUsage counts the advisor's calls under the advisor's model), and the
+// advisor calls themselves. Measured on a headless run: an assistant event's usage carries no `iterations`; a call is
+// a `server_tool_use` block named `advisor` in the message's content, so those are counted, once per block id.
+export function readRun(stdout) {
+  let res = null; const seen = new Set();
+  for (const l of String(stdout || '').split('\n')) {
+    let e; try { e = JSON.parse(l); } catch { continue; }
+    if (e.type === 'result') res = e;
+    if (e.type === 'assistant' && Array.isArray(e.message?.content)) {
+      for (const c of e.message.content) if (c?.type === 'server_tool_use' && c.name === 'advisor' && c.id) seen.add(c.id);
+    }
+  }
+  const advisorCalls = seen.size;
+  const byModel = Object.fromEntries(Object.entries(res?.modelUsage ?? {}).map(([k, u]) => [k, { cost: Number(u.costUSD ?? 0),
+    input: u.inputTokens ?? 0, output: u.outputTokens ?? 0, cacheRead: u.cacheReadInputTokens ?? 0, cacheWrite: u.cacheCreationInputTokens ?? 0 }]));
+  return { res, byModel, advisorCalls };
+}
+
+const runFields = ({ res, byModel, advisorCalls }) => ({
+  cost: Number(res?.total_cost_usd ?? 0), turns: res?.num_turns ?? null, ms: res?.duration_ms ?? null, byModel, advisorCalls,
+  tokens: res?.usage ? { input: res.usage.input_tokens, output: res.usage.output_tokens, cacheRead: res.usage.cache_read_input_tokens, cacheWrite: res.usage.cache_creation_input_tokens } : null });
+const runError = (res, r) => (!res ? `no result event (rc ${r.status}${r.error ? `, ${r.error.code}` : ''})` : (res.is_error ? (res.subtype || 'error') : null));
+// The CLI. CREW_BENCH_CLAUDE names a node script that stands in for it (the selfcheck's fake, on every platform: a
+// script called `claude` cannot be spawned on Windows without a shell).
+const claude = (args, opts) => (process.env.CREW_BENCH_CLAUDE
+  ? spawnSync(process.execPath, [process.env.CREW_BENCH_CLAUDE, ...args], opts) : spawnSync('claude', args, opts));
+const SPAWN = (b) => ({ cwd: b.root, encoding: 'utf8', env: b.env, maxBuffer: 256 << 20, timeout: 12 * 60 * 1000, killSignal: 'SIGKILL', input: '' });
+
+// A task: the session does the work, then the check it never saw grades it.
+function runTask(c, b, o) {
+  const r = claude(claudeArgs(c.intent, o, 'Bash Read Grep Glob Edit Write'), SPAWN(b));
+  const run = readRun(r.stdout);
+  if (fs.existsSync(path.join(c.dir, 'check'))) fs.cpSync(path.join(c.dir, 'check'), b.root, { recursive: true });
+  const [cmd, ...rest] = c.check ?? [];
+  const k = cmd ? spawnSync(cmd, rest, { cwd: b.root, encoding: 'utf8', env: b.env, timeout: 120000 }) : null;
+  const tail = `${k?.stdout ?? ''}${k?.stderr ?? ''}`.trim().split('\n').slice(-3).join(' | ');
+  return { findings: [], source: 'check', done: !!k && k.status === 0, check: k ? `rc ${k.status}${tail ? ` · ${tail.slice(0, 200)}` : ''}` : 'no check',
+    ...runFields(run), error: runError(run.res, r) };
+}
+
+function runAgent(c, b, o) {
+  copyKit(b.root);
+  const prompt = `files: the staged change (git diff --cached)\nchange: audit\nverify: none\n\n`
+    + `Review the staged change. Its author says: "${c.intent}". Follow crew-code-review: run the pre-pass first `
+    + `(bash .claude/eval/lib/review-prepass/prepass.sh), carry its findings over, then write your own findings to `
+    + `.crewforth/review.json with bash .claude/eval/lib/findings/findings.sh write review <file> (the refuter is `
+    + `not installed here, so write without --verified). Change no other file. End with your findings in the format `
+    + `file:line · severity · category · observation · what to do instead.`;
+  const r = claude(['--agent', 'crew-review-agent', ...claudeArgs(prompt, o, 'Bash Read Grep Glob Write')], SPAWN(b));
+  const run = readRun(r.stdout); const { res } = run;
+  const read = (n) => { try { return JSON.parse(fs.readFileSync(path.join(b.root, '.crewforth', `${n}.json`), 'utf8')); } catch { return null; } };
+  const review = read('review'); const pre = read('review-prepass');
+  const reviewOk = !!review && validate(review, b.root).ok;
+  const own = reviewOk ? flat(review) : fromText(res?.result ?? '');
+  const seen = new Set(); const all = [];
+  for (const f of [...own, ...flat(pre)]) { const k = `${f.file}:${f.line}:${f.rule ?? f.title}`; if (f.file && f.line && !seen.has(k)) { seen.add(k); all.push(f); } }
+  return { findings: all, source: reviewOk ? 'review.json' : 'final answer', prepassRan: !!pre, ...runFields(run), error: runError(res, r) };
+}
+
+export function run(o) {
+  const cases = loadCases(o.cases, o);
+  const tmp = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'crew-bench-')));
+  const rows = []; let spent = 0;
+  try {
+    for (const c of cases) {
+      if (o.arm === 'agent' && spent + o.caseBudget > o.maxCost) { rows.push({ id: c.id, skipped: `the run's cap ($${o.maxCost}) would be passed` }); continue; }
+      const b = build(c, tmp);
+      let out;
+      if (c.kind === 'task') {
+        if (o.arm !== 'agent') { rows.push({ id: c.id, kind: c.kind, skipped: 'a task needs the agent arm' }); continue; }
+        out = runTask(c, b, o); spent += out.cost;
+      }
+      else if (o.arm === 'prepass') {
+        const keep = process.env.CREW_SHELLCHECK; if (o.noTools) process.env.CREW_SHELLCHECK = path.join(tmp, 'no-shellcheck');
+        try { out = { findings: flat(prepass(b.root)), source: 'prepass' }; } finally { if (o.noTools) { if (keep === undefined) delete process.env.CREW_SHELLCHECK; else process.env.CREW_SHELLCHECK = keep; } }
+      }
+      else { out = runAgent(c, b, o); spent += out.cost; }
+      rows.push({ id: c.id, kind: c.kind, stratum: c.stratum, holdout: !!c.holdout, hint: c.kind === 'defect' ? hintInDiff(b.diff) : null, ...out, ...score(c, out.findings) });
+    }
+  } finally { fs.rmSync(tmp, { recursive: true, force: true }); }
+  return summarise(rows, o, spent);
+}
+
+export function summarise(rows, o, spent = 0) {
+  const done = rows.filter((r) => !r.skipped);
+  const defects = done.filter((r) => r.kind === 'defect'); const tasks = done.filter((r) => r.kind === 'task');
+  const pct = (a, b) => (b ? Math.round((1000 * a) / b) / 10 : null);
+  const rec = (list) => ({ caught: list.filter((r) => r.caught).length, of: list.length, pct: pct(list.filter((r) => r.caught).length, list.length) });
+  const claims = done.reduce((s, r) => s + r.claims, 0); const matched = done.reduce((s, r) => s + r.matched, 0);
+  const sum = (k) => done.reduce((n, r) => n + (Number(r[k]) || 0), 0);
+  return { arm: o.arm, model: o.arm === 'agent' ? o.model : null, advisor: o.arm === 'agent' ? o.advisor ?? null : null, split: o.split, cases: rows.length, ran: done.length,
+    tasks: { done: tasks.filter((r) => r.done).length, of: tasks.length }, ms: sum('ms'), advisorCalls: sum('advisorCalls'),
+    recall: rec(defects), recallNoHint: rec(defects.filter((r) => !r.hint)), recallHint: rec(defects.filter((r) => r.hint)),
+    byStratum: Object.fromEntries(['pattern', 'judgement'].map((s) => [s, rec(defects.filter((r) => r.stratum === s))])),
+    precision: { matched, of: claims, pct: pct(matched, claims) },
+    falseAlarms: done.flatMap((r) => r.falseAlarms), cost: Math.round(spent * 100) / 100, errors: done.filter((r) => r.error).map((r) => `${r.id}: ${r.error}`), rows };
+}
+
+function main(argv) {
+  let o; try { o = args(argv); } catch (e) { console.error(`review-bench: ${e.message}`); return 2; }
+  if (o.arm === 'agent' && claude(['--version'], { encoding: 'utf8' }).status !== 0) { console.error('review-bench: the claude CLI does not run here; the agent arm needs it'); return 2; }
+  const s = run(o);
+  const line = (n, x) => `${n} ${x.caught}/${x.of}${x.pct === null ? '' : ` (${x.pct}%)`}`;
+  console.log(`review-bench · arm ${s.arm}${s.model ? ` (${s.model}${s.advisor ? ` + ${s.advisor} advisor` : ''})` : ''} · split ${s.split} · ${s.ran}/${s.cases} cases`);
+  console.log(`  ${line('recall', s.recall)} · ${line('without a hint', s.recallNoHint)} · ${line('with one', s.recallHint)}`);
+  console.log(`  ${line('pattern', s.byStratum.pattern)} · ${line('judgement', s.byStratum.judgement)}`);
+  console.log(`  precision ${s.precision.matched}/${s.precision.of}${s.precision.pct === null ? ' (no claim made)' : ` (${s.precision.pct}%)`} · ${s.falseAlarms.length} false alarm(s)${s.arm === 'agent' ? ` · cost $${s.cost}` : ''}`);
+  if (s.tasks.of) console.log(`  tasks done ${s.tasks.done}/${s.tasks.of}`);
+  if (s.arm === 'agent') console.log(`  time ${Math.round(s.ms / 1000)} s · advisor calls ${s.advisorCalls}`);
+  for (const f of s.falseAlarms) console.log(`    FALSE ALARM ${f.case} ${f.file}:${f.line} ${f.severity} ${String(f.title).slice(0, 100)}`);
+  for (const e of s.errors) console.log(`    ERROR ${e}`);
+  for (const r of s.rows.filter((x) => x.skipped)) console.log(`    NOT RUN ${r.id}: ${r.skipped}`);
+  if (o.out) fs.writeFileSync(o.out, JSON.stringify(s, null, 2) + '\n');
+  let rc = s.errors.length && o.arm === 'agent' ? 1 : 0;
+  if (o.floors) {
+    const fl = JSON.parse(fs.readFileSync(o.floors, 'utf8'))[o.arm] ?? {};
+    for (const [k, v] of Object.entries(fl)) {
+      const got = k === 'precision' ? s.precision.pct : k === 'falseAlarms' ? s.falseAlarms.length : (k === 'recall' ? s.recall : s.byStratum[k.replace(/^recall-/, '')])?.pct;
+      const ok = k === 'falseAlarms' ? got <= v : got !== null && got !== undefined && got >= v;
+      console.log(`  floor ${k}: ${got} ${k === 'falseAlarms' ? '≤' : '≥'} ${v} ${ok ? 'ok' : 'BELOW THE FLOOR'}`);
+      if (!ok) rc = 1;
+    }
+  }
+  return rc;
+}
+
+if (process.argv[1] && fs.realpathSync(process.argv[1]) === fileURLToPath(import.meta.url)) process.exitCode = main(process.argv.slice(2));