crewforth 3.2.0-rc.1 → 3.2.0-rc.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +133 -3
- package/README.md +2 -0
- package/VERSION +1 -1
- package/adopt.sh +55 -6
- package/bin/cli.js +3 -2
- package/kit/CLAUDE.md +6 -6
- package/kit/agents/crew-review-agent.md +2 -0
- package/kit/eval/doctor.sh +37 -3
- package/kit/eval/gate-report.sh +2 -0
- package/kit/eval/lib/crew-map/narrative.mjs +42 -8
- package/kit/eval/lib/crew-map/project.mjs +3 -2
- package/kit/eval/lib/crew-map/rules/tech.json +35 -0
- package/kit/eval/lib/crew-map/scan.mjs +56 -4
- package/kit/eval/lib/crew-map/stacks.mjs +114 -8
- package/kit/eval/lib/crew-map/surface.mjs +62 -1
- package/kit/eval/lib/gate-log.sh +52 -0
- package/kit/eval/lib/json-string.awk +27 -0
- package/kit/eval/lib/preset.sh +109 -0
- package/kit/eval/lib/presets/default.json +7 -0
- package/kit/eval/lib/presets/solo.json +7 -0
- package/kit/eval/lib/presets/strict.json +11 -0
- package/kit/eval/lib/preview.sh +150 -0
- package/kit/eval/lib/review-bench/bench.mjs +273 -0
- package/kit/eval/lib/review-prepass/prepass.mjs +203 -0
- package/kit/eval/lib/review-prepass/prepass.sh +23 -0
- package/kit/eval/lib/settings-default.sh +36 -0
- package/kit/eval/lib/settings-json.awk +14 -4
- package/kit/eval/lib/statusline.sh +4 -19
- package/kit/eval/lib/verify-cmd.sh +17 -0
- package/kit/eval/lib/version.sh +41 -0
- package/kit/eval/review-bench.sh +26 -0
- package/kit/eval/smoke-test.sh +251 -21
- package/kit/hooks/agent-outcome.sh +20 -9
- package/kit/hooks/board-sync.sh +2 -23
- package/kit/hooks/guard-agent-model.sh +114 -15
- package/kit/hooks/guard-bash.sh +36 -21
- package/kit/hooks/guard-write.sh +3 -7
- package/kit/hooks/precompact-snapshot.sh +75 -0
- package/kit/hooks/session-rehydrate.sh +25 -2
- package/kit/hooks/session-update-check.sh +9 -8
- package/kit/settings.json +12 -0
- package/kit/skills/crew-code-review/SKILL.md +16 -0
- package/kit/skills/crew-map/SKILL.md +16 -5
- package/kit/skills/crew-review/SKILL.md +2 -1
- package/kit/studio/README.md +21 -10
- package/kit/studio/server/index.js +85 -22
- package/kit/studio/server/lib/code-stamp.js +35 -0
- package/kit/studio/server/lib/graph.js +9 -1
- package/kit/studio/server/lib/kit-telemetry.js +30 -5
- package/kit/studio/server/lib/lan.js +87 -0
- package/kit/studio/server/lib/owned-stream.js +56 -0
- package/kit/studio/server/lib/projects.js +11 -4
- package/kit/studio/server/lib/redact.js +60 -0
- package/kit/studio/server/lib/remote.js +104 -18
- package/kit/studio/server/lib/session.js +1 -1
- package/kit/studio/server/lib/terminal.js +51 -13
- package/kit/studio/server/lib/transcript.js +32 -0
- package/kit/studio/web/app.js +79 -11
- package/kit/studio/web/canvas.js +14 -2
- package/kit/studio/web/chat.js +114 -24
- package/kit/studio/web/graph-plan.js +3 -2
- package/kit/studio/web/liveness.js +14 -0
- package/kit/studio/web/map-plan.js +72 -3
- package/kit/studio/web/map.js +103 -10
- package/kit/studio/web/qr.js +200 -0
- package/kit/studio/web/style.css +40 -0
- package/kit/studio/web/tokens.css +6 -0
- package/package.json +1 -1
- package/start.sh +14 -1
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
# Sourced, not run. What a risky command would have touched, in one line, for the gate that stops or asks about it:
|
|
2
|
+
# the user reads "it would delete 7 uncommitted changes", not only "git reset --hard".
|
|
3
|
+
#
|
|
4
|
+
# crew_preview <kind> <command> -> _CREW_PV ("" when there is nothing to say)
|
|
5
|
+
# reset git reset --hard: the uncommitted changes to tracked files it discards, and the commits it drops
|
|
6
|
+
# from the branch when it names a target (git reset --hard HEAD~3)
|
|
7
|
+
# clean git clean -f…: the same command as a dry run (-n, with its -d / -x / -X), and how many paths it lists
|
|
8
|
+
# push git push --force: the commits on the remote branch the local one does not have, as of the last fetch
|
|
9
|
+
# rm rm -r…: the files under each named target inside the project; a target outside it, or a pattern, is
|
|
10
|
+
# not counted and the line says so
|
|
11
|
+
# commit an approved commit: the files and lines it takes (the staged ones, the tracked changes too for -a)
|
|
12
|
+
# push-ask an approved push: the commits it sends, the first three by subject
|
|
13
|
+
#
|
|
14
|
+
# READ ONLY. Every command here only reads (status, diff --stat, rev-list, log, clean -n, find), nothing fetches,
|
|
15
|
+
# and a count of files stops at 10 000. The caller runs this in a subshell and prints what comes back: a preview
|
|
16
|
+
# that fails, or ends the shell, leaves the gate's decision as it was. It never decides anything.
|
|
17
|
+
|
|
18
|
+
_PV_MAX=10000
|
|
19
|
+
|
|
20
|
+
# Is this a repository, without a process where the gate log's helper is loaded (it is, in the gates).
|
|
21
|
+
_pv_repo(){ if declare -F crew_git_dir >/dev/null 2>&1; then crew_git_dir .; else git rev-parse --is-inside-work-tree >/dev/null 2>&1; fi; }
|
|
22
|
+
|
|
23
|
+
# The current branch's upstream as remote/branch, read from HEAD and the config with builtins: _PV_UP ("" = none).
|
|
24
|
+
# A push is asked about on the hot path of a gate whose cost is pinned per call, so the common case (no upstream
|
|
25
|
+
# yet) opens no process at all. A linked worktree keeps HEAD in its own git dir and the config in the common one.
|
|
26
|
+
_pv_upstream(){
|
|
27
|
+
_PV_UP=""; local gd="" cd l br="" sec="" rem="" mer=""
|
|
28
|
+
declare -F crew_git_dir >/dev/null 2>&1 && crew_git_dir . && gd="$_CREW_GD"
|
|
29
|
+
[ -n "$gd" ] && [ -f "$gd/HEAD" ] || return 1
|
|
30
|
+
IFS= read -r l < "$gd/HEAD" || return 1; l="${l%$'\r'}"
|
|
31
|
+
case "$l" in "ref: refs/heads/"*) br="${l#ref: refs/heads/}" ;; *) return 1 ;; esac
|
|
32
|
+
cd="$gd"; if [ -f "$gd/commondir" ]; then IFS= read -r l < "$gd/commondir"; l="${l%$'\r'}"; case "$l" in /*|[A-Za-z]:*) cd="$l" ;; *) cd="$gd/$l" ;; esac; fi
|
|
33
|
+
[ -f "$cd/config" ] || return 1
|
|
34
|
+
while IFS= read -r l || [ -n "$l" ]; do
|
|
35
|
+
l="${l%$'\r'}"; l="${l#"${l%%[![:space:]]*}"}"
|
|
36
|
+
case "$l" in
|
|
37
|
+
"[branch \"$br\"]") sec=1 ;;
|
|
38
|
+
"["*) sec="" ;;
|
|
39
|
+
remote*=*) [ -n "$sec" ] && { rem="${l#*=}"; rem="${rem#"${rem%%[![:space:]]*}"}"; } ;;
|
|
40
|
+
merge*=*) [ -n "$sec" ] && { mer="${l#*=}"; mer="${mer#"${mer%%[![:space:]]*}"}"; } ;;
|
|
41
|
+
esac
|
|
42
|
+
done < "$cd/config"
|
|
43
|
+
[ -n "$rem" ] && [ -n "$mer" ] && [ "$rem" != . ] && _PV_UP="$rem/${mer#refs/heads/}"
|
|
44
|
+
return 0
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
_pv_words(){ # $1 = command -> _PV_W (the words after the git subcommand $2, quotes dropped)
|
|
48
|
+
local s="$1" sub="$2"
|
|
49
|
+
s="${s#*git}"; s="${s#*"$sub"}"; s="${s%%[;&|]*}"; s="${s//\"/}"; s="${s//\'/}"
|
|
50
|
+
_PV_W="$s"
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
# With pathname expansion off for the whole preview: the words of the command are split, never expanded here, so
|
|
54
|
+
# `rm -rf build/*` is said as a pattern instead of each file the hook's own shell would find (measured on Windows).
|
|
55
|
+
crew_preview(){
|
|
56
|
+
local _pvf=0; case "$-" in *f*) _pvf=1 ;; esac
|
|
57
|
+
set -f; _crew_preview "$@"; [ "$_pvf" = 1 ] || set +f
|
|
58
|
+
return 0
|
|
59
|
+
}
|
|
60
|
+
_crew_preview(){
|
|
61
|
+
_CREW_PV=""
|
|
62
|
+
local kind="$1" cmd="$2" n t w a flags line
|
|
63
|
+
case "$kind" in
|
|
64
|
+
reset|clean|push|commit|push-ask)
|
|
65
|
+
_pv_repo || { _CREW_PV="Preview: not a git work tree here, nothing counted."; return 0; } ;;
|
|
66
|
+
esac
|
|
67
|
+
case "$kind" in
|
|
68
|
+
reset)
|
|
69
|
+
n="$(git status --porcelain --untracked-files=no 2>/dev/null | wc -l)"; n="${n//[!0-9]/}"
|
|
70
|
+
_pv_words "$cmd" reset; t=""
|
|
71
|
+
for w in $_PV_W; do case "$w" in -*) ;; *) t="$w"; break ;; esac; done
|
|
72
|
+
a=""
|
|
73
|
+
if [ -n "$t" ] && git rev-parse -q --verify "$t^{commit}" >/dev/null 2>&1; then
|
|
74
|
+
a="$(git rev-list --count "$t..HEAD" 2>/dev/null)"; a="${a//[!0-9]/}"
|
|
75
|
+
fi
|
|
76
|
+
_CREW_PV="Preview: it would discard ${n:-0} uncommitted change(s) to tracked files${a:+ and drop $a commit(s) from the branch (back to $t)}."
|
|
77
|
+
;;
|
|
78
|
+
clean)
|
|
79
|
+
_pv_words "$cmd" clean; flags="-n"
|
|
80
|
+
for w in $_PV_W; do case "$w" in
|
|
81
|
+
--) break ;;
|
|
82
|
+
-*) case "$w" in --*) ;; *) case "$w" in *d*) flags="$flags -d" ;; esac; case "$w" in *x*) flags="$flags -x" ;; *X*) flags="$flags -X" ;; esac ;; esac ;;
|
|
83
|
+
esac; done
|
|
84
|
+
# shellcheck disable=SC2086 # the flags are words of our own
|
|
85
|
+
n="$(git clean $flags 2>/dev/null | wc -l)"; n="${n//[!0-9]/}"
|
|
86
|
+
_CREW_PV="Preview: a dry run (git clean $flags) lists ${n:-0} path(s) it would delete."
|
|
87
|
+
;;
|
|
88
|
+
push)
|
|
89
|
+
_pv_words "$cmd" push; local rem="" ref="" up=""
|
|
90
|
+
for w in $_PV_W; do case "$w" in -*) ;; *) if [ -z "$rem" ]; then rem="$w"; elif [ -z "$ref" ]; then ref="$w"; fi ;; esac; done
|
|
91
|
+
ref="${ref#+}"; ref="${ref##*:}"
|
|
92
|
+
if [ -n "$rem" ] && [ -n "$ref" ]; then up="refs/remotes/$rem/$ref"
|
|
93
|
+
else up="$(git rev-parse --abbrev-ref --symbolic-full-name '@{u}' 2>/dev/null)"; fi
|
|
94
|
+
if [ -n "$up" ] && git rev-parse -q --verify "$up^{commit}" >/dev/null 2>&1; then
|
|
95
|
+
n="$(git rev-list --count "HEAD..$up" 2>/dev/null)"; n="${n//[!0-9]/}"; up="${up#refs/remotes/}"
|
|
96
|
+
if [ "${n:-0}" -gt 0 ]; then _CREW_PV="Preview: $up has $n commit(s) this branch does not, as of the last fetch; a forced push would overwrite them."
|
|
97
|
+
else _CREW_PV="Preview: as of the last fetch, $up has no commit this branch lacks; a fetch now could show otherwise."; fi
|
|
98
|
+
else
|
|
99
|
+
_CREW_PV="Preview: no remote branch known for it here (nothing fetched), so what it would overwrite is not counted."
|
|
100
|
+
fi
|
|
101
|
+
;;
|
|
102
|
+
rm)
|
|
103
|
+
local root out="" one c
|
|
104
|
+
root="$(pwd -P 2>/dev/null)"
|
|
105
|
+
_pv_words "x $cmd" rm
|
|
106
|
+
for w in $_PV_W; do
|
|
107
|
+
case "$w" in -*|'') continue ;; *'*'*|*'?'*|'~'*|'$'*) out="${out:+$out; }$w: a pattern, not counted"; continue ;; esac
|
|
108
|
+
# A drive-letter path (D:/x, as Git Bash passes it) is absolute too: measured on Windows, it read as relative
|
|
109
|
+
# and the line said "not there" for a folder that was.
|
|
110
|
+
one="$w"; case "$one" in /*|[A-Za-z]:[/\\]*) ;; *) one="$root/$one" ;; esac
|
|
111
|
+
if [ ! -e "$one" ]; then out="${out:+$out; }$w: not there"; continue; fi
|
|
112
|
+
c="$(cd "$one" 2>/dev/null && pwd -P)" || c="$(cd "$(dirname "$one")" 2>/dev/null && pwd -P)/${one##*/}"
|
|
113
|
+
case "$c/" in "$root"/*) ;; *) out="${out:+$out; }$w: outside the project, not counted"; continue ;; esac
|
|
114
|
+
n="$(find "$one" -type f 2>/dev/null | head -n $((_PV_MAX + 1)) | wc -l)"; n="${n//[!0-9]/}"
|
|
115
|
+
[ "${n:-0}" -gt "$_PV_MAX" ] && n="more than $_PV_MAX"
|
|
116
|
+
out="${out:+$out; }$w: ${n:-0} file(s)"
|
|
117
|
+
done
|
|
118
|
+
[ -n "$out" ] && _CREW_PV="Preview: $out."
|
|
119
|
+
;;
|
|
120
|
+
commit)
|
|
121
|
+
case " $cmd " in *' -a'*|*' --all'*|*' -[A-Za-z]*a'*) line="$(git diff HEAD --shortstat 2>/dev/null)" ;; *) line="$(git diff --cached --shortstat 2>/dev/null)" ;; esac
|
|
122
|
+
line="${line# }"
|
|
123
|
+
_CREW_PV="Preview: ${line:-nothing staged}."
|
|
124
|
+
;;
|
|
125
|
+
push-ask)
|
|
126
|
+
local subj="" k=0 rem="" ref="" src="" tip=HEAD
|
|
127
|
+
# A push that names its remote and branch (git push origin feat/x) sends to that branch, not the upstream; and
|
|
128
|
+
# what it sends is the named LOCAL branch's commits (src of src:dst, or the name itself), not the current one's.
|
|
129
|
+
_pv_words "$cmd" push
|
|
130
|
+
for w in $_PV_W; do case "$w" in -*) ;; *) if [ -z "$rem" ]; then rem="$w"; elif [ -z "$ref" ]; then ref="$w"; fi ;; esac; done
|
|
131
|
+
ref="${ref#+}"; src="${ref%%:*}"; ref="${ref##*:}"
|
|
132
|
+
if [ -n "$src" ] && [ "$src" != HEAD ]; then
|
|
133
|
+
if [ -f "${_CREW_GD:-.git}/refs/heads/$src" ] || git rev-parse -q --verify "refs/heads/$src" >/dev/null 2>&1; then tip="refs/heads/$src"; fi
|
|
134
|
+
fi
|
|
135
|
+
if [ -n "$rem" ] && [ -n "$ref" ] && [ "$ref" != HEAD ]; then
|
|
136
|
+
if [ -f "${_CREW_GD:-.git}/refs/remotes/$rem/$ref" ] || git rev-parse -q --verify "refs/remotes/$rem/$ref" >/dev/null 2>&1; then _PV_UP="$rem/$ref"
|
|
137
|
+
else _CREW_PV="Preview: $rem/$ref is not known here; the push would create it on the remote."; return 0; fi
|
|
138
|
+
else _pv_upstream; fi
|
|
139
|
+
if [ -n "$_PV_UP" ]; then
|
|
140
|
+
# One process: the subjects of what it sends, counted here and the first three named.
|
|
141
|
+
while IFS= read -r line; do k=$((k+1)); [ "$k" -le 3 ] && subj="${subj:+$subj; }${line:0:72}"; done < <(git log --format=%s "$_PV_UP..$tip" 2>/dev/null)
|
|
142
|
+
[ "$k" -gt 3 ] && subj="$subj …"
|
|
143
|
+
_CREW_PV="Preview: it sends $k commit(s) to $_PV_UP${subj:+ — $subj}."
|
|
144
|
+
else
|
|
145
|
+
_CREW_PV="Preview: the branch has no upstream set here; what it sends is not counted."
|
|
146
|
+
fi
|
|
147
|
+
;;
|
|
148
|
+
esac
|
|
149
|
+
return 0
|
|
150
|
+
}
|
|
@@ -0,0 +1,273 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// bench.mjs — how right the review is, in numbers: run labelled cases, match what was found to where the defect
|
|
3
|
+
// is, and say the precision, the recall and every false alarm by name.
|
|
4
|
+
//
|
|
5
|
+
// node bench.mjs --cases DIR [--arm prepass|agent] [--split all|train|holdout] [--only id,id]
|
|
6
|
+
// [--model sonnet] [--advisor opus] [--case-budget 0.75] [--max-cost 12] [--floors FILE] [--out FILE]
|
|
7
|
+
// [--no-tools]
|
|
8
|
+
// --no-tools the pre-pass runs without shellcheck, so its numbers do not depend on what the machine has
|
|
9
|
+
// installed (the gate in CI uses it; shellcheck's own precision is not part of the floors)
|
|
10
|
+
// rc 0 = ran (and, with --floors, at or above every floor) · 1 = below a floor, or the agent arm could not run a
|
|
11
|
+
// case · 2 = usage
|
|
12
|
+
//
|
|
13
|
+
// A CASE is a directory: case.json, before/ and after/ (the touched files only, with their modes in case.json).
|
|
14
|
+
// It is built as a git repository with `before` committed and `after` staged, so a reviewer meets it the way it
|
|
15
|
+
// meets real work: `git diff --cached`. case.json says what the change claims to do (`intent`, the only thing the
|
|
16
|
+
// reviewer is told) and where the defect is (`expect`: file and line range in the after side; a null range is a
|
|
17
|
+
// file-level defect, a mode bit). A clean case expects nothing.
|
|
18
|
+
//
|
|
19
|
+
// SCORING. A finding matches an expected location when it cites the same file within three lines of the range
|
|
20
|
+
// (any line, for a file-level defect). A defect case is CAUGHT when any finding matches any of its locations:
|
|
21
|
+
// one defect is one defect, however many hunks its fix took. Recall = caught / defect cases. Precision =
|
|
22
|
+
// matching findings / all findings of severity low or above (info is a note, not a claim). Every finding that
|
|
23
|
+
// matches nothing is listed as a false alarm with its case, line and title.
|
|
24
|
+
//
|
|
25
|
+
// A defect case made by undoing a fix can show the fix's own explanation among the removed lines. That is what a
|
|
26
|
+
// reviewer of a real revert would see too, but it is a hint, so such cases are marked and recall is reported for
|
|
27
|
+
// both halves.
|
|
28
|
+
//
|
|
29
|
+
// ARMS. prepass: kit/eval/lib/review-prepass, in process, no model. agent: Claude Code headless, the session run
|
|
30
|
+
// AS crew-review-agent (--agent) with Crewforth's review agent, skills and eval/lib copied into the case's .claude/,
|
|
31
|
+
// on --model, each case capped by --case-budget and the run by --max-cost (it stops before a case that could pass
|
|
32
|
+
// it). The agent arm scores what it wrote to .crewforth/review.json together with the pre-pass it was told to carry
|
|
33
|
+
// over; when it wrote no valid file, its final answer's `file:line · severity` lines are read instead, and the
|
|
34
|
+
// result says so. It runs one review pass: the refuter is not installed in the case, so nothing is --verified.
|
|
35
|
+
//
|
|
36
|
+
// --advisor MODEL runs the agent arm with that advisor (the `advisorModel` setting, passed with --settings; the user's
|
|
37
|
+
// own settings are not read). What a run cost is the result's own total, which counts the advisor's calls; each row
|
|
38
|
+
// also carries the cost and tokens by model (the result's modelUsage) and how many advisor calls were made.
|
|
39
|
+
//
|
|
40
|
+
// A TASK is a case of `kind: "task"`: an implementation, not a review. before/ is the repository, `intent` is what
|
|
41
|
+
// the session is asked to do, check/ is copied in only after the session has ended and `check` (argv, run in the
|
|
42
|
+
// repository) grades it: rc 0 is done. The session runs as itself, not as the review agent.
|
|
43
|
+
import fs from 'node:fs';
|
|
44
|
+
import os from 'node:os';
|
|
45
|
+
import path from 'node:path';
|
|
46
|
+
import { spawnSync } from 'node:child_process';
|
|
47
|
+
import { fileURLToPath } from 'node:url';
|
|
48
|
+
import { prepass } from '../review-prepass/prepass.mjs';
|
|
49
|
+
import { validate } from '../findings/findings.mjs';
|
|
50
|
+
|
|
51
|
+
const HERE = path.dirname(fileURLToPath(import.meta.url));
|
|
52
|
+
const KIT = path.resolve(HERE, '..', '..', '..'); // kit/ (source) or .claude/ (installed)
|
|
53
|
+
const NEAR = 3;
|
|
54
|
+
const RANK = { info: 0, low: 1, medium: 2, high: 3, critical: 4 };
|
|
55
|
+
const TOKEN = `ghp_${'R3v1'.repeat(9)}`;
|
|
56
|
+
|
|
57
|
+
function args(argv) {
|
|
58
|
+
const o = { arm: 'prepass', split: 'all', model: 'sonnet', advisor: null, caseBudget: 0.75, maxCost: 12 };
|
|
59
|
+
for (let i = 0; i < argv.length; i++) {
|
|
60
|
+
const a = argv[i]; const v = () => { const x = argv[++i]; if (x === undefined) throw new Error(`${a} needs a value`); return x; };
|
|
61
|
+
if (a === '--cases') o.cases = v(); else if (a === '--arm') o.arm = v(); else if (a === '--split') o.split = v();
|
|
62
|
+
else if (a === '--only') o.only = v().split(','); else if (a === '--model') o.model = v(); else if (a === '--advisor') o.advisor = v();
|
|
63
|
+
else if (a === '--case-budget') o.caseBudget = Number(v()); else if (a === '--max-cost') o.maxCost = Number(v());
|
|
64
|
+
else if (a === '--floors') o.floors = v(); else if (a === '--out') o.out = v(); else if (a === '--no-tools') o.noTools = true;
|
|
65
|
+
else throw new Error(`unknown argument ${a}`);
|
|
66
|
+
}
|
|
67
|
+
if (!o.cases || !['prepass', 'agent'].includes(o.arm) || !['all', 'train', 'holdout'].includes(o.split)) throw new Error('--cases DIR is required; --arm prepass|agent; --split all|train|holdout');
|
|
68
|
+
return o;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export function loadCases(dir, { split = 'all', only = null } = {}) {
|
|
72
|
+
return fs.readdirSync(dir).sort().filter((d) => fs.existsSync(path.join(dir, d, 'case.json')))
|
|
73
|
+
.map((d) => ({ ...JSON.parse(fs.readFileSync(path.join(dir, d, 'case.json'), 'utf8')), dir: path.join(dir, d) }))
|
|
74
|
+
.filter((c) => (split === 'all' || (split === 'holdout') === !!c.holdout) && (!only || only.includes(c.id)));
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
const ENV = (tmp) => ({ ...process.env, GIT_CONFIG_GLOBAL: path.join(tmp, 'gitconfig'), GIT_CONFIG_NOSYSTEM: '1' });
|
|
78
|
+
function git(root, env, ...a) { const r = spawnSync('git', ['-C', root, ...a], { encoding: 'utf8', env }); if (r.status !== 0) throw new Error(`git ${a.join(' ')}: ${r.stderr}`); return r.stdout; }
|
|
79
|
+
// A folder a build writes (bin, dist, …) is stored with a leading '=' inside a case, so a repository's own commit
|
|
80
|
+
// gates do not read a case's past source file as a build artifact; the case is built under the real name.
|
|
81
|
+
const BUILD_DIRS = new Set(['node_modules', 'bower_components', 'dist', 'build', 'out', 'target', 'bin', 'obj', 'vendor', 'coverage']);
|
|
82
|
+
export const storedPath = (p) => p.split('/').map((s, i, a) => (i < a.length - 1 && BUILD_DIRS.has(s) ? `=${s}` : s)).join('/');
|
|
83
|
+
export const realPath = (p) => p.split('/').map((s) => (s.startsWith('=') && BUILD_DIRS.has(s.slice(1)) ? s.slice(1) : s)).join('/');
|
|
84
|
+
function tree(dir) { const out = []; const walk = (d) => { if (!fs.existsSync(d)) return; for (const e of fs.readdirSync(d, { withFileTypes: true })) { const p = path.join(d, e.name); if (e.isDirectory()) walk(p); else out.push(path.relative(dir, p).split(path.sep).join('/')); } }; walk(dir); return out.sort(); }
|
|
85
|
+
|
|
86
|
+
// The case as a repository: before committed, after staged. Returns { root, env, diff }.
|
|
87
|
+
export function build(c, tmp) {
|
|
88
|
+
const root = path.join(tmp, c.id); fs.mkdirSync(root, { recursive: true });
|
|
89
|
+
fs.writeFileSync(path.join(tmp, 'gitconfig'), '[user]\n\tname = bench\n\temail = bench@bench.invalid\n[core]\n\tautocrlf = false\n\tfileMode = false\n[init]\n\tdefaultBranch = main\n');
|
|
90
|
+
const env = ENV(tmp);
|
|
91
|
+
const put = (side) => {
|
|
92
|
+
for (const s of tree(path.join(c.dir, side))) {
|
|
93
|
+
const f = realPath(s);
|
|
94
|
+
const out = path.join(root, f); fs.mkdirSync(path.dirname(out), { recursive: true });
|
|
95
|
+
fs.writeFileSync(out, fs.readFileSync(path.join(c.dir, side, s), 'utf8').replaceAll('{{TOKEN}}', TOKEN));
|
|
96
|
+
fs.chmodSync(out, c.modes?.[side]?.[f] === '100755' ? 0o755 : 0o644);
|
|
97
|
+
}
|
|
98
|
+
};
|
|
99
|
+
// A file's mode is written into the index, not read from the disk: Windows has no executable bit, so a case whose
|
|
100
|
+
// whole diff is a mode (mode-bit) built as no diff at all there. `git init` probes the disk and writes its own
|
|
101
|
+
// core.filemode into the repository (true on macOS and Linux, false on Windows), so it is set after init, to false,
|
|
102
|
+
// and every platform takes this one path.
|
|
103
|
+
const modes = (side) => { for (const [f, m] of Object.entries(c.modes?.[side] ?? {})) git(root, env, 'update-index', `--chmod=${m === '100755' ? '+x' : '-x'}`, '--', f); };
|
|
104
|
+
git(root, env, 'init', '-q', '.'); git(root, env, 'config', 'core.fileMode', 'false');
|
|
105
|
+
put('before'); git(root, env, 'add', '-A'); modes('before'); git(root, env, 'commit', '-qm', 'base', '--allow-empty');
|
|
106
|
+
if (c.kind !== 'task') {
|
|
107
|
+
for (const s of tree(path.join(c.dir, 'before'))) if (!fs.existsSync(path.join(c.dir, 'after', s))) fs.rmSync(path.join(root, realPath(s)));
|
|
108
|
+
put('after'); git(root, env, 'add', '-A'); modes('after');
|
|
109
|
+
}
|
|
110
|
+
fs.appendFileSync(path.join(root, '.git', 'info', 'exclude'), '.claude/\n.crewforth/\n');
|
|
111
|
+
const diff = git(root, env, 'diff', '--cached', '--no-ext-diff', '-U3');
|
|
112
|
+
return { root, env, diff };
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
// Whether the removed lines of the diff carry a comment: the fix's own explanation, visible to the reviewer.
|
|
116
|
+
export const hintInDiff = (diff) => diff.split('\n').some((l) => /^-(?!--)\s*(#|\/\/|\/\*|\*)\s*\S/.test(l) && !/^-\s*#!/.test(l));
|
|
117
|
+
|
|
118
|
+
export function score(c, findings) {
|
|
119
|
+
const claims = findings.filter((f) => (RANK[f.severity] ?? 1) >= RANK.low);
|
|
120
|
+
const hits = (f) => (c.expect ?? []).some((e) => e.file === f.file && (e.from == null || (f.line >= e.from - NEAR && f.line <= e.to + NEAR)));
|
|
121
|
+
const matched = claims.filter(hits);
|
|
122
|
+
return { caught: c.kind === 'defect' ? findings.some(hits) : null, claims: claims.length, matched: matched.length,
|
|
123
|
+
falseAlarms: claims.filter((f) => !hits(f)).map((f) => ({ case: c.id, file: f.file, line: f.line, severity: f.severity, title: f.title })) };
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
const flat = (doc) => (doc?.findings ?? []).map((f) => ({ file: f.evidence?.[0]?.file, line: f.evidence?.[0]?.line, severity: f.severity ?? 'low', title: f.title, rule: f.rule }));
|
|
127
|
+
// `file:line · severity · …` lines of a final answer, for a review that wrote no file.
|
|
128
|
+
export const fromText = (text) => [...String(text).matchAll(/([\w./-]+\.[\w]+):(\d+)\s*[·|-]\s*(critical|high|medium|low|info)\b[^\n]*/gi)]
|
|
129
|
+
.map((m) => ({ file: m[1].replace(/^\.\//, ''), line: Number(m[2]), severity: m[3].toLowerCase(), title: m[0].slice(0, 160) }));
|
|
130
|
+
|
|
131
|
+
function copyKit(root) {
|
|
132
|
+
const dst = path.join(root, '.claude');
|
|
133
|
+
const parts = [['agents/crew-review-agent.md'], ['skills/crew-code-review'], ['skills/docs-writer'], ['eval/lib/review-prepass'], ['eval/lib/findings'], ['eval/lib/crew-map'], ['hooks/secret-blocklist.txt']];
|
|
134
|
+
for (const [p] of parts) { const s = path.join(KIT, p); if (fs.existsSync(s)) fs.cpSync(s, path.join(dst, p), { recursive: true }); }
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
// The flags every headless run shares. The bench measures the model and Crewforth, not the machine it runs on: no
|
|
138
|
+
// user settings (an advisor model, hooks), no MCP server of the user's, no session written. Seen without them: a run
|
|
139
|
+
// took the user's advisor model, and the process stayed alive after its answer. An advisor is the arm's own choice.
|
|
140
|
+
function claudeArgs(prompt, o, tools) {
|
|
141
|
+
return ['-p', prompt, '--model', o.model, '--output-format', 'stream-json', '--verbose',
|
|
142
|
+
'--permission-mode', 'bypassPermissions', '--allowedTools', tools, '--max-budget-usd', String(o.caseBudget),
|
|
143
|
+
'--setting-sources', 'project,local', '--strict-mcp-config', '--no-session-persistence',
|
|
144
|
+
...(o.advisor ? ['--settings', JSON.stringify({ advisorModel: o.advisor })] : [])];
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
// The result event, what each model cost (modelUsage counts the advisor's calls under the advisor's model), and the
|
|
148
|
+
// advisor calls themselves. Measured on a headless run: an assistant event's usage carries no `iterations`; a call is
|
|
149
|
+
// a `server_tool_use` block named `advisor` in the message's content, so those are counted, once per block id.
|
|
150
|
+
export function readRun(stdout) {
|
|
151
|
+
let res = null; const seen = new Set();
|
|
152
|
+
for (const l of String(stdout || '').split('\n')) {
|
|
153
|
+
let e; try { e = JSON.parse(l); } catch { continue; }
|
|
154
|
+
if (e.type === 'result') res = e;
|
|
155
|
+
if (e.type === 'assistant' && Array.isArray(e.message?.content)) {
|
|
156
|
+
for (const c of e.message.content) if (c?.type === 'server_tool_use' && c.name === 'advisor' && c.id) seen.add(c.id);
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
const advisorCalls = seen.size;
|
|
160
|
+
const byModel = Object.fromEntries(Object.entries(res?.modelUsage ?? {}).map(([k, u]) => [k, { cost: Number(u.costUSD ?? 0),
|
|
161
|
+
input: u.inputTokens ?? 0, output: u.outputTokens ?? 0, cacheRead: u.cacheReadInputTokens ?? 0, cacheWrite: u.cacheCreationInputTokens ?? 0 }]));
|
|
162
|
+
return { res, byModel, advisorCalls };
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
const runFields = ({ res, byModel, advisorCalls }) => ({
|
|
166
|
+
cost: Number(res?.total_cost_usd ?? 0), turns: res?.num_turns ?? null, ms: res?.duration_ms ?? null, byModel, advisorCalls,
|
|
167
|
+
tokens: res?.usage ? { input: res.usage.input_tokens, output: res.usage.output_tokens, cacheRead: res.usage.cache_read_input_tokens, cacheWrite: res.usage.cache_creation_input_tokens } : null });
|
|
168
|
+
const runError = (res, r) => (!res ? `no result event (rc ${r.status}${r.error ? `, ${r.error.code}` : ''})` : (res.is_error ? (res.subtype || 'error') : null));
|
|
169
|
+
// The CLI. CREW_BENCH_CLAUDE names a node script that stands in for it (the selfcheck's fake, on every platform: a
|
|
170
|
+
// script called `claude` cannot be spawned on Windows without a shell).
|
|
171
|
+
const claude = (args, opts) => (process.env.CREW_BENCH_CLAUDE
|
|
172
|
+
? spawnSync(process.execPath, [process.env.CREW_BENCH_CLAUDE, ...args], opts) : spawnSync('claude', args, opts));
|
|
173
|
+
const SPAWN = (b) => ({ cwd: b.root, encoding: 'utf8', env: b.env, maxBuffer: 256 << 20, timeout: 12 * 60 * 1000, killSignal: 'SIGKILL', input: '' });
|
|
174
|
+
|
|
175
|
+
// A task: the session does the work, then the check it never saw grades it.
|
|
176
|
+
function runTask(c, b, o) {
|
|
177
|
+
const r = claude(claudeArgs(c.intent, o, 'Bash Read Grep Glob Edit Write'), SPAWN(b));
|
|
178
|
+
const run = readRun(r.stdout);
|
|
179
|
+
if (fs.existsSync(path.join(c.dir, 'check'))) fs.cpSync(path.join(c.dir, 'check'), b.root, { recursive: true });
|
|
180
|
+
const [cmd, ...rest] = c.check ?? [];
|
|
181
|
+
const k = cmd ? spawnSync(cmd, rest, { cwd: b.root, encoding: 'utf8', env: b.env, timeout: 120000 }) : null;
|
|
182
|
+
const tail = `${k?.stdout ?? ''}${k?.stderr ?? ''}`.trim().split('\n').slice(-3).join(' | ');
|
|
183
|
+
return { findings: [], source: 'check', done: !!k && k.status === 0, check: k ? `rc ${k.status}${tail ? ` · ${tail.slice(0, 200)}` : ''}` : 'no check',
|
|
184
|
+
...runFields(run), error: runError(run.res, r) };
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
function runAgent(c, b, o) {
|
|
188
|
+
copyKit(b.root);
|
|
189
|
+
const prompt = `files: the staged change (git diff --cached)\nchange: audit\nverify: none\n\n`
|
|
190
|
+
+ `Review the staged change. Its author says: "${c.intent}". Follow crew-code-review: run the pre-pass first `
|
|
191
|
+
+ `(bash .claude/eval/lib/review-prepass/prepass.sh), carry its findings over, then write your own findings to `
|
|
192
|
+
+ `.crewforth/review.json with bash .claude/eval/lib/findings/findings.sh write review <file> (the refuter is `
|
|
193
|
+
+ `not installed here, so write without --verified). Change no other file. End with your findings in the format `
|
|
194
|
+
+ `file:line · severity · category · observation · what to do instead.`;
|
|
195
|
+
const r = claude(['--agent', 'crew-review-agent', ...claudeArgs(prompt, o, 'Bash Read Grep Glob Write')], SPAWN(b));
|
|
196
|
+
const run = readRun(r.stdout); const { res } = run;
|
|
197
|
+
const read = (n) => { try { return JSON.parse(fs.readFileSync(path.join(b.root, '.crewforth', `${n}.json`), 'utf8')); } catch { return null; } };
|
|
198
|
+
const review = read('review'); const pre = read('review-prepass');
|
|
199
|
+
const reviewOk = !!review && validate(review, b.root).ok;
|
|
200
|
+
const own = reviewOk ? flat(review) : fromText(res?.result ?? '');
|
|
201
|
+
const seen = new Set(); const all = [];
|
|
202
|
+
for (const f of [...own, ...flat(pre)]) { const k = `${f.file}:${f.line}:${f.rule ?? f.title}`; if (f.file && f.line && !seen.has(k)) { seen.add(k); all.push(f); } }
|
|
203
|
+
return { findings: all, source: reviewOk ? 'review.json' : 'final answer', prepassRan: !!pre, ...runFields(run), error: runError(res, r) };
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
export function run(o) {
|
|
207
|
+
const cases = loadCases(o.cases, o);
|
|
208
|
+
const tmp = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'crew-bench-')));
|
|
209
|
+
const rows = []; let spent = 0;
|
|
210
|
+
try {
|
|
211
|
+
for (const c of cases) {
|
|
212
|
+
if (o.arm === 'agent' && spent + o.caseBudget > o.maxCost) { rows.push({ id: c.id, skipped: `the run's cap ($${o.maxCost}) would be passed` }); continue; }
|
|
213
|
+
const b = build(c, tmp);
|
|
214
|
+
let out;
|
|
215
|
+
if (c.kind === 'task') {
|
|
216
|
+
if (o.arm !== 'agent') { rows.push({ id: c.id, kind: c.kind, skipped: 'a task needs the agent arm' }); continue; }
|
|
217
|
+
out = runTask(c, b, o); spent += out.cost;
|
|
218
|
+
}
|
|
219
|
+
else if (o.arm === 'prepass') {
|
|
220
|
+
const keep = process.env.CREW_SHELLCHECK; if (o.noTools) process.env.CREW_SHELLCHECK = path.join(tmp, 'no-shellcheck');
|
|
221
|
+
try { out = { findings: flat(prepass(b.root)), source: 'prepass' }; } finally { if (o.noTools) { if (keep === undefined) delete process.env.CREW_SHELLCHECK; else process.env.CREW_SHELLCHECK = keep; } }
|
|
222
|
+
}
|
|
223
|
+
else { out = runAgent(c, b, o); spent += out.cost; }
|
|
224
|
+
rows.push({ id: c.id, kind: c.kind, stratum: c.stratum, holdout: !!c.holdout, hint: c.kind === 'defect' ? hintInDiff(b.diff) : null, ...out, ...score(c, out.findings) });
|
|
225
|
+
}
|
|
226
|
+
} finally { fs.rmSync(tmp, { recursive: true, force: true }); }
|
|
227
|
+
return summarise(rows, o, spent);
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
export function summarise(rows, o, spent = 0) {
|
|
231
|
+
const done = rows.filter((r) => !r.skipped);
|
|
232
|
+
const defects = done.filter((r) => r.kind === 'defect'); const tasks = done.filter((r) => r.kind === 'task');
|
|
233
|
+
const pct = (a, b) => (b ? Math.round((1000 * a) / b) / 10 : null);
|
|
234
|
+
const rec = (list) => ({ caught: list.filter((r) => r.caught).length, of: list.length, pct: pct(list.filter((r) => r.caught).length, list.length) });
|
|
235
|
+
const claims = done.reduce((s, r) => s + r.claims, 0); const matched = done.reduce((s, r) => s + r.matched, 0);
|
|
236
|
+
const sum = (k) => done.reduce((n, r) => n + (Number(r[k]) || 0), 0);
|
|
237
|
+
return { arm: o.arm, model: o.arm === 'agent' ? o.model : null, advisor: o.arm === 'agent' ? o.advisor ?? null : null, split: o.split, cases: rows.length, ran: done.length,
|
|
238
|
+
tasks: { done: tasks.filter((r) => r.done).length, of: tasks.length }, ms: sum('ms'), advisorCalls: sum('advisorCalls'),
|
|
239
|
+
recall: rec(defects), recallNoHint: rec(defects.filter((r) => !r.hint)), recallHint: rec(defects.filter((r) => r.hint)),
|
|
240
|
+
byStratum: Object.fromEntries(['pattern', 'judgement'].map((s) => [s, rec(defects.filter((r) => r.stratum === s))])),
|
|
241
|
+
precision: { matched, of: claims, pct: pct(matched, claims) },
|
|
242
|
+
falseAlarms: done.flatMap((r) => r.falseAlarms), cost: Math.round(spent * 100) / 100, errors: done.filter((r) => r.error).map((r) => `${r.id}: ${r.error}`), rows };
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
function main(argv) {
|
|
246
|
+
let o; try { o = args(argv); } catch (e) { console.error(`review-bench: ${e.message}`); return 2; }
|
|
247
|
+
if (o.arm === 'agent' && claude(['--version'], { encoding: 'utf8' }).status !== 0) { console.error('review-bench: the claude CLI does not run here; the agent arm needs it'); return 2; }
|
|
248
|
+
const s = run(o);
|
|
249
|
+
const line = (n, x) => `${n} ${x.caught}/${x.of}${x.pct === null ? '' : ` (${x.pct}%)`}`;
|
|
250
|
+
console.log(`review-bench · arm ${s.arm}${s.model ? ` (${s.model}${s.advisor ? ` + ${s.advisor} advisor` : ''})` : ''} · split ${s.split} · ${s.ran}/${s.cases} cases`);
|
|
251
|
+
console.log(` ${line('recall', s.recall)} · ${line('without a hint', s.recallNoHint)} · ${line('with one', s.recallHint)}`);
|
|
252
|
+
console.log(` ${line('pattern', s.byStratum.pattern)} · ${line('judgement', s.byStratum.judgement)}`);
|
|
253
|
+
console.log(` precision ${s.precision.matched}/${s.precision.of}${s.precision.pct === null ? ' (no claim made)' : ` (${s.precision.pct}%)`} · ${s.falseAlarms.length} false alarm(s)${s.arm === 'agent' ? ` · cost $${s.cost}` : ''}`);
|
|
254
|
+
if (s.tasks.of) console.log(` tasks done ${s.tasks.done}/${s.tasks.of}`);
|
|
255
|
+
if (s.arm === 'agent') console.log(` time ${Math.round(s.ms / 1000)} s · advisor calls ${s.advisorCalls}`);
|
|
256
|
+
for (const f of s.falseAlarms) console.log(` FALSE ALARM ${f.case} ${f.file}:${f.line} ${f.severity} ${String(f.title).slice(0, 100)}`);
|
|
257
|
+
for (const e of s.errors) console.log(` ERROR ${e}`);
|
|
258
|
+
for (const r of s.rows.filter((x) => x.skipped)) console.log(` NOT RUN ${r.id}: ${r.skipped}`);
|
|
259
|
+
if (o.out) fs.writeFileSync(o.out, JSON.stringify(s, null, 2) + '\n');
|
|
260
|
+
let rc = s.errors.length && o.arm === 'agent' ? 1 : 0;
|
|
261
|
+
if (o.floors) {
|
|
262
|
+
const fl = JSON.parse(fs.readFileSync(o.floors, 'utf8'))[o.arm] ?? {};
|
|
263
|
+
for (const [k, v] of Object.entries(fl)) {
|
|
264
|
+
const got = k === 'precision' ? s.precision.pct : k === 'falseAlarms' ? s.falseAlarms.length : (k === 'recall' ? s.recall : s.byStratum[k.replace(/^recall-/, '')])?.pct;
|
|
265
|
+
const ok = k === 'falseAlarms' ? got <= v : got !== null && got !== undefined && got >= v;
|
|
266
|
+
console.log(` floor ${k}: ${got} ${k === 'falseAlarms' ? '≤' : '≥'} ${v} ${ok ? 'ok' : 'BELOW THE FLOOR'}`);
|
|
267
|
+
if (!ok) rc = 1;
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
return rc;
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
if (process.argv[1] && fs.realpathSync(process.argv[1]) === fileURLToPath(import.meta.url)) process.exitCode = main(process.argv.slice(2));
|
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// prepass.mjs — the review's deterministic first pass: cheap checks over the lines a change ADDS, run before the
|
|
3
|
+
// review agent reads the diff, written as a findings document (crewforth.findings/1) the agent starts from.
|
|
4
|
+
//
|
|
5
|
+
// node prepass.mjs [--root DIR] [--base REF] [--stdout]
|
|
6
|
+
// default the staged change (git diff --cached), what the next commit will hold
|
|
7
|
+
// --base the working tree against REF (a branch under review: --base origin/next)
|
|
8
|
+
// --stdout print the document, write nothing; otherwise .crewforth/review-prepass.json, through findings.mjs
|
|
9
|
+
// rc 0 = written (or printed), whatever it found · 1 = the document did not verify, nothing written ·
|
|
10
|
+
// 2 = usage, or not a git repository
|
|
11
|
+
//
|
|
12
|
+
// The checks are chosen by the file, and kept narrow on purpose: a first pass that cries wolf trains the reader to
|
|
13
|
+
// skip it. Each one either reads the added line alone (proven) or one fact next to it (inferred):
|
|
14
|
+
//
|
|
15
|
+
// prepass-secret a secret-blocklist.txt pattern on an added line (the pre-commit gate's own list and
|
|
16
|
+
// .secret-allowlist.txt). Cited by sha256 only: the value is never quoted.
|
|
17
|
+
// prepass-debug-call console.log / debugger (JS, TS), breakpoint() / pdb (Python), print( (Dart), outside a
|
|
18
|
+
// test and outside a command-line script (a shebang, or a bin/, scripts/ or tool/ folder).
|
|
19
|
+
// prepass-todo a TODO, FIXME, XXX or HACK opening a comment added in code, outside tests
|
|
20
|
+
// (Markdown is prose, not a promise; the word in a string or a pattern is not a marker).
|
|
21
|
+
// prepass-shellcheck shellcheck's own findings on a shell file, kept to the added lines. Without shellcheck
|
|
22
|
+
// the shell files are listed as not measured: nothing is installed.
|
|
23
|
+
// prepass-lockfile a lock file changed and the manifest beside it did not (inferred: a regenerated lock
|
|
24
|
+
// can be on purpose).
|
|
25
|
+
// prepass-large-change one file gains more than 500 lines (generated files, locks and snapshots aside).
|
|
26
|
+
//
|
|
27
|
+
// What it does not do: judge. A finding here says a line is there; whether it matters is the reviewer's call, and
|
|
28
|
+
// the reviewer writes its own findings beside these (.crewforth/review.json), in the same shape.
|
|
29
|
+
// Host-independent like findings.mjs: git, Node's standard library, and shellcheck when it is there.
|
|
30
|
+
|
|
31
|
+
import fs from 'node:fs';
|
|
32
|
+
import os from 'node:os';
|
|
33
|
+
import path from 'node:path';
|
|
34
|
+
import crypto from 'node:crypto';
|
|
35
|
+
import { spawnSync } from 'node:child_process';
|
|
36
|
+
import { fileURLToPath } from 'node:url';
|
|
37
|
+
import { loadSecretPatterns, isTestPath } from '../crew-map/project.mjs';
|
|
38
|
+
import { SCHEMA_ID, validate, write, serialize } from '../findings/findings.mjs';
|
|
39
|
+
|
|
40
|
+
const HERE = path.dirname(fileURLToPath(import.meta.url));
|
|
41
|
+
export const NAME = 'review-prepass';
|
|
42
|
+
const LARGE = 500;
|
|
43
|
+
const QUOTE = 200;
|
|
44
|
+
|
|
45
|
+
const DEBUG = [
|
|
46
|
+
{ files: /\.(m?[jt]sx?|cjs|cts|vue|svelte)$/, re: /\bconsole\.(log|debug|trace)\(|^\s*debugger\s*;?\s*$/, what: 'a console call or debugger statement' },
|
|
47
|
+
{ files: /\.py$/, re: /\bbreakpoint\(\)|\bpdb\.set_trace\(|^\s*import\s+pdb\b/, what: 'a debugger breakpoint' },
|
|
48
|
+
{ files: /\.dart$/, re: /^\s*print\(/, what: 'a print call' },
|
|
49
|
+
];
|
|
50
|
+
const SCRIPT_DIR = /(^|\/)(bin|scripts?|tools?|tool)\//;
|
|
51
|
+
// A marker opens a comment; the word inside a string, a pattern or prose that names it is not one.
|
|
52
|
+
const TODO = /(?:\/\/|#|\/\*|^\s*\*|--|<!--)\s*(TODO|FIXME|XXX|HACK)\b/;
|
|
53
|
+
const PROSE = /\.(md|markdown|txt|rst|adoc)$/i;
|
|
54
|
+
const SHELL = /\.(sh|bash)$/;
|
|
55
|
+
const SECRET_LISTS = /(^|\/)secret-(block|allow)list\.txt$/;
|
|
56
|
+
// A lock file and the manifest that declares what it pins, in the same folder.
|
|
57
|
+
const LOCKS = {
|
|
58
|
+
'package-lock.json': 'package.json', 'npm-shrinkwrap.json': 'package.json', 'yarn.lock': 'package.json',
|
|
59
|
+
'pnpm-lock.yaml': 'package.json', 'bun.lockb': 'package.json', 'pubspec.lock': 'pubspec.yaml',
|
|
60
|
+
'poetry.lock': 'pyproject.toml', 'uv.lock': 'pyproject.toml', 'Pipfile.lock': 'Pipfile', 'go.sum': 'go.mod',
|
|
61
|
+
'Cargo.lock': 'Cargo.toml', 'composer.lock': 'composer.json', 'Gemfile.lock': 'Gemfile',
|
|
62
|
+
};
|
|
63
|
+
const GENERATED = /(^|\/)(dist|build|vendor|node_modules|__snapshots__)\/|\.(g|freezed|gr|pb)\.dart$|\.min\.(js|css)$|\.snap$|\.lock$|-lock\.(json|yaml)$/;
|
|
64
|
+
|
|
65
|
+
const sha = (s) => crypto.createHash('sha256').update(s).digest('hex');
|
|
66
|
+
const git = (root, args) => spawnSync('git', ['-C', root, '-c', 'core.quotePath=false', ...args], { encoding: 'utf8', maxBuffer: 256 * 1024 * 1024 });
|
|
67
|
+
|
|
68
|
+
// The added lines of a unified diff with no context: { file, line, text }[] and the files it touches.
|
|
69
|
+
export function addedLines(diff) {
|
|
70
|
+
const added = []; const files = new Set(); const binary = new Set();
|
|
71
|
+
let file = null; let n = 0;
|
|
72
|
+
for (const raw of diff.split('\n')) {
|
|
73
|
+
if (raw.startsWith('diff --git ')) { file = null; continue; }
|
|
74
|
+
if (raw.startsWith('+++ ')) { file = raw === '+++ /dev/null' ? null : raw.slice(6).replace(/\t$/, ''); if (file) files.add(file); continue; }
|
|
75
|
+
if (raw.startsWith('Binary files ')) { const b = / and b\/(.*) differ$/.exec(raw); if (b) binary.add(b[1]); continue; }
|
|
76
|
+
const h = /^@@ -\d+(?:,\d+)? \+(\d+)(?:,\d+)? @@/.exec(raw);
|
|
77
|
+
if (h) { n = Number(h[1]); continue; }
|
|
78
|
+
if (file && raw.startsWith('+')) { added.push({ file, line: n, text: raw.slice(1).replace(/\r$/, '') }); n++; }
|
|
79
|
+
}
|
|
80
|
+
return { added, files: [...files].sort(), binary: [...binary].sort() };
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
// A finding id from a rule, a file and a line: lower case, the schema's characters only, unique in the document.
|
|
84
|
+
function idOf(seen, rule, file, line) {
|
|
85
|
+
let id = `${rule}:${file.toLowerCase().replace(/[^a-z0-9._-]+/g, '-')}:${line}`.slice(0, 190);
|
|
86
|
+
if (seen.has(id)) { let k = 2; while (seen.has(`${id}.${k}`)) k++; id = `${id}.${k}`; }
|
|
87
|
+
seen.add(id); return id;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
export function prepass(root = '.', { base = null } = {}) {
|
|
91
|
+
const rootReal = fs.realpathSync(root);
|
|
92
|
+
const top = git(rootReal, ['rev-parse', '--show-toplevel']);
|
|
93
|
+
if (top.status !== 0) throw new Error('not a git repository');
|
|
94
|
+
const diffArgs = ['diff', '--no-ext-diff', '--no-textconv', '--no-color', '-U0', '--diff-filter=AMR'];
|
|
95
|
+
const d = git(rootReal, base ? [...diffArgs, base, '--'] : [...diffArgs, '--cached', '--']);
|
|
96
|
+
if (d.status !== 0) throw new Error(`git diff failed: ${(d.stderr || '').trim().split('\n')[0]}`);
|
|
97
|
+
const { added, files, binary } = addedLines(d.stdout);
|
|
98
|
+
|
|
99
|
+
const doc = { schema: SCHEMA_ID, producer: { name: NAME, version: '1' }, findings: [], notMeasured: [] };
|
|
100
|
+
const seen = new Set();
|
|
101
|
+
const machine = [rootReal, os.homedir()].filter((p) => p && p.length > 1);
|
|
102
|
+
const add = (rule, severity, confidence, title, at, detail, quote = true) => {
|
|
103
|
+
// A line that carries this machine's directory, or must not be quoted, is cited by its hash.
|
|
104
|
+
const ev = { file: at.file, line: at.line };
|
|
105
|
+
if (quote && !machine.some((m) => at.text.includes(m)) && at.text.trim()) ev.text = at.text.trim().slice(0, QUOTE);
|
|
106
|
+
else ev.sha256 = sha(at.text);
|
|
107
|
+
doc.findings.push({ id: idOf(seen, rule, at.file, at.line), rule, title, confidence, severity, ...(detail ? { detail } : {}), evidence: [ev] });
|
|
108
|
+
};
|
|
109
|
+
|
|
110
|
+
// A staged line the working tree no longer has cannot be cited (the citation is checked against the tree).
|
|
111
|
+
const tree = new Map();
|
|
112
|
+
const linesOf = (f) => { if (!tree.has(f)) { let t = null; try { t = fs.readFileSync(path.join(rootReal, f), 'utf8').split('\n').map((l) => l.replace(/\r$/, '')); } catch { /* gone */ } tree.set(f, t); } return tree.get(f); };
|
|
113
|
+
const stale = new Set();
|
|
114
|
+
const lines = added.filter((a) => { const t = linesOf(a.file); if (t && t[a.line - 1] === a.text) return true; stale.add(a.file); return false; });
|
|
115
|
+
for (const f of [...stale].sort()) doc.notMeasured.push({ what: f, reason: 'edited again after it was staged: its staged lines are not the ones on disk, so they were not read' });
|
|
116
|
+
for (const f of binary) doc.notMeasured.push({ what: f, reason: 'a binary file: no line to read' });
|
|
117
|
+
|
|
118
|
+
// ---- secrets: the pre-commit gate's list, minus the project's allowlist ----
|
|
119
|
+
let allow = new Set();
|
|
120
|
+
try { allow = new Set(fs.readFileSync(path.join(rootReal, '.secret-allowlist.txt'), 'utf8').split('\n').map((l) => l.replace(/\r$/, '')).filter(Boolean)); } catch { /* none */ }
|
|
121
|
+
const patterns = loadSecretPatterns(path.join(HERE, '..', 'crew-map'), allow);
|
|
122
|
+
if (!patterns) doc.notMeasured.push({ what: 'prepass-secret', reason: 'secret-blocklist.txt was not found beside the hooks, so no line was checked for a secret' });
|
|
123
|
+
const byFile = new Map();
|
|
124
|
+
for (const a of lines) { if (!byFile.has(a.file)) byFile.set(a.file, []); byFile.get(a.file).push(a); }
|
|
125
|
+
|
|
126
|
+
for (const [f, list] of [...byFile].sort((x, y) => (x[0] < y[0] ? -1 : 1))) {
|
|
127
|
+
const test = isTestPath(f);
|
|
128
|
+
const first = linesOf(f)?.[0] ?? '';
|
|
129
|
+
const script = first.startsWith('#!') || SCRIPT_DIR.test(f);
|
|
130
|
+
const debug = DEBUG.find((x) => x.files.test(f));
|
|
131
|
+
for (const a of list) {
|
|
132
|
+
if (patterns && !SECRET_LISTS.test(f) && patterns.some((re) => re.test(a.text))) {
|
|
133
|
+
add('prepass-secret', 'high', 'proven', 'A secret pattern on an added line', a, 'It matches a pattern of secret-blocklist.txt, the list the pre-commit gate refuses; the value is not quoted here.', false);
|
|
134
|
+
continue;
|
|
135
|
+
}
|
|
136
|
+
if (debug && !test && !script && debug.re.test(a.text)) add('prepass-debug-call', 'low', 'proven', `An added ${debug.what}`, a);
|
|
137
|
+
if (!PROSE.test(f) && !test && TODO.test(a.text)) add('prepass-todo', 'info', 'proven', `An added ${TODO.exec(a.text)[1]}`, a);
|
|
138
|
+
}
|
|
139
|
+
if (list.length > LARGE && !GENERATED.test(f)) add('prepass-large-change', 'low', 'proven', `${list.length} lines added to one file`, list[0], 'A change this size is hard to review in one pass; split it, or say in the description what to read first.');
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
// ---- lock files: the manifest beside them should have changed too ----
|
|
143
|
+
const changed = new Set(files);
|
|
144
|
+
for (const f of files) {
|
|
145
|
+
const want = LOCKS[path.posix.basename(f)]; if (!want) continue;
|
|
146
|
+
const dir = path.posix.dirname(f); const manifest = dir === '.' ? want : `${dir}/${want}`;
|
|
147
|
+
if (changed.has(manifest) || !fs.existsSync(path.join(rootReal, manifest))) continue;
|
|
148
|
+
const at = byFile.get(f)?.[0]; if (!at) continue;
|
|
149
|
+
add('prepass-lockfile', 'low', 'inferred', `${path.posix.basename(f)} changed and ${want} did not`, at, 'A lock file that moves on its own pins different versions than the manifest asks for; regenerate it from the manifest, or say why it moved.');
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
// ---- shell: shellcheck's findings on the added lines ----
|
|
153
|
+
const shells = [...byFile.keys()].filter((f) => SHELL.test(f) || /^#!.*\b(ba|z|da)?sh\b/.test(linesOf(f)?.[0] ?? '')).sort();
|
|
154
|
+
if (shells.length) {
|
|
155
|
+
// CREW_SHELLCHECK names another shellcheck. One ending in .mjs is run with this Node: the self-check's stand-in,
|
|
156
|
+
// which a Windows runner could not start as a `#!/bin/sh` script (spawnSync runs no interpreter line there).
|
|
157
|
+
const sc = process.env.CREW_SHELLCHECK || 'shellcheck';
|
|
158
|
+
const run = (args, opts) => (sc.endsWith('.mjs') ? spawnSync(process.execPath, [sc, ...args], opts) : spawnSync(sc, args, opts));
|
|
159
|
+
const v = run(['--version'], { encoding: 'utf8' });
|
|
160
|
+
if (v.status !== 0) doc.notMeasured.push({ what: `prepass-shellcheck (${shells.length} shell file${shells.length === 1 ? '' : 's'})`, reason: 'shellcheck is not installed here, so the shell files were not linted; nothing was installed' });
|
|
161
|
+
else {
|
|
162
|
+
for (const f of shells) {
|
|
163
|
+
const r = run(['-f', 'json1', '-x', f], { cwd: rootReal, encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 });
|
|
164
|
+
let out; try { out = JSON.parse(r.stdout || '{}'); } catch { doc.notMeasured.push({ what: f, reason: 'shellcheck gave no output this pass can read' }); continue; }
|
|
165
|
+
const at = new Map(byFile.get(f).map((a) => [a.line, a]));
|
|
166
|
+
for (const c of out.comments ?? []) {
|
|
167
|
+
const a = at.get(c.line); if (!a) continue;
|
|
168
|
+
const sev = { error: 'medium', warning: 'low' }[c.level] ?? 'info';
|
|
169
|
+
add('prepass-shellcheck', sev, 'proven', `SC${c.code}: ${String(c.message).slice(0, 180)}`, a);
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
doc.findings.sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
|
|
175
|
+
return doc;
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
function main(argv) {
|
|
179
|
+
let root = '.'; let base = null; let stdout = false;
|
|
180
|
+
for (let i = 0; i < argv.length; i++) {
|
|
181
|
+
const a = argv[i];
|
|
182
|
+
if (a === '--root') root = argv[++i];
|
|
183
|
+
else if (a === '--base') base = argv[++i];
|
|
184
|
+
else if (a === '--stdout') stdout = true;
|
|
185
|
+
else { console.error(`prepass: unknown argument ${a}\nusage: prepass.mjs [--root DIR] [--base REF] [--stdout]`); return 2; }
|
|
186
|
+
if (root === undefined || base === undefined) { console.error('prepass: --root and --base need a value'); return 2; }
|
|
187
|
+
}
|
|
188
|
+
let doc;
|
|
189
|
+
try { doc = prepass(root, { base }); } catch (e) { console.error(`prepass: ${e.message}`); return 2; }
|
|
190
|
+
const count = (s) => doc.findings.filter((f) => f.severity === s).length;
|
|
191
|
+
const line = `review-prepass: ${doc.findings.length} finding(s) · high ${count('high')} · medium ${count('medium')} · low ${count('low')} · info ${count('info')} · ${doc.notMeasured.length} not measured`;
|
|
192
|
+
if (stdout) {
|
|
193
|
+
const r = validate(doc, root);
|
|
194
|
+
if (!r.ok) { for (const p of r.problems) console.error(`prepass: ${p.at} ${p.code}: ${p.message}`); return 1; }
|
|
195
|
+
process.stdout.write(serialize(doc)); console.error(line); return 0;
|
|
196
|
+
}
|
|
197
|
+
const r = write(doc, root, NAME);
|
|
198
|
+
if (!r.ok) { for (const p of r.problems) console.error(`prepass: ${p.at} ${p.code}: ${p.message}`); return 1; }
|
|
199
|
+
console.log(`${line} → ${r.file}`);
|
|
200
|
+
return 0;
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
if (process.argv[1] && fs.realpathSync(process.argv[1]) === fileURLToPath(import.meta.url)) process.exitCode = main(process.argv.slice(2));
|