@biffo/cli 0.315.6 → 0.315.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -62,6 +62,50 @@
62
62
  # or the copies stop being interchangeable and `shared-sync.sh --check` starts
63
63
  # reporting drift that is really divergence.
64
64
  #
65
+ # ## Pre-existing vs introduced (#2040)
66
+ #
67
+ # Every check above answers "can this audit be trusted" (network flake vs a
68
+ # real finding). None of them answer WHEN the vulnerable version got there —
69
+ # so a genuine high/critical advisory published overnight against a package
70
+ # already sitting on `dev`, untouched by this diff, used to read identically
71
+ # to a PR that actually introduced the vulnerable version. #1880 (2026-09-04)
72
+ # tried to paper over exactly that by moving this whole check to a
73
+ # `continue-on-error`, non-required job — which stopped a real per-PR finding
74
+ # from blocking too, and produced ten separate biffo-fleet tickets and $86.72
75
+ # of dispatch spend for two routine advisories that no PR under test could
76
+ # have fixed (#2040's own cost accounting). `py-dependency-audit.sh` closed
77
+ # this same gap for Python in #1673; this is the JS side of the same fix.
78
+ #
79
+ # The fix compares the ADVISORY IDS found against the SAME tree's pnpm audit
80
+ # run against the PR's BASE branch, not against the diff itself: a PR can
81
+ # leave pnpm-lock.yaml untouched while a sibling change moves a transitive
82
+ # version, or the base branch may already carry the flagged version. An
83
+ # advisory ID present in both base and head is pre-existing and does not
84
+ # block; an advisory ID present only in head is introduced by this diff (a
85
+ # new dependency, or an upgrade/downgrade into a vulnerable version) and
86
+ # blocks exactly as before. This mirrors the issue's own "run against base
87
+ # and head, diff the advisory IDs, fail on head-only" design rather than
88
+ # hand-parsing pnpm-lock.yaml's YAML — `pnpm audit` already works against a
89
+ # bare copy of just the lockfile with no install (verified directly against
90
+ # this repo's own pnpm 9.15.9), so the base side is a second real audit
91
+ # call, not a reimplementation of what `pnpm audit` already does.
92
+ #
93
+ # `GITHUB_BASE_REF` is the base branch name and is set ONLY for a
94
+ # `pull_request`/`pull_request_target` event — empty on `push`,
95
+ # `workflow_dispatch` and `merge_group`. That is deliberate, not a gap:
96
+ # outside a PR there is no "diff" to attribute a finding to, so every finding
97
+ # blocks exactly as before (the correct behaviour for `dev` itself, and for
98
+ # the scheduled base-branch scan added alongside this file in #2040, which
99
+ # wants every current finding reported, not just new ones).
100
+ #
101
+ # `origin/$GITHUB_BASE_REF` must already be a resolvable local ref for the
102
+ # comparison to run at all — the `js` job's checkout uses `fetch-depth: 0`
103
+ # precisely so it is (see that job's own comment). If it is NOT resolvable
104
+ # (a shallow checkout, or a distributed copy of this script running
105
+ # somewhere that checkout is missing), comparison fails CLOSED: every
106
+ # finding blocks, same as before this fix, rather than silently waving a
107
+ # real regression through because the check could not be performed.
108
+ #
65
109
  # POSIX sh (the CI step runs `sh scripts/...`, i.e. dash) — no `pipefail`.
66
110
  set -u
67
111
 
@@ -118,12 +162,60 @@ failed=0
118
162
  # invocation has finished.
119
163
  #
120
164
  # `$1` is the directory, `$2` a human label, `$3` extra pnpm flags, `$4` the
121
- # result file this invocation must write its verdict to.
165
+ # result file this invocation must write its verdict to, `$5` this tree's
166
+ # pnpm-lock.yaml path relative to the repo root — used to look up the SAME
167
+ # lockfile's advisories on the base branch (#2040) — never empty, every
168
+ # caller passes one.
169
+ #
170
+ # Returns (on stdout) the set of advisory IDs recorded against the SAME
171
+ # lockfile path on the PR's base branch, one per line — any severity, not
172
+ # just high/critical, because presence is all classification needs. Prints
173
+ # nothing and returns non-zero when a comparison genuinely cannot be made
174
+ # (network/parse failure, a temp-dir failure) — the caller then treats every
175
+ # finding as introduced, the same fail-closed default #1673 established for
176
+ # Python. An empty base lockfile (the path did not exist on the base branch
177
+ # at all — a brand-new lockfile or a brand-new vendored tree this diff
178
+ # itself introduced) is NOT a failure: it prints nothing and returns 0,
179
+ # because an empty set is the correct answer — there is nothing to be
180
+ # pre-existing against, so every finding in a tree the base never had is
181
+ # introduced by definition.
182
+ _base_advisory_ids() {
183
+ lock_rel="$1"
184
+
185
+ base_content="$(git show "${BASE_REMOTE_REF}:${lock_rel}" 2>/dev/null)"
186
+ if [ -z "$base_content" ]; then
187
+ return 0
188
+ fi
189
+
190
+ workdir=$(mktemp -d "${TMPDIR:-/tmp}/js-dependency-audit-base.XXXXXX") || return 1
191
+ printf '%s' "$base_content" >"$workdir/pnpm-lock.yaml"
192
+
193
+ for attempt in $(seq 1 "$attempts"); do
194
+ # shellcheck disable=SC2086
195
+ base_out="$(cd "$workdir" && timeout "$AUDIT_TIMEOUT_SECS" pnpm audit --json --ignore-workspace 2>/dev/null)"
196
+ base_status=$?
197
+ if [ "$base_status" -eq 124 ]; then
198
+ [ "$attempt" -lt "$attempts" ] && sleep "$((attempt * 2))"
199
+ continue
200
+ fi
201
+ if printf '%s' "$base_out" | jq -e '.metadata.vulnerabilities' >/dev/null 2>&1; then
202
+ printf '%s' "$base_out" | jq -r '.advisories[]? | (.github_advisory_id // (.id|tostring))' 2>/dev/null
203
+ rm -rf "$workdir"
204
+ return 0
205
+ fi
206
+ [ "$attempt" -lt "$attempts" ] && sleep "$((attempt * 2))"
207
+ done
208
+
209
+ rm -rf "$workdir"
210
+ return 1
211
+ }
212
+
122
213
  audit_dir() {
123
214
  dir="$1"
124
215
  label="$2"
125
216
  extra="$3"
126
217
  resultfile="$4"
218
+ lock_rel="$5"
127
219
 
128
220
  for attempt in $(seq 1 "$attempts"); do
129
221
  # printf, never echo: the CI step runs `sh scripts/...` i.e. dash, whose
@@ -172,10 +264,54 @@ audit_dir() {
172
264
  low="$(printf '%s' "$out" | jq '.metadata.vulnerabilities.low // 0')"
173
265
  total="$(printf '%s' "$out" | jq '.metadata.totalDependencies // 0')"
174
266
  if [ "$((high + crit))" -gt 0 ]; then
175
- echo "::error::${label}: ${crit} critical + ${high} high advisory(ies) across ${total} package(s); registry answered ${seen_at}."
267
+ # Classify each qualifying (high/critical) advisory against the base
268
+ # branch's SAME lockfile before deciding to block (#2040). Written to
269
+ # a regular file, not a pipe, and read with `while read ... done <
270
+ # file` rather than `| while read`, for the same reason
271
+ # py-dependency-audit.sh's findings loop does: a pipeline runs the
272
+ # loop in a subshell, and a subshell's counter updates vanish the
273
+ # instant it exits.
274
+ findings_file="$(mktemp)"
275
+ printf '%s' "$out" | jq -r '.advisories[]? | select(.severity=="high" or .severity=="critical") | "\(.github_advisory_id // (.id|tostring))\t\(.severity)\t\(.module_name)"' >"$findings_file"
276
+
277
+ base_ids_file=""
278
+ base_available=0
279
+ if [ "$COMPARE_MODE" -eq 1 ]; then
280
+ candidate_ids="$(mktemp)"
281
+ if _base_advisory_ids "$lock_rel" >"$candidate_ids" 2>/dev/null; then
282
+ base_ids_file="$candidate_ids"
283
+ base_available=1
284
+ else
285
+ rm -f "$candidate_ids"
286
+ fi
287
+ fi
288
+
289
+ introduced_count=0
290
+ preexisting_count=0
291
+ while IFS="$(printf '\t')" read -r f_id f_sev f_mod; do
292
+ [ -z "$f_id" ] && continue
293
+ if [ "$base_available" -eq 1 ] && grep -qxF "$f_id" "$base_ids_file" 2>/dev/null; then
294
+ preexisting_count=$((preexisting_count + 1))
295
+ echo "::warning::${label}: ${f_mod} advisory ${f_id} (${f_sev}) is already present in ${BASE_REMOTE_REF}'s lockfile — pre-existing, not introduced by this diff (#2040)."
296
+ else
297
+ introduced_count=$((introduced_count + 1))
298
+ echo "::error::${label}: ${f_mod} advisory ${f_id} (${f_sev}) — new to this tree, or a version this diff introduced/upgraded (or a base-branch comparison was not possible)."
299
+ fi
300
+ done <"$findings_file"
301
+ rm -f "$findings_file"
302
+ [ -n "$base_ids_file" ] && rm -f "$base_ids_file"
303
+
176
304
  printf '%s' "$out" | jq '.advisories // .metadata.vulnerabilities' 2>/dev/null | head -c 4000
177
- echo "fail" >"$resultfile"
178
- return 1
305
+
306
+ if [ "$introduced_count" -gt 0 ]; then
307
+ echo "::error::${label}: ${introduced_count} critical/high advisory(ies) introduced or upgraded by this diff across ${total} package(s) (${preexisting_count} more pre-existing, not counted against it); registry answered ${seen_at}."
308
+ echo "fail" >"$resultfile"
309
+ return 1
310
+ fi
311
+
312
+ echo "${label}: ${preexisting_count} critical/high advisory(ies) found, all pre-existing on ${BASE_REMOTE_REF:-the base branch} and unrelated to this diff — not blocking (#2040); registry answered ${seen_at}."
313
+ echo "ok" >"$resultfile"
314
+ return 0
179
315
  fi
180
316
  # A bare "no advisories" is not falsifiable. State the population, the
181
317
  # severities that did NOT block, and when the registry was asked, so a
@@ -210,6 +346,23 @@ fi
210
346
 
211
347
  WORKSPACE_ABS=$(pwd -P)
212
348
 
349
+ # Decide once, for the whole run, whether a base-branch comparison is even
350
+ # possible (#2040 — see the "Pre-existing vs introduced" docstring above
351
+ # `set -u`). `GITHUB_BASE_REF` is only ever set by a
352
+ # `pull_request`/`pull_request_target` event; everywhere else COMPARE_MODE
353
+ # stays 0 and every finding blocks, unchanged from before this fix.
354
+ BASE_REMOTE_REF=""
355
+ COMPARE_MODE=0
356
+ if [ -n "${GITHUB_BASE_REF:-}" ]; then
357
+ candidate="origin/${GITHUB_BASE_REF}"
358
+ if git rev-parse --verify --quiet "${candidate}^{commit}" >/dev/null 2>&1; then
359
+ BASE_REMOTE_REF="$candidate"
360
+ COMPARE_MODE=1
361
+ else
362
+ echo "::warning::js-dependency-audit: base branch is '${GITHUB_BASE_REF}' but '${candidate}' does not resolve locally (shallow checkout?) — cannot tell a pre-existing finding from one this diff introduced, so every finding will block, same as before #2040."
363
+ fi
364
+ fi
365
+
213
366
  # shellcheck disable=SC2016
214
367
  ALL_LOCKS=$(find "$REPO_ROOT" \
215
368
  \( -name node_modules -o -name .git -o -name .worktrees \) -prune -o \
@@ -286,10 +439,14 @@ for lock in $ALL_LOCKS; do
286
439
  esac
287
440
  i=$((i + 1))
288
441
  resultfile="$TMP_DIR/result.$i"
442
+ # `find` was rooted at $REPO_ROOT, so $lock is always an absolute path
443
+ # under it — this strip is unconditional, unlike $rel's dir_abs case above
444
+ # (which also has to tolerate a symlink resolving outside the root).
445
+ lock_rel=${lock#"$REPO_ROOT"/}
289
446
  if [ "$dir_abs" = "$WORKSPACE_ABS" ]; then
290
- audit_dir "$dir" "pnpm audit (workspace: ${rel})" "" "$resultfile" &
447
+ audit_dir "$dir" "pnpm audit (workspace: ${rel})" "" "$resultfile" "$lock_rel" &
291
448
  else
292
- audit_dir "$dir" "pnpm audit (${rel})" "--ignore-workspace" "$resultfile" &
449
+ audit_dir "$dir" "pnpm audit (${rel})" "--ignore-workspace" "$resultfile" "$lock_rel" &
293
450
  fi
294
451
  done
295
452
 
package/dist/index.js CHANGED
@@ -7632,10 +7632,6 @@ function rewriteDesignTokens(targetDir, tokens) {
7632
7632
  " timeout-minutes: 20\n defaults:\n run:\n working-directory: apps/frontend\n steps:",
7633
7633
  " timeout-minutes: 20\n permissions:\n contents: read\n packages: read\n defaults:\n run:\n working-directory: apps/frontend\n steps:"
7634
7634
  );
7635
- yml = yml.replace(
7636
- " continue-on-error: true\n defaults:\n run:\n working-directory: apps/frontend\n steps:",
7637
- " continue-on-error: true\n permissions:\n contents: read\n packages: read\n defaults:\n run:\n working-directory: apps/frontend\n steps:"
7638
- );
7639
7635
  yml = yml.replace(
7640
7636
  /( {6}- run: pnpm install --frozen-lockfile)\n(?! {8}env:)/g,
7641
7637
  "$1\n env:\n NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n"
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@biffo/cli",
3
- "version": "0.315.6",
3
+ "version": "0.315.8",
4
4
  "description": "Biffo project scaffolding CLI",
5
5
  "license": "MIT",
6
6
  "type": "module",
@@ -33,9 +33,7 @@
33
33
  "scripts/pgtest-diff-check.sh",
34
34
  "scripts/gate-coverage.sh",
35
35
  "scripts/verify.sh",
36
- "scripts/runner-drop-forensics.mjs",
37
- "scripts/practices-metrics.mjs",
38
- "scripts/practices-corpus.mjs"
36
+ "scripts/runner-drop-forensics.mjs"
39
37
  ],
40
38
  "scripts": {
41
39
  "build": "tsup src/index.ts --format esm --dts --clean --external typescript",
@@ -14,10 +14,9 @@
14
14
  * reading of a red branch and the lazy one are indistinguishable, and #982
15
15
  * showed the estate had been counting these as broken code for months.
16
16
  *
17
- * `isRunnerKill` (in `practices-metrics.mjs`, imported below rather than
18
- * re-implemented) already answers *"did a runner die?"* from the run's own step
19
- * conclusions. It cannot answer *"why?"* — and "why" is what decides whether
20
- * anyone should be looking at the code at all.
17
+ * `isRunnerKill` below already answers *"did a runner die?"* from the run's own
18
+ * step conclusions. It cannot answer *"why?"* — and "why" is what decides
19
+ * whether anyone should be looking at the code at all.
21
20
  *
22
21
  * ## The join nobody had written
23
22
  *
@@ -61,7 +60,80 @@
61
60
 
62
61
  // @ts-check
63
62
  import { execFileSync } from 'node:child_process'
64
- import { isRunnerKill } from './practices-metrics.mjs'
63
+
64
+ /**
65
+ * Step conclusions that mean the step **stopped without a verdict**.
66
+ *
67
+ * A dying runner produces two different signatures and #982 caught only the
68
+ * first, so `biffo-platform` kept two failures it had not earned:
69
+ *
70
+ * - `null` — the step was still executing when the lights went out. A deploy
71
+ * frozen on "Package and deploy Lambda", six steps left `pending`.
72
+ * - `cancelled` — the step was stopped, and every later step reads `skipped`.
73
+ * Two `biffo-platform` CI runs died 64 seconds in this way, on "Type check"
74
+ * and "Lint".
75
+ *
76
+ * ## Why `cancelled` here is not an ordinary cancellation
77
+ *
78
+ * The obvious objection is that this launders someone hitting cancel, or a
79
+ * `cancel-in-progress` supersession. It does not, and the reason is structural:
80
+ * **those conclude the run `cancelled`**, which `isRunnerKill` is only ever
81
+ * reached for a run that concluded `failure`. A run that concluded `failure`
82
+ * while no step ever returned a verdict was therefore stopped by something
83
+ * that is not a cancellation.
84
+ *
85
+ * A step that hits `timeout-minutes` (20 since #980) is expected to be marked
86
+ * `failure` and so stays a real failure.
87
+ */
88
+ const STOPPED_SHORT = new Set([null, undefined, 'cancelled'])
89
+
90
+ /**
91
+ * Did this run fail because a **runner died**, rather than because a gate
92
+ * rejected the change? (#982)
93
+ *
94
+ * ## The hole this closes
95
+ *
96
+ * A killed or superseded run naturally concludes `cancelled`, which is not a
97
+ * defect. That reasoning is right and its coverage is only partial: **a runner
98
+ * killed mid-job reports `cancelled` only sometimes.** The rest of the time
99
+ * GitHub concludes the run `failure` with *no failing step* — the same
100
+ * physical event, a different label, and the second label was counted as if
101
+ * code had broken.
102
+ *
103
+ * Measured on `tabsii-com/tabsii-platform`, 2026-07-31: **all six** `dev`
104
+ * failures inspected had zero failing steps and 3–21 steps left incomplete. One
105
+ * deploy succeeded through thirteen steps and froze on "Package and deploy
106
+ * Lambda". Not one gate rejected a change.
107
+ *
108
+ * ## The rule, and why it errs the way it does
109
+ *
110
+ * A failed run is a runner kill when **no job reports a failing step** and **at
111
+ * least one failed job has a step that stopped without a verdict** — see
112
+ * {@link STOPPED_SHORT} for the two signatures that means, and why `cancelled`
113
+ * among them is not an ordinary cancellation. Both halves matter: the first
114
+ * says nothing rejected the change, the second says work was still outstanding
115
+ * when the lights went out.
116
+ *
117
+ * A failed run with no steps recorded at all is deliberately **not** classified
118
+ * as a kill. It stays a failure. That is the conservative direction for a
119
+ * counter-metric — it can still refute an experiment the author would prefer to
120
+ * confirm — and this module's whole purpose is to make that the default.
121
+ *
122
+ * A job that hits its `timeout-minutes` (20 since #980) marks the offending step
123
+ * `failure`, so a genuine hang stays a genuine failure and is not laundered
124
+ * through here.
125
+ *
126
+ * @param {Array<Record<string, any>>} jobs the `jobs` array of one run
127
+ * @returns {boolean}
128
+ */
129
+ export function isRunnerKill(jobs) {
130
+ const failed = (jobs ?? []).filter((job) => job.conclusion === 'failure')
131
+ if (failed.length === 0) return false
132
+ const steps = failed.flatMap((job) => job.steps ?? [])
133
+ if (steps.length === 0) return false
134
+ if (steps.some((step) => step.conclusion === 'failure')) return false
135
+ return steps.some((step) => STOPPED_SHORT.has(step.conclusion))
136
+ }
65
137
 
66
138
  /**
67
139
  * How far outside a job's own start/finish window an eviction may fall and
package/scripts/verify.sh CHANGED
@@ -460,9 +460,11 @@ fi
460
460
  # and NO_CI must not fire just because a repo has not adopted the split (a
461
461
  # sibling never will; an instance not yet upgraded past #1319 has not yet).
462
462
  # But if it EXISTS and cannot be READ, that is the identical #1218 shape as
463
- # ci.yml itself, and ci_has() must search it too or "practices-monotonic"
464
- # (moved into it) would silently stop being locally mirrored the moment it
465
- # left ci.yml -- covered less than verify.sh claims, with nothing saying so.
463
+ # ci.yml itself, and ci_has() must search it too -- a check that lives ONLY
464
+ # in release-guards.yml (any of the ones this repo's own header explains
465
+ # moved here to avoid re-running the whole ci.yml matrix on a PR edit) would
466
+ # otherwise silently stop being locally mirrored, covered less than verify.sh
467
+ # claims, with nothing saying so.
466
468
  RELEASE_GUARDS_YML_UNREADABLE=""
467
469
  if [ -f .github/workflows/release-guards.yml ] && [ ! -r .github/workflows/release-guards.yml ]; then
468
470
  RELEASE_GUARDS_YML_UNREADABLE=1
@@ -1544,15 +1546,6 @@ fi
1544
1546
  [ -f scripts/check-orphan-ratchet-instance.test.sh ] &&
1545
1547
  run_check orphan-ratchet-instance-selftest sh scripts/check-orphan-ratchet-instance.test.sh
1546
1548
 
1547
- # The append-only corpus guard (#778). CI runs it in Release Guards, and it was
1548
- # invisible to the parity test until #897 widened the harvester -- it is neither
1549
- # `pnpm`, `uv`, `terraform` nor `sh scripts/`, so the guard whose property is
1550
- # "every CI check is in the gate or explicitly excluded" could not see it at all.
1551
- # Measured 0.06s here, which is cheaper than every other check in this file.
1552
- if [ -f scripts/practices-monotonic.mjs ]; then
1553
- ci_has "practices-monotonic" && run_check corpus-append-only node scripts/practices-monotonic.mjs
1554
- fi
1555
-
1556
1549
  # Terraform plan artefacts, refused by CONTENT (biffo-runners#1).
1557
1550
  #
1558
1551
  # A saved plan is a zip. `strings`/`grep` over it is a false-negative machine —
@@ -1,202 +0,0 @@
1
- /**
2
- * Shared read/write helpers for the practices evidence corpus (#1132).
3
- *
4
- * ## Why a directory, not one shared file
5
- *
6
- * `docs/practices/evidence.jsonl` was a single file every concurrent session
7
- * appended to. N writers, one path — conflicts **by construction**, the same
8
- * class already fixed twice in this repo (`core.version` #423, the generated
9
- * tally block #953). The lever is the same: stop sharing the path. New rows go
10
- * into their own file under `docs/practices/evidence/`, one per entry, e.g.
11
- *
12
- * docs/practices/evidence/2026-08-03-metric-denominator-blindness.json
13
- *
14
- * Two sessions writing on the same day still never collide — their filenames
15
- * differ.
16
- *
17
- * ## Migration: read both, split nothing
18
- *
19
- * Splitting the ~430 existing rows into ~430 files was rejected: it is more
20
- * expensive than the alternative for no benefit, and it would re-serialise a
21
- * file that must never be re-serialised (whole-file rewrites are the exact
22
- * defect being fixed). Instead `evidence.jsonl` is now a **frozen legacy
23
- * file** — nothing ever appends to it again — and the read side merges it
24
- * with the directory. See `practices-monotonic.mjs` for the guard that keeps
25
- * it frozen rather than shrunk.
26
- *
27
- * ## Ordering
28
- *
29
- * Filenames carry the date (`YYYY-MM-DD-slug.json`), so the read side sorts
30
- * the directory listing by filename rather than relying on directory order,
31
- * which the filesystem does not guarantee. Legacy rows keep their existing
32
- * file order (untouched) and sort BEFORE every directory row — they predate
33
- * all of them by construction.
34
- */
35
-
36
- import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from 'node:fs'
37
- import { join } from 'node:path'
38
-
39
- export const LEGACY_EVIDENCE = 'docs/practices/evidence.jsonl'
40
- export const EVIDENCE_DIR = 'docs/practices/evidence'
41
-
42
- /** The per-entry directory that goes with a legacy `.jsonl` path. */
43
- export function corpusDirFor(legacyFile) {
44
- return legacyFile.replace(/\.jsonl$/, '')
45
- }
46
-
47
- /**
48
- * Parse the legacy newline-delimited JSON file, leniently: one malformed line
49
- * is dropped rather than failing the whole read. Matches the tolerance this
50
- * file's readers already had before #1132 (a scan for ranking, not a strict
51
- * audit — `readCorpusStrict` below is the strict counterpart).
52
- */
53
- export function readLegacyEvidence(file = LEGACY_EVIDENCE) {
54
- if (!existsSync(file)) return []
55
- return readFileSync(file, 'utf8')
56
- .split('\n')
57
- .filter((l) => l.trim() !== '')
58
- .map((line) => {
59
- try {
60
- return JSON.parse(line)
61
- } catch {
62
- return null
63
- }
64
- })
65
- .filter(Boolean)
66
- }
67
-
68
- /** `*.json` filenames directly under the evidence directory, sorted so date-prefixed names order chronologically. */
69
- export function listEvidenceFiles(dir = EVIDENCE_DIR) {
70
- if (!existsSync(dir)) return []
71
- return readdirSync(dir)
72
- .filter((f) => f.endsWith('.json'))
73
- .sort()
74
- }
75
-
76
- /** Every per-entry file, parsed, alongside the filename it came from — needed to rewrite a specific entry (e.g. `--enrich`). */
77
- export function readEvidenceDirEntries(dir = EVIDENCE_DIR) {
78
- return listEvidenceFiles(dir)
79
- .map((file) => {
80
- try {
81
- return { file, row: JSON.parse(readFileSync(join(dir, file), 'utf8')) }
82
- } catch {
83
- return null
84
- }
85
- })
86
- .filter(Boolean)
87
- }
88
-
89
- /** Every per-entry file's row, sorted by filename. Malformed files are dropped, not fatal. */
90
- export function readEvidenceDir(dir = EVIDENCE_DIR) {
91
- return readEvidenceDirEntries(dir).map((e) => e.row)
92
- }
93
-
94
- /**
95
- * The full corpus, lenient: legacy rows (their existing order, untouched)
96
- * followed by directory rows (sorted by filename). A concatenation, not a
97
- * merge — the two never name the same entry, so there is nothing to
98
- * reconcile.
99
- *
100
- * @param {string} legacyFile path to the legacy `.jsonl`; its sibling
101
- * directory is derived from it (`corpusDirFor`)
102
- */
103
- export function readCorpus(legacyFile = LEGACY_EVIDENCE) {
104
- return [...readLegacyEvidence(legacyFile), ...readEvidenceDir(corpusDirFor(legacyFile))]
105
- }
106
-
107
- /**
108
- * The full corpus, strict: throws on the first line or file that fails to
109
- * parse, and throws if neither the legacy file nor the directory has
110
- * anything to read. For callers whose whole point is "never report a zero
111
- * that could actually be 'could not read this'" (`summariseFailOpenBacklog`)
112
- * — a corpus that half-parses must not silently look like a smaller valid
113
- * one.
114
- *
115
- * @param {string} legacyFile
116
- */
117
- export function readCorpusStrict(legacyFile = LEGACY_EVIDENCE) {
118
- const dir = corpusDirFor(legacyFile)
119
- const legacyExists = existsSync(legacyFile)
120
- const dirFiles = listEvidenceFiles(dir)
121
- if (!legacyExists && dirFiles.length === 0) {
122
- throw new Error(`no corpus at ${legacyFile} or ${dir}`)
123
- }
124
- const legacyRows = legacyExists
125
- ? readFileSync(legacyFile, 'utf8')
126
- .split('\n')
127
- .filter((l) => l.trim() !== '')
128
- .map((l) => JSON.parse(l))
129
- : []
130
- const dirRows = dirFiles.map((f) => JSON.parse(readFileSync(join(dir, f), 'utf8')))
131
- return [...legacyRows, ...dirRows]
132
- }
133
-
134
- /** Filename-safe token from a row's summary. */
135
- export function slugify(text) {
136
- return String(text ?? '')
137
- .toLowerCase()
138
- .replace(/[^a-z0-9]+/g, '-')
139
- .replace(/^-+|-+$/g, '')
140
- .slice(0, 60)
141
- }
142
-
143
- /**
144
- * Write ONE new evidence entry as its own file. This is the write path every
145
- * future session uses — never append to `evidence.jsonl`, which is frozen.
146
- *
147
- * Refuses to overwrite an existing file: a collision means the slug needs to
148
- * be more specific, not that the earlier entry should be silently replaced.
149
- *
150
- * The refusal is an atomic `wx` create, not an `existsSync` check followed by
151
- * a write (#1222). This corpus has concurrent writers BY DESIGN — several
152
- * agent sessions run against this estate at once — so the window between a
153
- * check and a write is not theoretical here: two sessions writing the same
154
- * `date-slug` is the exact case the guard exists for, and the check-then-write
155
- * form lost the earlier entry rather than refusing. `EEXIST` is translated
156
- * back into the same message, so nothing else changes.
157
- *
158
- * @param {Record<string, any>} row
159
- * @param {{dir?: string, date?: string, slug?: string}} [opts]
160
- * @returns {string} the path written, relative to `opts.dir`'s base
161
- */
162
- export function writeEvidenceEntry(row, opts = {}) {
163
- const dir = opts.dir ?? EVIDENCE_DIR
164
- // `undefined` means "nobody said"; `null` means "known to be unknown". Both
165
- // must reach the stored field as null rather than today's date.
166
- //
167
- // This used to read `opts.date ?? row.date ?? new Date()…` and write that
168
- // single value to BOTH the filename and the row. The module docstring says
169
- // the opposite in as many words — "Rows citing nothing keep `date: null` —
170
- // never a guess, because a fabricated date would corrupt exactly the ranking
171
- // this exists to enable" — and `--extract` even passes `date: row.date ??
172
- // null` to say so explicitly. `null ?? today` discarded that.
173
- //
174
- // It was invisible while rows were extracted the day they were written, and
175
- // surfaced on 2026-08-09 when extracting five rows also swept up eighteen
176
- // older ones and stamped every one with that day. `--enrich` recovers real
177
- // dates from the cited issues afterwards, and it can only do that for rows
178
- // whose date is *absent*; a fabricated one looks recovered and is skipped.
179
- const date = opts.date ?? row.date ?? null
180
- const slug = opts.slug ?? slugify(row.summary) ?? 'entry'
181
- mkdirSync(dir, { recursive: true })
182
- // The filename is used only for sorting and uniqueness — every reader parses
183
- // the JSON body — so an undated row says so here too, rather than carrying a
184
- // date prefix that the data it contains denies.
185
- const file = `${date ?? 'undated'}-${slug || 'entry'}.json`
186
- const path = join(dir, file)
187
- try {
188
- writeFileSync(path, `${JSON.stringify({ ...row, date }, null, 2)}\n`, { flag: 'wx' })
189
- } catch (err) {
190
- if (err && err.code === 'EEXIST') {
191
- throw new Error(`${path} already exists — choose a more specific slug or date`)
192
- }
193
- throw err
194
- }
195
- return path
196
- }
197
-
198
- /** Overwrite one already-existing per-entry file in place (e.g. `--enrich` filling in a date). Never touches the legacy file. */
199
- export function writeEvidenceFile(dir, file, row) {
200
- mkdirSync(dir, { recursive: true })
201
- writeFileSync(join(dir, file), `${JSON.stringify(row, null, 2)}\n`)
202
- }