@blamejs/exceptd-skills 0.18.8 → 0.18.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/CHANGELOG.md +28 -0
  2. package/bin/exceptd.js +197 -118
  3. package/data/_indexes/_meta.json +3 -3
  4. package/data/_indexes/frequency.json +2 -2
  5. package/data/d3fend-catalog.json +6 -6
  6. package/data/playbooks/identity-sso-compromise.json +2 -2
  7. package/data/playbooks/sbom.json +1 -1
  8. package/lib/citation-resolve.js +11 -0
  9. package/lib/collectors/containers.js +13 -0
  10. package/lib/cross-ref-api.js +29 -7
  11. package/lib/cve-regression-watcher.js +47 -15
  12. package/lib/framework-gap.js +27 -5
  13. package/lib/gap-detectors.js +8 -3
  14. package/lib/lint-skills.js +3 -2
  15. package/lib/playbook-runner.js +60 -5
  16. package/lib/refresh-external.js +58 -7
  17. package/lib/refresh-network.js +24 -8
  18. package/lib/rfc-cli.js +108 -18
  19. package/lib/schemas/playbook.schema.json +1 -1
  20. package/lib/scoring.js +31 -1
  21. package/lib/source-advisories.js +58 -9
  22. package/lib/ttp-mapper.js +31 -3
  23. package/lib/upstream-check-cli.js +13 -1
  24. package/lib/validate-catalog-meta.js +51 -7
  25. package/lib/validate-cve-catalog.js +10 -0
  26. package/lib/validate-playbooks.js +19 -1
  27. package/lib/xml-tokenizer.js +187 -25
  28. package/manifest.json +53 -53
  29. package/orchestrator/dispatcher.js +45 -9
  30. package/orchestrator/index.js +9 -7
  31. package/orchestrator/pipeline.js +62 -14
  32. package/orchestrator/scanner.js +40 -9
  33. package/package.json +1 -1
  34. package/sbom.cdx.json +105 -90
  35. package/scripts/build-indexes.js +21 -3
  36. package/scripts/builders/section-offsets.js +17 -8
  37. package/scripts/check-catalog-gap-budget.js +3 -3
  38. package/scripts/check-codebase-patterns.js +124 -11
  39. package/scripts/check-sbom-currency.js +69 -3
  40. package/scripts/check-test-count.js +28 -16
  41. package/scripts/check-test-subjects.js +127 -0
  42. package/scripts/check-version-tags.js +24 -5
  43. package/scripts/predeploy.js +13 -0
  44. package/scripts/refresh-upstream-catalogs.js +150 -42
  45. package/scripts/release.js +28 -11
  46. package/scripts/validate-vendor-online.js +12 -9
@@ -74,9 +74,27 @@ function writeJson(name, obj) {
74
74
  // is atomic on POSIX and Windows, so a reader (or the next build's
75
75
  // parse-check) only ever sees the complete old or complete new file.
76
76
  const abs = path.join(IDX, name);
77
- const tmp = `${abs}.tmp-${process.pid}`;
77
+ // Random suffix (not just pid) so a Promise.all fan-out (--parallel) sharing
78
+ // a pid can't race the same tmp path, and the in-repo temp name isn't
79
+ // predictable. Mirrors lib/citation-resolve.js cachePut.
80
+ const tmp = `${abs}.tmp-${process.pid}.${crypto.randomBytes(4).toString("hex")}`;
78
81
  fs.writeFileSync(tmp, JSON.stringify(obj, null, 2) + "\n", "utf8");
79
- fs.renameSync(tmp, abs);
82
+ // rename-over-existing is atomic, but on Windows a sync client / AV / indexer
83
+ // (e.g. Dropbox holding the target open for a few ms) can make it transiently
84
+ // EPERM/EACCES/EBUSY. Retry with short backoff so a build doesn't fail on a
85
+ // lock that clears in milliseconds; on POSIX the first attempt always wins.
86
+ let lastErr;
87
+ for (let attempt = 0; attempt < 10; attempt++) {
88
+ try { fs.renameSync(tmp, abs); return; }
89
+ catch (e) {
90
+ lastErr = e;
91
+ if (e.code !== "EPERM" && e.code !== "EACCES" && e.code !== "EBUSY") break;
92
+ Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 20 * (attempt + 1));
93
+ }
94
+ }
95
+ // Persistent failure: drop the temp sibling so it doesn't orphan, then surface.
96
+ try { fs.unlinkSync(tmp); } catch { /* best effort */ }
97
+ throw lastErr;
80
98
  }
81
99
 
82
100
  function readJson(p) {
@@ -795,7 +813,7 @@ async function main() {
795
813
  for (const n of wanted) {
796
814
  if (!OUTPUTS.find((o) => o.name === n)) {
797
815
  console.error(`build-indexes: unknown output "${n}". Valid: ${OUTPUTS.map((o) => o.name).join(", ")}`);
798
- process.exit(2);
816
+ process.exitCode = 2; return;
799
817
  }
800
818
  }
801
819
  chosen = withDependencyClosure(wanted);
@@ -68,13 +68,19 @@ function buildOne(absPath, relPath) {
68
68
  const totalBytes = buf.length;
69
69
  const text = buf.toString("utf8");
70
70
  const lines = text.split(/\r?\n/);
71
- const lineByteOffsets = [];
72
- let cursor = 0;
73
- for (const line of lines) {
74
- lineByteOffsets.push(cursor);
75
- // +1 for the newline. Counts as one byte for LF; CRLF would skew slightly
76
- // but the file is written via the project's tooling which is LF-uniform.
77
- cursor += Buffer.byteLength(line, "utf8") + 1;
71
+ // EOL-aware line-start byte offsets: walk the actual terminator bytes off the
72
+ // decoded text rather than assuming a fixed 1-byte newline. On a pure-LF body
73
+ // every terminator is 1 byte so the offsets are identical to the old `+ 1`
74
+ // accumulator; on a stray CRLF body the \r\n terminator is 2 bytes and the
75
+ // offsets stay correct (the old fixed-constant approach undercounted by 1
76
+ // byte per line and silently misaligned every token-budget slice). The split
77
+ // above discards terminator bytes, so the width can only be recovered by
78
+ // reading the terminators off the raw text — done here.
79
+ const lineByteOffsets = [0];
80
+ const eolRe = /\r?\n/g;
81
+ let m;
82
+ while ((m = eolRe.exec(text)) !== null) {
83
+ lineByteOffsets.push(Buffer.byteLength(text.slice(0, m.index + m[0].length), "utf8"));
78
84
  }
79
85
 
80
86
  // Frontmatter: lines between the first "---" and the second "---".
@@ -167,4 +173,7 @@ function buildSectionOffsets({ root, skills }) {
167
173
  };
168
174
  }
169
175
 
170
- module.exports = { buildSectionOffsets };
176
+ // buildOne is exported for regression testing of the EOL-aware byte offsets
177
+ // (a CRLF body must still produce byte_start values that point at the real
178
+ // "## " byte in the raw file).
179
+ module.exports = { buildSectionOffsets, buildOne };
@@ -116,12 +116,12 @@ function main() {
116
116
  console.error("Add an explicit budget entry in both:");
117
117
  console.error(" scripts/check-catalog-gap-budget.js");
118
118
  console.error(" tests/shipped-catalog-integrity.test.js");
119
- process.exit(1);
119
+ process.exitCode = 1; return;
120
120
  }
121
121
  if (missingBudget.length > 0) {
122
122
  console.error("\n[check-catalog-gap-budget] BUDGET missing entries for declared classes:");
123
123
  for (const c of missingBudget) console.error(` ${c}: declared by lib/gap-detectors.js DETECTOR_CLASSES, no BUDGET entry`);
124
- process.exit(1);
124
+ process.exitCode = 1; return;
125
125
  }
126
126
  if (regressions.length > 0) {
127
127
  console.error("\n[check-catalog-gap-budget] REGRESSION beyond budget:");
@@ -131,7 +131,7 @@ function main() {
131
131
  console.error("\nClose the gap in this PR (preferred) or update BUDGET in both:");
132
132
  console.error(" scripts/check-catalog-gap-budget.js");
133
133
  console.error(" tests/shipped-catalog-integrity.test.js");
134
- process.exit(1);
134
+ process.exitCode = 1; return;
135
135
  }
136
136
  console.log("[check-catalog-gap-budget] all classes within budget; every class is budgeted.");
137
137
  }
@@ -148,6 +148,79 @@ function filterMarkers(hits, cls) {
148
148
 
149
149
  // ---- require.main block ranges -------------------------------------------
150
150
 
151
+ // Count `{` / `}` in `line` that are in REAL CODE context, advancing a
152
+ // stateful tokenizer that tracks string / template / comment regions across
153
+ // lines. Braces inside a single/double/template string, a `//` line comment,
154
+ // or a `/* */` block comment do NOT affect depth — otherwise a `{` or `}`
155
+ // typed inside a string literal in the require.main block miscounts the brace
156
+ // balance and the computed block range slides onto an unrelated later function
157
+ // (whose process.exit() is then wrongly treated as a CLI-entry exit and not
158
+ // flagged). `inTemplate` and `inBlockComment` are the cross-line states a
159
+ // per-line stripper cannot model, so the tokenizer state object is threaded
160
+ // line-to-line by the caller.
161
+ //
162
+ // `state` is mutated in place: { inSingle, inDouble, inTemplate, inBlock,
163
+ // templateDepth } — `templateDepth` tracks `${ … }` interpolation nesting so
164
+ // the closing `}` of an interpolation is treated as template punctuation, not
165
+ // a code brace, while braces INSIDE the interpolation expression still count.
166
+ function countCodeBraces(line, state) {
167
+ let delta = 0;
168
+ for (let i = 0; i < line.length; i++) {
169
+ const ch = line[i];
170
+ const next = line[i + 1];
171
+ if (state.inBlock) {
172
+ if (ch === "*" && next === "/") { state.inBlock = false; i++; }
173
+ continue;
174
+ }
175
+ if (state.inSingle) {
176
+ if (ch === "\\") { i++; continue; }
177
+ if (ch === "'") state.inSingle = false;
178
+ continue;
179
+ }
180
+ if (state.inDouble) {
181
+ if (ch === "\\") { i++; continue; }
182
+ if (ch === '"') state.inDouble = false;
183
+ continue;
184
+ }
185
+ if (state.inTemplate) {
186
+ if (ch === "\\") { i++; continue; }
187
+ if (ch === "`") { state.inTemplate = false; continue; }
188
+ if (ch === "$" && next === "{") {
189
+ // Enter an interpolation expression: braces inside ARE code.
190
+ state.templateExpr.push(0);
191
+ state.inTemplate = false;
192
+ i++; // skip the `{`; the `${` opener is template punctuation
193
+ continue;
194
+ }
195
+ continue;
196
+ }
197
+ // Code context (possibly inside a template interpolation expression).
198
+ if (ch === "/" && next === "/") break; // `//` — rest of the line is a comment
199
+ if (ch === "/" && next === "*") { state.inBlock = true; i++; continue; }
200
+ if (ch === "'") { state.inSingle = true; continue; }
201
+ if (ch === '"') { state.inDouble = true; continue; }
202
+ if (ch === "`") { state.inTemplate = true; continue; }
203
+ if (ch === "{") {
204
+ if (state.templateExpr.length) state.templateExpr[state.templateExpr.length - 1]++;
205
+ delta++;
206
+ } else if (ch === "}") {
207
+ if (state.templateExpr.length && state.templateExpr[state.templateExpr.length - 1] === 0) {
208
+ // Closes the `${ … }` interpolation — back to template body.
209
+ state.templateExpr.pop();
210
+ state.inTemplate = true;
211
+ } else {
212
+ if (state.templateExpr.length) state.templateExpr[state.templateExpr.length - 1]--;
213
+ delta--;
214
+ }
215
+ }
216
+ }
217
+ return delta;
218
+ }
219
+
220
+ function newBraceState() {
221
+ return { inSingle: false, inDouble: false, inTemplate: false, inBlock: false, templateExpr: [] };
222
+ }
223
+
151
224
  // Line ranges (1-based, inclusive) of `if (require.main === module) { ... }`
152
225
  // blocks — the dual-mode CLI-entry section where synchronous-print-then-exit
153
226
  // is correct. process.exit there is owned by tests/safe-exit-grep.test.js and
@@ -156,15 +229,16 @@ function requireMainRanges(lines) {
156
229
  const ranges = [];
157
230
  for (let i = 0; i < lines.length; i++) {
158
231
  if (/\brequire\.main\s*===\s*module\b/.test(lines[i])) {
159
- // Find the opening brace (same line or next few), then balance.
232
+ // Find the opening brace (same line or next few), then balance —
233
+ // string/comment/template-aware so braces inside literals don't skew the
234
+ // depth (see countCodeBraces).
160
235
  let depth = 0;
161
236
  let started = false;
162
237
  let j = i;
238
+ const state = newBraceState();
163
239
  for (; j < lines.length; j++) {
164
- for (const ch of lines[j]) {
165
- if (ch === "{") { depth++; started = true; }
166
- else if (ch === "}") { depth--; }
167
- }
240
+ depth += countCodeBraces(lines[j], state);
241
+ if (depth > 0) started = true;
168
242
  if (started && depth <= 0) break;
169
243
  }
170
244
  if (started) ranges.push([i + 1, j + 1]);
@@ -181,8 +255,14 @@ function inRanges(ranges, lineNo) {
181
255
 
182
256
  // A line that opens a new function body (so a backward stdout-write scan stops
183
257
  // at the enclosing function and doesn't arm an exit from an unrelated earlier
184
- // function).
185
- const FUNCTION_START = /(^|[^.\w])function\b|=>\s*\{?\s*$|^\s*(async\s+)?[A-Za-z_$][\w$]*\s*\([^)]*\)\s*\{/;
258
+ // function). The bare-identifier (third) alternative matches a declaration /
259
+ // method-shorthand opener (`foo() {`, `async bar() {`), but it must REFUSE
260
+ // control-flow openers (`for (…) {`, `if (…) {`, `while/switch/catch (…) {`):
261
+ // a control-flow block sitting between a stdout write and a process.exit() is
262
+ // inside the SAME function, so stopping the backward scan there would wrongly
263
+ // leave the exit unflagged. The negative lookahead excludes the control-flow
264
+ // keywords; `function` and arrow alternatives are unchanged.
265
+ const FUNCTION_START = /(^|[^.\w])function\b|=>\s*\{?\s*$|^\s*(async\s+)?(?!(?:if|for|while|switch|catch|do|else|with|finally|return)\b)[A-Za-z_$][\w$]*\s*\([^)]*\)\s*\{/;
186
266
 
187
267
  function detectProcessExitAfterStdout(files) {
188
268
  const hits = [];
@@ -210,6 +290,15 @@ function detectProcessExitAfterStdout(files) {
210
290
  return filterMarkers(hits, "process-exit-after-stdout-write");
211
291
  }
212
292
 
293
+ // The first non-whitespace char of the first arg is `"`, `'`, or `/` => a
294
+ // string/regex literal => static, safe. Anything else (an identifier, a `(`,
295
+ // a backtick template) is operator-derivable and flagged. Backtick is NOT
296
+ // exempt — a template literal can interpolate operator input, so it must be
297
+ // flagged the same as a bare identifier.
298
+ function isStaticRegexFirstChar(ch) {
299
+ return ch === '"' || ch === "'" || ch === "/";
300
+ }
301
+
213
302
  function detectDynamicRegex(files) {
214
303
  const hits = [];
215
304
  for (const rel of (files || filesUnder(["lib", "orchestrator", "bin/exceptd.js"]))) {
@@ -217,10 +306,29 @@ function detectDynamicRegex(files) {
217
306
  for (let i = 0; i < lines.length; i++) {
218
307
  const code = stripLineComment(lines[i]);
219
308
  const m = code.match(/\bnew RegExp\s*\(\s*(.)/);
220
- if (!m) continue;
221
- // Literal first arg => a quote or a `/` regex literal => static, safe.
222
- const firstChar = m[1];
223
- if (firstChar === '"' || firstChar === "'" || firstChar === "/") continue;
309
+ if (m) {
310
+ if (isStaticRegexFirstChar(m[1])) continue;
311
+ hits.push({ file: rel, line: i + 1, content: lines[i].trim() });
312
+ continue;
313
+ }
314
+ // Multi-line form: `new RegExp(` ends the (comment-stripped) line with the
315
+ // open paren as the last token, and the pattern arg is on a following
316
+ // line. The single-line match above can't see the first-arg char, so it
317
+ // would silently pass a dynamic RegExp whose argument starts next line.
318
+ // Look ahead, skipping blank and comment-only lines (capped at 5), and
319
+ // inspect the first code line's first non-whitespace char.
320
+ if (!/\bnew RegExp\s*\(\s*$/.test(code)) continue;
321
+ let firstChar = null;
322
+ for (let k = i + 1; k <= i + 5 && k < lines.length; k++) {
323
+ const ahead = stripLineComment(lines[k]).replace(/^\s+/, "");
324
+ if (ahead === "") continue; // blank or comment-only — skip
325
+ firstChar = ahead[0];
326
+ break;
327
+ }
328
+ // A `new RegExp(` with nothing parseable after it within the cap is
329
+ // suspicious — flag conservatively. Otherwise apply the SAME literal
330
+ // exemption as the single-line path.
331
+ if (firstChar !== null && isStaticRegexFirstChar(firstChar)) continue;
224
332
  hits.push({ file: rel, line: i + 1, content: lines[i].trim() });
225
333
  }
226
334
  }
@@ -453,6 +561,11 @@ module.exports = {
453
561
  detectMisalignedMarkedRun,
454
562
  scanUnsortedMarkedArray,
455
563
  scanMisalignedMarkedRun,
564
+ requireMainRanges,
565
+ countCodeBraces,
566
+ newBraceState,
567
+ isStaticRegexFirstChar,
568
+ FUNCTION_START,
456
569
  filesUnder,
457
570
  };
458
571
 
@@ -50,6 +50,66 @@ function catalogEntryCount(dataDir, file) {
50
50
  return 0;
51
51
  }
52
52
 
53
+ // Files/prefixes that refresh-sbom's expandAllowlist excludes from the shipped
54
+ // file: component inventory. Kept in sync with scripts/refresh-sbom.js
55
+ // (SELF_EXCLUDED + DERIVABLE_PREFIXES): the SBOM never hashes itself, and the
56
+ // pre-computed index cache under data/_indexes/ is regenerated/test-mutated, so
57
+ // it carries no per-file component. The completeness check below must apply the
58
+ // SAME exclusions or it would demand a component for a file the generator never
59
+ // emits one for.
60
+ const SBOM_SELF_EXCLUDED = new Set(["sbom.cdx.json"]);
61
+ const SBOM_DERIVABLE_PREFIXES = ["data/_indexes/"];
62
+
63
+ function sbomIsDerivable(rel) {
64
+ return SBOM_DERIVABLE_PREFIXES.some(
65
+ (p) => rel === p.replace(/\/$/, "") || rel.startsWith(p)
66
+ );
67
+ }
68
+
69
+ // Recursively list every regular file under absDir, returned as absolute paths.
70
+ // Mirrors refresh-sbom's walkFiles (which is root-agnostic — it walks whatever
71
+ // absolute dir it is given).
72
+ function walkFilesAbs(absDir) {
73
+ const out = [];
74
+ let entries;
75
+ try { entries = fs.readdirSync(absDir, { withFileTypes: true }); }
76
+ catch { return out; }
77
+ for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) {
78
+ const abs = path.join(absDir, entry.name);
79
+ if (entry.isDirectory()) out.push(...walkFilesAbs(abs));
80
+ else if (entry.isFile()) out.push(abs);
81
+ }
82
+ return out;
83
+ }
84
+
85
+ // Root-aware allowlist expansion: the shipped-file set that must each have a
86
+ // file: component, computed against the TARGET tree (`root`), not the running
87
+ // script's source repo. refresh-sbom's exported expandAllowlist is bound to its
88
+ // own REPO_ROOT (it joins/relativizes against __dirname/..), so under a `--root`
89
+ // target the completeness check would otherwise validate the SOURCE repo's file
90
+ // list against the TARGET SBOM — the wrong tree. Replicating the expansion here
91
+ // (same SELF_EXCLUDED + DERIVABLE_PREFIXES exclusions, same dedupe+sort) keeps
92
+ // the gate honest under `--root` without reaching across into the generator's
93
+ // module-level root.
94
+ function expandAllowlistAt(allowlist, root) {
95
+ const abs = [];
96
+ for (const entry of allowlist) {
97
+ const full = path.join(root, entry);
98
+ let stat;
99
+ try { stat = fs.statSync(full); }
100
+ catch { continue; } // tolerate a stale entry; presence checks elsewhere flag
101
+ if (stat.isDirectory()) abs.push(...walkFilesAbs(full));
102
+ else if (stat.isFile()) abs.push(full);
103
+ }
104
+ const rel = Array.from(
105
+ new Set(abs.map((a) => path.relative(root, a).split(path.sep).join("/")))
106
+ )
107
+ .filter((r) => !SBOM_SELF_EXCLUDED.has(r))
108
+ .filter((r) => !sbomIsDerivable(r))
109
+ .sort();
110
+ return rel;
111
+ }
112
+
53
113
  // The description string embeds per-catalog ENTRY counts as free text, e.g.
54
114
  // "11 catalogs (439 CVEs / 177 CWEs / 805 ATT&CK + ICS / 170 ATLAS /
55
115
  // 468 D3FEND / 8888 RFCs)". Each token maps to one data/*.json catalog whose
@@ -332,9 +392,15 @@ function checkSbomCurrency(root) {
332
392
  // refresh-sbom's exact allowlist expansion + digest so the gate can't drift
333
393
  // from the generator.
334
394
  try {
335
- const { expandAllowlist, bundleDigest } = require("./refresh-sbom");
395
+ // bundleDigest operates purely on the file: component objects (no tree
396
+ // walk), so it is root-agnostic and safe to reuse from the generator. The
397
+ // allowlist expansion, by contrast, MUST run against the target `root`
398
+ // (expandAllowlistAt below) — refresh-sbom's exported expandAllowlist is
399
+ // pinned to its own source-repo REPO_ROOT and would validate the wrong tree
400
+ // under `--root`.
401
+ const { bundleDigest } = require("./refresh-sbom");
336
402
  const pkg = JSON.parse(fs.readFileSync(path.join(root, "package.json"), "utf8"));
337
- const expected = expandAllowlist(pkg.files || []);
403
+ const expected = expandAllowlistAt(pkg.files || [], root);
338
404
  const fileComps = components.filter(
339
405
  (c) => typeof c["bom-ref"] === "string" && c["bom-ref"].startsWith("file:")
340
406
  );
@@ -402,6 +468,6 @@ function main() {
402
468
  );
403
469
  }
404
470
 
405
- module.exports = { checkSbomCurrency, resolveRoot, DESCRIPTION_ENTRY_TOKENS, catalogEntryCount };
471
+ module.exports = { checkSbomCurrency, resolveRoot, DESCRIPTION_ENTRY_TOKENS, catalogEntryCount, expandAllowlistAt };
406
472
 
407
473
  if (require.main === module) main();
@@ -11,18 +11,24 @@
11
11
  * catches test-set shrinkage.
12
12
  *
13
13
  * Scope + blind spot. This counts test DECLARATIONS, so it detects deleted
14
- * files / removed `test(` calls / glob-exclusions. It does NOT detect a test
15
- * neutered in place: `test('name', { skip: true }, fn)` and `test.skip(` both
16
- * still count as one declaration, so flipping a running test to permanently
17
- * skipped leaves the count unchanged. Guarding against skip-in-place would
18
- * need runnable-vs-skipped tracking; that is out of scope for this gate.
14
+ * files / removed `test(`/`it(` calls / glob-exclusions. It does NOT detect a
15
+ * test neutered in place: `test('name', { skip: true }, fn)`, `test.skip(`,
16
+ * and `it.skip(` all still count as one declaration, so flipping a running
17
+ * test to permanently skipped leaves the count unchanged. Guarding against
18
+ * skip-in-place would need runnable-vs-skipped tracking; that is out of scope
19
+ * for this gate.
19
20
  *
20
- * Mechanism: count `test(`, `test.only(`, and `test.skip(` declarations
21
- * across `tests/*.test.js` via static analysis (faster than running). Compare
22
- * to a baseline pinned in `tests/.test-count-baseline.json`. Fail if the
23
- * observed count drops MORE than the configured tolerance (default 1) below
24
- * the baseline. Growth above baseline is fine; if the count grows by more
25
- * than `update_baseline_when_growth_exceeds`, surface a notice that the
21
+ * Mechanism: count `test(`/`it(` declarations (including the `.only`/`.skip`
22
+ * variants) across `tests/*.test.js` via static analysis (faster than
23
+ * running). node:test supports both `test()` and the BDD-style `it()`/
24
+ * `describe()` aliases as first-class declarations, and the suite mixes both,
25
+ * so counting only `test(` was blind to every `it(` test. `describe(` is NOT
26
+ * counted — those are containers, not tests, so counting them would inflate
27
+ * the total and conflate removing a grouping block with removing a test.
28
+ * Compare to a baseline pinned in `tests/.test-count-baseline.json`. Fail if
29
+ * the observed count drops MORE than the configured tolerance (default 1)
30
+ * below the baseline. Growth above baseline is fine; if the count grows by
31
+ * more than `update_baseline_when_growth_exceeds`, surface a notice that the
26
32
  * baseline file should be refreshed (operator commits the refresh as part
27
33
  * of the release that added the tests).
28
34
  *
@@ -71,7 +77,7 @@ function countTests(filePath) {
71
77
  // Drop a trailing line comment too (`test('x'); // disabled`).
72
78
  const stripped = noStrings.replace(/\/\/.*$/, '').trim();
73
79
  if (!stripped) continue;
74
- if (/(?<![A-Za-z0-9_$.])test(?:\.only|\.skip)?\s*\(/.test(stripped)) count++;
80
+ if (/(?<![A-Za-z0-9_$.])(?:test|it)(?:\.only|\.skip)?\s*\(/.test(stripped)) count++;
75
81
  }
76
82
  return count;
77
83
  }
@@ -165,14 +171,20 @@ function main() {
165
171
 
166
172
  if (status === 'shrunk_beyond_tolerance') {
167
173
  console.error(`[check-test-count] FAIL - test count dropped from ${baseline} to ${observed} (delta ${delta}, tolerance -${tolerance}).`);
168
- console.error('[check-test-count] Either a test file was accidentally removed, a test() invocation was deleted, OR the baseline is stale.');
174
+ console.error('[check-test-count] Either a test file was accidentally removed, a test()/it() invocation was deleted, OR the baseline is stale.');
169
175
  console.error('[check-test-count] If the drop is intentional, run: node scripts/check-test-count.js --update-baseline');
170
- process.exit(1);
176
+ // exitCode + return (not process.exit) so the buffered stdout write above
177
+ // (the --json result / one-line summary) drains before the event loop ends
178
+ // — process.exit() can truncate piped output.
179
+ process.exitCode = 1;
180
+ return;
171
181
  }
172
182
  if (status === 'grew_beyond_threshold_consider_bump') {
173
183
  console.error(`[check-test-count] NOTICE - test count grew by ${delta} (above the ${updateThreshold} notice threshold). Consider refreshing the baseline: node scripts/check-test-count.js --update-baseline`);
174
184
  }
175
- process.exit(0);
185
+ process.exitCode = 0;
176
186
  }
177
187
 
178
- main();
188
+ module.exports = { countTests, listTestFiles };
189
+
190
+ if (require.main === module) main();
@@ -0,0 +1,127 @@
1
+ #!/usr/bin/env node
2
+ "use strict";
3
+ /**
4
+ * check-test-subjects.js — bidirectional test↔subject gate (and reorg driver).
5
+ *
6
+ * Every test file must be named after a real SUBJECT the codebase actually has,
7
+ * and every subject must have a test. A "subject" is derived dynamically from
8
+ * the codebase so this list is never hand-maintained:
9
+ * - a source MODULE basename (lib/x.js -> x; lib/collectors/x.js -> x and
10
+ * collectors-x; orchestrator/index.js -> orchestrator; bin/exceptd.js -> cli)
11
+ * - an exported FUNCTION / CLASS name (kebab-cased) — per-function granularity
12
+ * - a data PRIMITIVE: a data/*.json catalog FILE, plus each catalog ENTRY that
13
+ * is itself a primitive — every CVE/MAL/GHSA id in data/cve-catalog.json and
14
+ * every playbook in data/playbooks/ — so one CVE == one test file
15
+ * - a .github/workflows/*.yml WORKFLOW (release -> release-workflow, etc.)
16
+ * - a CLI verb dispatched by bin/exceptd.js
17
+ *
18
+ * FORWARD violation : a tests/<x>.test.js where <x> is not a valid subject.
19
+ * REVERSE violation : a subject (module / CVE / playbook / workflow) with no
20
+ * tests/<subject>.test.js.
21
+ *
22
+ * Run with --worklist for the machine-readable reorg work list (JSON on stdout).
23
+ * Run with no flag for a human summary; exits non-zero while any violation
24
+ * remains (so once the suite conforms this becomes a standing predeploy gate).
25
+ */
26
+ const fs = require("node:fs");
27
+ const path = require("node:path");
28
+ const ROOT = path.resolve(__dirname, "..");
29
+
30
+ function camelKebab(s) { return s.replace(/([a-z0-9])([A-Z])/g, "$1-$2").replace(/_/g, "-").toLowerCase(); }
31
+ function read(p) { try { return fs.readFileSync(path.join(ROOT, p), "utf8"); } catch { return ""; } }
32
+ function ls(d) { try { return fs.readdirSync(path.join(ROOT, d), { withFileTypes: true }); } catch { return []; } }
33
+
34
+ function deriveSubjects() {
35
+ const subjects = new Map(); // name -> kind
36
+ const add = (s, kind) => { if (s && !subjects.has(s.toLowerCase())) subjects.set(s.toLowerCase(), kind); };
37
+
38
+ function walkSrc(d) {
39
+ for (const e of ls(d)) {
40
+ if (e.name === "node_modules") continue;
41
+ const rel = d + "/" + e.name;
42
+ if (e.isDirectory()) { walkSrc(rel); continue; }
43
+ if (!e.name.endsWith(".js")) continue;
44
+ const base = e.name.replace(/\.js$/, "");
45
+ // index.js is a directory entry point; the directory/canonical subject
46
+ // covers it, so treat the bare "index" basename as an alias, not a
47
+ // separately reverse-required module.
48
+ add(base, (base === "index" ? "alias:" : "module:") + rel);
49
+ const parent = path.basename(path.dirname(rel));
50
+ // parent-prefixed name (collectors-x, builders-x, validators-x) is an
51
+ // ALIAS of the canonical basename subject — a valid test target, but the
52
+ // canonical <base>.test.js already satisfies coverage, so don't double-
53
+ // count the alias as its own reverse gap.
54
+ if (!["lib", "scripts", "orchestrator", "bin"].includes(parent)) add(parent + "-" + base, "alias:" + rel);
55
+ const txt = read(rel);
56
+ for (const m of txt.matchAll(/(?:^|\n)\s*(?:async\s+)?(?:function|class)\s+([A-Za-z_$][\w$]*)/g)) add(camelKebab(m[1]), "fn:" + rel);
57
+ const exp = txt.match(/module\.exports\s*=\s*\{([\s\S]*?)\}/);
58
+ if (exp) for (const k of exp[1].matchAll(/([A-Za-z_$][\w$]*)\s*[,:}\n]/g)) add(camelKebab(k[1]), "fn:" + rel);
59
+ }
60
+ }
61
+ ["lib", "orchestrator", "scripts", "bin", "sources/validators"].forEach(walkSrc);
62
+ add("orchestrator", "module:orchestrator/index.js");
63
+ add("cli", "module:bin/exceptd.js");
64
+ // Vendored (pinned third-party) modules are valid test SUBJECTS but are not
65
+ // reverse-required — we don't force a dedicated test per vendored file.
66
+ (function walkVendor(d) { for (const e of ls(d)) { const rel = d + "/" + e.name; if (e.isDirectory()) walkVendor(rel); else if (e.name.endsWith(".js")) add(e.name.replace(/\.js$/, ""), "vendor:" + rel); } })("vendor");
67
+
68
+ // CLI verbs — both the switch-case form and the dispatch-table form
69
+ // (verb: () => path.join(...)) that bin/exceptd.js uses for most subcommands.
70
+ const cliSrc = read("bin/exceptd.js");
71
+ for (const m of cliSrc.matchAll(/case\s+['"]([a-z][a-z0-9-]+)['"]/g)) { add("cli-" + m[1], "cli-verb"); add(m[1], "cli-verb"); }
72
+ for (const m of cliSrc.matchAll(/^\s*["']?([a-z][a-z0-9-]+)["']?:\s*\(\)\s*=>/gm)) { add("cli-" + m[1], "cli-verb"); add(m[1], "cli-verb"); }
73
+
74
+ // data catalog files
75
+ for (const e of ls("data")) if (e.isFile() && e.name.endsWith(".json")) add(e.name.replace(/\.json$/, ""), "data");
76
+ // data ENTRY primitives: every CVE id + every playbook
77
+ try { const cat = JSON.parse(read("data/cve-catalog.json")); for (const k of Object.keys(cat)) if (k !== "_meta") add(k.toLowerCase(), "cve-primitive"); } catch {}
78
+ for (const e of ls("data/playbooks")) if (e.isFile() && e.name.endsWith(".json")) { const b = e.name.replace(/\.json$/, ""); add(b, "playbook-primitive"); add("playbook-" + b, "alias:playbook"); }
79
+ // workflows
80
+ for (const e of ls(".github/workflows")) if (/\.ya?ml$/.test(e.name)) { const b = e.name.replace(/\.ya?ml$/, ""); add(b, "workflow"); add(b + "-workflow", "workflow"); }
81
+
82
+ // Repo-artifact subjects: shipped root config/doc files, the docker build
83
+ // context, the agents/ directory, and aggregate catalog directories. A test
84
+ // that pins one of these artifacts (its content, counts, or cross-references)
85
+ // is named after a durable subject, not a release — so these are valid test
86
+ // targets. Kind is not module/cve/playbook, so they are NOT reverse-required
87
+ // (we don't force a dedicated test per doc file).
88
+ for (const f of ["package.json", "manifest.json", "manifest-snapshot.json", "README.md", "AGENTS.md", "SECURITY.md", "ARCHITECTURE.md", "CONTEXT.md", "CHANGELOG.md", "CONTRIBUTING.md", "CODE_OF_CONDUCT.md", "LICENSE", "NOTICE"]) {
89
+ add(f.replace(/\.[^.]*$/, "").toLowerCase().replace(/_/g, "-"), "repo:" + f);
90
+ }
91
+ add("agents-md", "repo:AGENTS.md");
92
+ add("docker", "repo:docker/test.Dockerfile");
93
+ add("agents", "repo:agents/");
94
+ add("playbooks", "aggregate:data/playbooks");
95
+ add("workflows", "aggregate:.github/workflows");
96
+ add("governance", "repo:governance-files"); // LICENSE/NOTICE/FUNDING/CoC/gitignore/gitleaks presence + integrity
97
+ return subjects;
98
+ }
99
+
100
+ function run() {
101
+ const subjects = deriveSubjects();
102
+ const testFiles = ls("tests").filter((e) => e.isFile() && e.name.endsWith(".test.js")).map((e) => e.name.replace(/\.test\.js$/, ""));
103
+ const testSet = new Set(testFiles.map((t) => t.toLowerCase()));
104
+
105
+ const suggest = (name) => {
106
+ const toks = name.toLowerCase().split("-");
107
+ for (let n = toks.length; n >= 1; n--) { const c = toks.slice(0, n).join("-"); if (subjects.has(c)) return c; }
108
+ return null;
109
+ };
110
+ const forward = [];
111
+ for (const t of testFiles) if (!subjects.has(t.toLowerCase())) forward.push({ file: "tests/" + t + ".test.js", suggested: suggest(t) });
112
+ const reverse = [];
113
+ for (const [s, kind] of subjects) if (!testSet.has(s)) reverse.push({ subject: s, kind });
114
+ return { subjects: subjects.size, forward, reverse };
115
+ }
116
+
117
+ if (require.main === module) {
118
+ const r = run();
119
+ if (process.argv.includes("--worklist")) { process.stdout.write(JSON.stringify(r) + "\n"); process.exitCode = (r.forward.length || r.reverse.length) ? 1 : 0; }
120
+ else {
121
+ const revMods = r.reverse.filter((x) => x.kind.startsWith("module:") || x.kind.startsWith("cve-primitive") || x.kind.startsWith("playbook-primitive"));
122
+ console.log(`[check-test-subjects] valid subjects=${r.subjects} | FORWARD violations=${r.forward.length} | REVERSE (module/cve/playbook) gaps=${revMods.length}`);
123
+ if (r.forward.length || revMods.length) { console.log("[check-test-subjects] FAIL — run with --worklist for the full list."); process.exitCode = 1; }
124
+ else console.log("[check-test-subjects] ok — every test maps to a subject and every subject has a test.");
125
+ }
126
+ }
127
+ module.exports = { deriveSubjects, run };
@@ -90,6 +90,13 @@ const COMMENT_EXEMPT = new Set([
90
90
  // literals are load-bearing data, not sprinkled release tags.
91
91
  "scripts/check-version-bump.js",
92
92
  "tests/version-bump-cadence.test.js",
93
+ // The version-tag gate's own regression test asserts the trailing-period /
94
+ // IPv4 / longer-run boundaries and the PHASE_RESIDUE_RES / FILENAME_VERSION_RE
95
+ // / countLineViolations exports, so it MUST embed literal stamps like
96
+ // `0.18.9.`, `0.18.99`, `Pre-0.13.22`, and `foo-v0_13_2.test.js` as the inputs
97
+ // under test — load-bearing data for the detector's boundary cases, not
98
+ // sprinkled release tags.
99
+ "tests/check-version-tags.test.js",
93
100
  ]);
94
101
 
95
102
  // Git-ignored files (a contributor's local-only working docs, scratch) are
@@ -115,11 +122,15 @@ function gitIgnoredSet(relPaths) {
115
122
  // Pattern: project version like `v0.13.22` or bare `0.13.22`. Matches
116
123
  // our pre-1.0 release range. External package versions like ATLAS
117
124
  // `v5.6.0` or CycloneDX `1.6` don't match because the major is 0.
118
- // The dot/digit lookarounds keep an IPv4 octet run (e.g. `127.0.0.1`, whose
119
- // `0.0.1` tail would otherwise count as a version tag) and any longer
120
- // dotted-numeric sequence from registering — a release version is never
121
- // embedded inside a larger digit.digit run.
122
- const VERSION_TAG_RE = /(?<![\d.])v?0\.\d+\.\d+(?![\d.])/;
125
+ // The trailing lookahead rejects a longer minor/patch digit (so `0.18.99`
126
+ // still matches, but the stamp can't be part of a wider number) and a
127
+ // dot-followed-by-digit (an IPv4 next octet / longer dotted-numeric run, e.g.
128
+ // `127.0.0.1`, whose `0.0.1` tail would otherwise register). A sentence-ending
129
+ // period after the patch (dot followed by non-digit / end-of-line, e.g.
130
+ // `// fixed in 0.18.9.`) is NOT excluded — that is exactly the version residue
131
+ // the gate must catch. The leading `(?<![\d.])` lookbehind keeps the IPv4
132
+ // suppression on the other side.
133
+ const VERSION_TAG_RE = /(?<![\d.])v?0\.\d+\.\d+(?!\d)(?!\.\d)/;
123
134
 
124
135
  // Phase residue patterns — broader than just version tags.
125
136
  const PHASE_RESIDUE_RES = [
@@ -314,5 +325,13 @@ function main() {
314
325
  process.exitCode = 1;
315
326
  }
316
327
 
328
+ module.exports = {
329
+ VERSION_TAG_RE,
330
+ PHASE_RESIDUE_RES,
331
+ FILENAME_VERSION_RE,
332
+ countLineViolations,
333
+ scanCurrent,
334
+ };
335
+
317
336
  if (require.main === module) main();
318
337
 
@@ -249,6 +249,19 @@ const GATES = [
249
249
  args: [path.join(ROOT, "scripts", "check-codebase-patterns.js")],
250
250
  ciJobName: "Data integrity (catalog + manifest snapshot)",
251
251
  },
252
+ {
253
+ // Test-subject coverage gate. Bidirectional: every tests/<x>.test.js must
254
+ // be named after a real SUBJECT the codebase has (a module / CLI verb /
255
+ // CVE id / playbook / data primitive / repo artifact), and every such
256
+ // subject must have a test. Blocks the naming drift that lets a test be
257
+ // filed under a version/finding label (where downstream readers can't find
258
+ // it) and surfaces any module/playbook that ships without a test. Derived
259
+ // dynamically from the source tree, so the list is never hand-maintained.
260
+ name: "Test-subject coverage (every test maps to a subject; every subject has a test)",
261
+ command: process.execPath,
262
+ args: [path.join(ROOT, "scripts", "check-test-subjects.js")],
263
+ ciJobName: "Data integrity (catalog + manifest snapshot)",
264
+ },
252
265
  {
253
266
  // Release-notes extract + quality gate. Runs the same `## <version>`
254
267
  // CHANGELOG extraction the release workflow publishes as the GitHub