@blamejs/exceptd-skills 0.18.9 → 0.18.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +24 -0
- package/bin/exceptd.js +197 -118
- package/data/_indexes/_meta.json +3 -3
- package/data/_indexes/frequency.json +2 -2
- package/data/d3fend-catalog.json +6 -6
- package/data/playbooks/identity-sso-compromise.json +2 -2
- package/data/playbooks/sbom.json +1 -1
- package/lib/citation-resolve.js +11 -0
- package/lib/collectors/containers.js +13 -0
- package/lib/cross-ref-api.js +29 -7
- package/lib/cve-regression-watcher.js +47 -15
- package/lib/framework-gap.js +27 -5
- package/lib/gap-detectors.js +8 -3
- package/lib/lint-skills.js +3 -2
- package/lib/playbook-runner.js +60 -5
- package/lib/refresh-external.js +58 -7
- package/lib/refresh-network.js +18 -5
- package/lib/rfc-cli.js +108 -18
- package/lib/schemas/playbook.schema.json +1 -1
- package/lib/scoring.js +31 -1
- package/lib/source-advisories.js +58 -9
- package/lib/ttp-mapper.js +31 -3
- package/lib/upstream-check-cli.js +13 -1
- package/lib/validate-catalog-meta.js +51 -7
- package/lib/validate-cve-catalog.js +10 -0
- package/lib/validate-playbooks.js +19 -1
- package/lib/xml-tokenizer.js +187 -25
- package/manifest.json +53 -53
- package/orchestrator/dispatcher.js +45 -9
- package/orchestrator/index.js +9 -7
- package/orchestrator/pipeline.js +62 -14
- package/orchestrator/scanner.js +40 -9
- package/package.json +1 -1
- package/sbom.cdx.json +103 -88
- package/scripts/build-indexes.js +21 -3
- package/scripts/builders/section-offsets.js +17 -8
- package/scripts/check-catalog-gap-budget.js +3 -3
- package/scripts/check-codebase-patterns.js +124 -11
- package/scripts/check-sbom-currency.js +69 -3
- package/scripts/check-test-count.js +28 -16
- package/scripts/check-test-subjects.js +127 -0
- package/scripts/check-version-tags.js +24 -5
- package/scripts/predeploy.js +13 -0
- package/scripts/refresh-upstream-catalogs.js +150 -42
- package/scripts/release.js +28 -11
|
@@ -68,13 +68,19 @@ function buildOne(absPath, relPath) {
|
|
|
68
68
|
const totalBytes = buf.length;
|
|
69
69
|
const text = buf.toString("utf8");
|
|
70
70
|
const lines = text.split(/\r?\n/);
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
71
|
+
// EOL-aware line-start byte offsets: walk the actual terminator bytes off the
|
|
72
|
+
// decoded text rather than assuming a fixed 1-byte newline. On a pure-LF body
|
|
73
|
+
// every terminator is 1 byte so the offsets are identical to the old `+ 1`
|
|
74
|
+
// accumulator; on a stray CRLF body the \r\n terminator is 2 bytes and the
|
|
75
|
+
// offsets stay correct (the old fixed-constant approach undercounted by 1
|
|
76
|
+
// byte per line and silently misaligned every token-budget slice). The split
|
|
77
|
+
// above discards terminator bytes, so the width can only be recovered by
|
|
78
|
+
// reading the terminators off the raw text — done here.
|
|
79
|
+
const lineByteOffsets = [0];
|
|
80
|
+
const eolRe = /\r?\n/g;
|
|
81
|
+
let m;
|
|
82
|
+
while ((m = eolRe.exec(text)) !== null) {
|
|
83
|
+
lineByteOffsets.push(Buffer.byteLength(text.slice(0, m.index + m[0].length), "utf8"));
|
|
78
84
|
}
|
|
79
85
|
|
|
80
86
|
// Frontmatter: lines between the first "---" and the second "---".
|
|
@@ -167,4 +173,7 @@ function buildSectionOffsets({ root, skills }) {
|
|
|
167
173
|
};
|
|
168
174
|
}
|
|
169
175
|
|
|
170
|
-
|
|
176
|
+
// buildOne is exported for regression testing of the EOL-aware byte offsets
|
|
177
|
+
// (a CRLF body must still produce byte_start values that point at the real
|
|
178
|
+
// "## " byte in the raw file).
|
|
179
|
+
module.exports = { buildSectionOffsets, buildOne };
|
|
@@ -116,12 +116,12 @@ function main() {
|
|
|
116
116
|
console.error("Add an explicit budget entry in both:");
|
|
117
117
|
console.error(" scripts/check-catalog-gap-budget.js");
|
|
118
118
|
console.error(" tests/shipped-catalog-integrity.test.js");
|
|
119
|
-
process.
|
|
119
|
+
process.exitCode = 1; return;
|
|
120
120
|
}
|
|
121
121
|
if (missingBudget.length > 0) {
|
|
122
122
|
console.error("\n[check-catalog-gap-budget] BUDGET missing entries for declared classes:");
|
|
123
123
|
for (const c of missingBudget) console.error(` ${c}: declared by lib/gap-detectors.js DETECTOR_CLASSES, no BUDGET entry`);
|
|
124
|
-
process.
|
|
124
|
+
process.exitCode = 1; return;
|
|
125
125
|
}
|
|
126
126
|
if (regressions.length > 0) {
|
|
127
127
|
console.error("\n[check-catalog-gap-budget] REGRESSION beyond budget:");
|
|
@@ -131,7 +131,7 @@ function main() {
|
|
|
131
131
|
console.error("\nClose the gap in this PR (preferred) or update BUDGET in both:");
|
|
132
132
|
console.error(" scripts/check-catalog-gap-budget.js");
|
|
133
133
|
console.error(" tests/shipped-catalog-integrity.test.js");
|
|
134
|
-
process.
|
|
134
|
+
process.exitCode = 1; return;
|
|
135
135
|
}
|
|
136
136
|
console.log("[check-catalog-gap-budget] all classes within budget; every class is budgeted.");
|
|
137
137
|
}
|
|
@@ -148,6 +148,79 @@ function filterMarkers(hits, cls) {
|
|
|
148
148
|
|
|
149
149
|
// ---- require.main block ranges -------------------------------------------
|
|
150
150
|
|
|
151
|
+
// Count `{` / `}` in `line` that are in REAL CODE context, advancing a
|
|
152
|
+
// stateful tokenizer that tracks string / template / comment regions across
|
|
153
|
+
// lines. Braces inside a single/double/template string, a `//` line comment,
|
|
154
|
+
// or a `/* */` block comment do NOT affect depth — otherwise a `{` or `}`
|
|
155
|
+
// typed inside a string literal in the require.main block miscounts the brace
|
|
156
|
+
// balance and the computed block range slides onto an unrelated later function
|
|
157
|
+
// (whose process.exit() is then wrongly treated as a CLI-entry exit and not
|
|
158
|
+
// flagged). `inTemplate` and `inBlockComment` are the cross-line states a
|
|
159
|
+
// per-line stripper cannot model, so the tokenizer state object is threaded
|
|
160
|
+
// line-to-line by the caller.
|
|
161
|
+
//
|
|
162
|
+
// `state` is mutated in place: { inSingle, inDouble, inTemplate, inBlock,
|
|
163
|
+
// templateDepth } — `templateDepth` tracks `${ … }` interpolation nesting so
|
|
164
|
+
// the closing `}` of an interpolation is treated as template punctuation, not
|
|
165
|
+
// a code brace, while braces INSIDE the interpolation expression still count.
|
|
166
|
+
function countCodeBraces(line, state) {
|
|
167
|
+
let delta = 0;
|
|
168
|
+
for (let i = 0; i < line.length; i++) {
|
|
169
|
+
const ch = line[i];
|
|
170
|
+
const next = line[i + 1];
|
|
171
|
+
if (state.inBlock) {
|
|
172
|
+
if (ch === "*" && next === "/") { state.inBlock = false; i++; }
|
|
173
|
+
continue;
|
|
174
|
+
}
|
|
175
|
+
if (state.inSingle) {
|
|
176
|
+
if (ch === "\\") { i++; continue; }
|
|
177
|
+
if (ch === "'") state.inSingle = false;
|
|
178
|
+
continue;
|
|
179
|
+
}
|
|
180
|
+
if (state.inDouble) {
|
|
181
|
+
if (ch === "\\") { i++; continue; }
|
|
182
|
+
if (ch === '"') state.inDouble = false;
|
|
183
|
+
continue;
|
|
184
|
+
}
|
|
185
|
+
if (state.inTemplate) {
|
|
186
|
+
if (ch === "\\") { i++; continue; }
|
|
187
|
+
if (ch === "`") { state.inTemplate = false; continue; }
|
|
188
|
+
if (ch === "$" && next === "{") {
|
|
189
|
+
// Enter an interpolation expression: braces inside ARE code.
|
|
190
|
+
state.templateExpr.push(0);
|
|
191
|
+
state.inTemplate = false;
|
|
192
|
+
i++; // skip the `{`; the `${` opener is template punctuation
|
|
193
|
+
continue;
|
|
194
|
+
}
|
|
195
|
+
continue;
|
|
196
|
+
}
|
|
197
|
+
// Code context (possibly inside a template interpolation expression).
|
|
198
|
+
if (ch === "/" && next === "/") break; // `//` — rest of the line is a comment
|
|
199
|
+
if (ch === "/" && next === "*") { state.inBlock = true; i++; continue; }
|
|
200
|
+
if (ch === "'") { state.inSingle = true; continue; }
|
|
201
|
+
if (ch === '"') { state.inDouble = true; continue; }
|
|
202
|
+
if (ch === "`") { state.inTemplate = true; continue; }
|
|
203
|
+
if (ch === "{") {
|
|
204
|
+
if (state.templateExpr.length) state.templateExpr[state.templateExpr.length - 1]++;
|
|
205
|
+
delta++;
|
|
206
|
+
} else if (ch === "}") {
|
|
207
|
+
if (state.templateExpr.length && state.templateExpr[state.templateExpr.length - 1] === 0) {
|
|
208
|
+
// Closes the `${ … }` interpolation — back to template body.
|
|
209
|
+
state.templateExpr.pop();
|
|
210
|
+
state.inTemplate = true;
|
|
211
|
+
} else {
|
|
212
|
+
if (state.templateExpr.length) state.templateExpr[state.templateExpr.length - 1]--;
|
|
213
|
+
delta--;
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
return delta;
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
function newBraceState() {
|
|
221
|
+
return { inSingle: false, inDouble: false, inTemplate: false, inBlock: false, templateExpr: [] };
|
|
222
|
+
}
|
|
223
|
+
|
|
151
224
|
// Line ranges (1-based, inclusive) of `if (require.main === module) { ... }`
|
|
152
225
|
// blocks — the dual-mode CLI-entry section where synchronous-print-then-exit
|
|
153
226
|
// is correct. process.exit there is owned by tests/safe-exit-grep.test.js and
|
|
@@ -156,15 +229,16 @@ function requireMainRanges(lines) {
|
|
|
156
229
|
const ranges = [];
|
|
157
230
|
for (let i = 0; i < lines.length; i++) {
|
|
158
231
|
if (/\brequire\.main\s*===\s*module\b/.test(lines[i])) {
|
|
159
|
-
// Find the opening brace (same line or next few), then balance
|
|
232
|
+
// Find the opening brace (same line or next few), then balance —
|
|
233
|
+
// string/comment/template-aware so braces inside literals don't skew the
|
|
234
|
+
// depth (see countCodeBraces).
|
|
160
235
|
let depth = 0;
|
|
161
236
|
let started = false;
|
|
162
237
|
let j = i;
|
|
238
|
+
const state = newBraceState();
|
|
163
239
|
for (; j < lines.length; j++) {
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
else if (ch === "}") { depth--; }
|
|
167
|
-
}
|
|
240
|
+
depth += countCodeBraces(lines[j], state);
|
|
241
|
+
if (depth > 0) started = true;
|
|
168
242
|
if (started && depth <= 0) break;
|
|
169
243
|
}
|
|
170
244
|
if (started) ranges.push([i + 1, j + 1]);
|
|
@@ -181,8 +255,14 @@ function inRanges(ranges, lineNo) {
|
|
|
181
255
|
|
|
182
256
|
// A line that opens a new function body (so a backward stdout-write scan stops
|
|
183
257
|
// at the enclosing function and doesn't arm an exit from an unrelated earlier
|
|
184
|
-
// function).
|
|
185
|
-
|
|
258
|
+
// function). The bare-identifier (third) alternative matches a declaration /
|
|
259
|
+
// method-shorthand opener (`foo() {`, `async bar() {`), but it must REFUSE
|
|
260
|
+
// control-flow openers (`for (…) {`, `if (…) {`, `while/switch/catch (…) {`):
|
|
261
|
+
// a control-flow block sitting between a stdout write and a process.exit() is
|
|
262
|
+
// inside the SAME function, so stopping the backward scan there would wrongly
|
|
263
|
+
// leave the exit unflagged. The negative lookahead excludes the control-flow
|
|
264
|
+
// keywords; `function` and arrow alternatives are unchanged.
|
|
265
|
+
const FUNCTION_START = /(^|[^.\w])function\b|=>\s*\{?\s*$|^\s*(async\s+)?(?!(?:if|for|while|switch|catch|do|else|with|finally|return)\b)[A-Za-z_$][\w$]*\s*\([^)]*\)\s*\{/;
|
|
186
266
|
|
|
187
267
|
function detectProcessExitAfterStdout(files) {
|
|
188
268
|
const hits = [];
|
|
@@ -210,6 +290,15 @@ function detectProcessExitAfterStdout(files) {
|
|
|
210
290
|
return filterMarkers(hits, "process-exit-after-stdout-write");
|
|
211
291
|
}
|
|
212
292
|
|
|
293
|
+
// The first non-whitespace char of the first arg is `"`, `'`, or `/` => a
|
|
294
|
+
// string/regex literal => static, safe. Anything else (an identifier, a `(`,
|
|
295
|
+
// a backtick template) is operator-derivable and flagged. Backtick is NOT
|
|
296
|
+
// exempt — a template literal can interpolate operator input, so it must be
|
|
297
|
+
// flagged the same as a bare identifier.
|
|
298
|
+
function isStaticRegexFirstChar(ch) {
|
|
299
|
+
return ch === '"' || ch === "'" || ch === "/";
|
|
300
|
+
}
|
|
301
|
+
|
|
213
302
|
function detectDynamicRegex(files) {
|
|
214
303
|
const hits = [];
|
|
215
304
|
for (const rel of (files || filesUnder(["lib", "orchestrator", "bin/exceptd.js"]))) {
|
|
@@ -217,10 +306,29 @@ function detectDynamicRegex(files) {
|
|
|
217
306
|
for (let i = 0; i < lines.length; i++) {
|
|
218
307
|
const code = stripLineComment(lines[i]);
|
|
219
308
|
const m = code.match(/\bnew RegExp\s*\(\s*(.)/);
|
|
220
|
-
if (
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
309
|
+
if (m) {
|
|
310
|
+
if (isStaticRegexFirstChar(m[1])) continue;
|
|
311
|
+
hits.push({ file: rel, line: i + 1, content: lines[i].trim() });
|
|
312
|
+
continue;
|
|
313
|
+
}
|
|
314
|
+
// Multi-line form: `new RegExp(` ends the (comment-stripped) line with the
|
|
315
|
+
// open paren as the last token, and the pattern arg is on a following
|
|
316
|
+
// line. The single-line match above can't see the first-arg char, so it
|
|
317
|
+
// would silently pass a dynamic RegExp whose argument starts next line.
|
|
318
|
+
// Look ahead, skipping blank and comment-only lines (capped at 5), and
|
|
319
|
+
// inspect the first code line's first non-whitespace char.
|
|
320
|
+
if (!/\bnew RegExp\s*\(\s*$/.test(code)) continue;
|
|
321
|
+
let firstChar = null;
|
|
322
|
+
for (let k = i + 1; k <= i + 5 && k < lines.length; k++) {
|
|
323
|
+
const ahead = stripLineComment(lines[k]).replace(/^\s+/, "");
|
|
324
|
+
if (ahead === "") continue; // blank or comment-only — skip
|
|
325
|
+
firstChar = ahead[0];
|
|
326
|
+
break;
|
|
327
|
+
}
|
|
328
|
+
// A `new RegExp(` with nothing parseable after it within the cap is
|
|
329
|
+
// suspicious — flag conservatively. Otherwise apply the SAME literal
|
|
330
|
+
// exemption as the single-line path.
|
|
331
|
+
if (firstChar !== null && isStaticRegexFirstChar(firstChar)) continue;
|
|
224
332
|
hits.push({ file: rel, line: i + 1, content: lines[i].trim() });
|
|
225
333
|
}
|
|
226
334
|
}
|
|
@@ -453,6 +561,11 @@ module.exports = {
|
|
|
453
561
|
detectMisalignedMarkedRun,
|
|
454
562
|
scanUnsortedMarkedArray,
|
|
455
563
|
scanMisalignedMarkedRun,
|
|
564
|
+
requireMainRanges,
|
|
565
|
+
countCodeBraces,
|
|
566
|
+
newBraceState,
|
|
567
|
+
isStaticRegexFirstChar,
|
|
568
|
+
FUNCTION_START,
|
|
456
569
|
filesUnder,
|
|
457
570
|
};
|
|
458
571
|
|
|
@@ -50,6 +50,66 @@ function catalogEntryCount(dataDir, file) {
|
|
|
50
50
|
return 0;
|
|
51
51
|
}
|
|
52
52
|
|
|
53
|
+
// Files/prefixes that refresh-sbom's expandAllowlist excludes from the shipped
|
|
54
|
+
// file: component inventory. Kept in sync with scripts/refresh-sbom.js
|
|
55
|
+
// (SELF_EXCLUDED + DERIVABLE_PREFIXES): the SBOM never hashes itself, and the
|
|
56
|
+
// pre-computed index cache under data/_indexes/ is regenerated/test-mutated, so
|
|
57
|
+
// it carries no per-file component. The completeness check below must apply the
|
|
58
|
+
// SAME exclusions or it would demand a component for a file the generator never
|
|
59
|
+
// emits one for.
|
|
60
|
+
const SBOM_SELF_EXCLUDED = new Set(["sbom.cdx.json"]);
|
|
61
|
+
const SBOM_DERIVABLE_PREFIXES = ["data/_indexes/"];
|
|
62
|
+
|
|
63
|
+
function sbomIsDerivable(rel) {
|
|
64
|
+
return SBOM_DERIVABLE_PREFIXES.some(
|
|
65
|
+
(p) => rel === p.replace(/\/$/, "") || rel.startsWith(p)
|
|
66
|
+
);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
// Recursively list every regular file under absDir, returned as absolute paths.
|
|
70
|
+
// Mirrors refresh-sbom's walkFiles (which is root-agnostic — it walks whatever
|
|
71
|
+
// absolute dir it is given).
|
|
72
|
+
function walkFilesAbs(absDir) {
|
|
73
|
+
const out = [];
|
|
74
|
+
let entries;
|
|
75
|
+
try { entries = fs.readdirSync(absDir, { withFileTypes: true }); }
|
|
76
|
+
catch { return out; }
|
|
77
|
+
for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) {
|
|
78
|
+
const abs = path.join(absDir, entry.name);
|
|
79
|
+
if (entry.isDirectory()) out.push(...walkFilesAbs(abs));
|
|
80
|
+
else if (entry.isFile()) out.push(abs);
|
|
81
|
+
}
|
|
82
|
+
return out;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
// Root-aware allowlist expansion: the shipped-file set that must each have a
|
|
86
|
+
// file: component, computed against the TARGET tree (`root`), not the running
|
|
87
|
+
// script's source repo. refresh-sbom's exported expandAllowlist is bound to its
|
|
88
|
+
// own REPO_ROOT (it joins/relativizes against __dirname/..), so under a `--root`
|
|
89
|
+
// target the completeness check would otherwise validate the SOURCE repo's file
|
|
90
|
+
// list against the TARGET SBOM — the wrong tree. Replicating the expansion here
|
|
91
|
+
// (same SELF_EXCLUDED + DERIVABLE_PREFIXES exclusions, same dedupe+sort) keeps
|
|
92
|
+
// the gate honest under `--root` without reaching across into the generator's
|
|
93
|
+
// module-level root.
|
|
94
|
+
function expandAllowlistAt(allowlist, root) {
|
|
95
|
+
const abs = [];
|
|
96
|
+
for (const entry of allowlist) {
|
|
97
|
+
const full = path.join(root, entry);
|
|
98
|
+
let stat;
|
|
99
|
+
try { stat = fs.statSync(full); }
|
|
100
|
+
catch { continue; } // tolerate a stale entry; presence checks elsewhere flag
|
|
101
|
+
if (stat.isDirectory()) abs.push(...walkFilesAbs(full));
|
|
102
|
+
else if (stat.isFile()) abs.push(full);
|
|
103
|
+
}
|
|
104
|
+
const rel = Array.from(
|
|
105
|
+
new Set(abs.map((a) => path.relative(root, a).split(path.sep).join("/")))
|
|
106
|
+
)
|
|
107
|
+
.filter((r) => !SBOM_SELF_EXCLUDED.has(r))
|
|
108
|
+
.filter((r) => !sbomIsDerivable(r))
|
|
109
|
+
.sort();
|
|
110
|
+
return rel;
|
|
111
|
+
}
|
|
112
|
+
|
|
53
113
|
// The description string embeds per-catalog ENTRY counts as free text, e.g.
|
|
54
114
|
// "11 catalogs (439 CVEs / 177 CWEs / 805 ATT&CK + ICS / 170 ATLAS /
|
|
55
115
|
// 468 D3FEND / 8888 RFCs)". Each token maps to one data/*.json catalog whose
|
|
@@ -332,9 +392,15 @@ function checkSbomCurrency(root) {
|
|
|
332
392
|
// refresh-sbom's exact allowlist expansion + digest so the gate can't drift
|
|
333
393
|
// from the generator.
|
|
334
394
|
try {
|
|
335
|
-
|
|
395
|
+
// bundleDigest operates purely on the file: component objects (no tree
|
|
396
|
+
// walk), so it is root-agnostic and safe to reuse from the generator. The
|
|
397
|
+
// allowlist expansion, by contrast, MUST run against the target `root`
|
|
398
|
+
// (expandAllowlistAt below) — refresh-sbom's exported expandAllowlist is
|
|
399
|
+
// pinned to its own source-repo REPO_ROOT and would validate the wrong tree
|
|
400
|
+
// under `--root`.
|
|
401
|
+
const { bundleDigest } = require("./refresh-sbom");
|
|
336
402
|
const pkg = JSON.parse(fs.readFileSync(path.join(root, "package.json"), "utf8"));
|
|
337
|
-
const expected =
|
|
403
|
+
const expected = expandAllowlistAt(pkg.files || [], root);
|
|
338
404
|
const fileComps = components.filter(
|
|
339
405
|
(c) => typeof c["bom-ref"] === "string" && c["bom-ref"].startsWith("file:")
|
|
340
406
|
);
|
|
@@ -402,6 +468,6 @@ function main() {
|
|
|
402
468
|
);
|
|
403
469
|
}
|
|
404
470
|
|
|
405
|
-
module.exports = { checkSbomCurrency, resolveRoot, DESCRIPTION_ENTRY_TOKENS, catalogEntryCount };
|
|
471
|
+
module.exports = { checkSbomCurrency, resolveRoot, DESCRIPTION_ENTRY_TOKENS, catalogEntryCount, expandAllowlistAt };
|
|
406
472
|
|
|
407
473
|
if (require.main === module) main();
|
|
@@ -11,18 +11,24 @@
|
|
|
11
11
|
* catches test-set shrinkage.
|
|
12
12
|
*
|
|
13
13
|
* Scope + blind spot. This counts test DECLARATIONS, so it detects deleted
|
|
14
|
-
* files / removed `test(` calls / glob-exclusions. It does NOT detect a
|
|
15
|
-
* neutered in place: `test('name', { skip: true }, fn)
|
|
16
|
-
* still count as one declaration, so flipping a running
|
|
17
|
-
* skipped leaves the count unchanged. Guarding against
|
|
18
|
-
* need runnable-vs-skipped tracking; that is out of scope
|
|
14
|
+
* files / removed `test(`/`it(` calls / glob-exclusions. It does NOT detect a
|
|
15
|
+
* test neutered in place: `test('name', { skip: true }, fn)`, `test.skip(`,
|
|
16
|
+
* and `it.skip(` all still count as one declaration, so flipping a running
|
|
17
|
+
* test to permanently skipped leaves the count unchanged. Guarding against
|
|
18
|
+
* skip-in-place would need runnable-vs-skipped tracking; that is out of scope
|
|
19
|
+
* for this gate.
|
|
19
20
|
*
|
|
20
|
-
* Mechanism: count `test(
|
|
21
|
-
* across `tests/*.test.js` via static analysis (faster than
|
|
22
|
-
*
|
|
23
|
-
*
|
|
24
|
-
*
|
|
25
|
-
*
|
|
21
|
+
* Mechanism: count `test(`/`it(` declarations (including the `.only`/`.skip`
|
|
22
|
+
* variants) across `tests/*.test.js` via static analysis (faster than
|
|
23
|
+
* running). node:test supports both `test()` and the BDD-style `it()`/
|
|
24
|
+
* `describe()` aliases as first-class declarations, and the suite mixes both,
|
|
25
|
+
* so counting only `test(` was blind to every `it(` test. `describe(` is NOT
|
|
26
|
+
* counted — those are containers, not tests, so counting them would inflate
|
|
27
|
+
* the total and conflate removing a grouping block with removing a test.
|
|
28
|
+
* Compare to a baseline pinned in `tests/.test-count-baseline.json`. Fail if
|
|
29
|
+
* the observed count drops MORE than the configured tolerance (default 1)
|
|
30
|
+
* below the baseline. Growth above baseline is fine; if the count grows by
|
|
31
|
+
* more than `update_baseline_when_growth_exceeds`, surface a notice that the
|
|
26
32
|
* baseline file should be refreshed (operator commits the refresh as part
|
|
27
33
|
* of the release that added the tests).
|
|
28
34
|
*
|
|
@@ -71,7 +77,7 @@ function countTests(filePath) {
|
|
|
71
77
|
// Drop a trailing line comment too (`test('x'); // disabled`).
|
|
72
78
|
const stripped = noStrings.replace(/\/\/.*$/, '').trim();
|
|
73
79
|
if (!stripped) continue;
|
|
74
|
-
if (/(?<![A-Za-z0-9_$.])test(?:\.only|\.skip)?\s*\(/.test(stripped)) count++;
|
|
80
|
+
if (/(?<![A-Za-z0-9_$.])(?:test|it)(?:\.only|\.skip)?\s*\(/.test(stripped)) count++;
|
|
75
81
|
}
|
|
76
82
|
return count;
|
|
77
83
|
}
|
|
@@ -165,14 +171,20 @@ function main() {
|
|
|
165
171
|
|
|
166
172
|
if (status === 'shrunk_beyond_tolerance') {
|
|
167
173
|
console.error(`[check-test-count] FAIL - test count dropped from ${baseline} to ${observed} (delta ${delta}, tolerance -${tolerance}).`);
|
|
168
|
-
console.error('[check-test-count] Either a test file was accidentally removed, a test() invocation was deleted, OR the baseline is stale.');
|
|
174
|
+
console.error('[check-test-count] Either a test file was accidentally removed, a test()/it() invocation was deleted, OR the baseline is stale.');
|
|
169
175
|
console.error('[check-test-count] If the drop is intentional, run: node scripts/check-test-count.js --update-baseline');
|
|
170
|
-
process.exit
|
|
176
|
+
// exitCode + return (not process.exit) so the buffered stdout write above
|
|
177
|
+
// (the --json result / one-line summary) drains before the event loop ends
|
|
178
|
+
// — process.exit() can truncate piped output.
|
|
179
|
+
process.exitCode = 1;
|
|
180
|
+
return;
|
|
171
181
|
}
|
|
172
182
|
if (status === 'grew_beyond_threshold_consider_bump') {
|
|
173
183
|
console.error(`[check-test-count] NOTICE - test count grew by ${delta} (above the ${updateThreshold} notice threshold). Consider refreshing the baseline: node scripts/check-test-count.js --update-baseline`);
|
|
174
184
|
}
|
|
175
|
-
process.
|
|
185
|
+
process.exitCode = 0;
|
|
176
186
|
}
|
|
177
187
|
|
|
178
|
-
|
|
188
|
+
module.exports = { countTests, listTestFiles };
|
|
189
|
+
|
|
190
|
+
if (require.main === module) main();
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
"use strict";
|
|
3
|
+
/**
|
|
4
|
+
* check-test-subjects.js — bidirectional test↔subject gate (and reorg driver).
|
|
5
|
+
*
|
|
6
|
+
* Every test file must be named after a real SUBJECT the codebase actually has,
|
|
7
|
+
* and every subject must have a test. A "subject" is derived dynamically from
|
|
8
|
+
* the codebase so this list is never hand-maintained:
|
|
9
|
+
* - a source MODULE basename (lib/x.js -> x; lib/collectors/x.js -> x and
|
|
10
|
+
* collectors-x; orchestrator/index.js -> orchestrator; bin/exceptd.js -> cli)
|
|
11
|
+
* - an exported FUNCTION / CLASS name (kebab-cased) — per-function granularity
|
|
12
|
+
* - a data PRIMITIVE: a data/*.json catalog FILE, plus each catalog ENTRY that
|
|
13
|
+
* is itself a primitive — every CVE/MAL/GHSA id in data/cve-catalog.json and
|
|
14
|
+
* every playbook in data/playbooks/ — so one CVE == one test file
|
|
15
|
+
* - a .github/workflows/*.yml WORKFLOW (release -> release-workflow, etc.)
|
|
16
|
+
* - a CLI verb dispatched by bin/exceptd.js
|
|
17
|
+
*
|
|
18
|
+
* FORWARD violation : a tests/<x>.test.js where <x> is not a valid subject.
|
|
19
|
+
* REVERSE violation : a subject (module / CVE / playbook / workflow) with no
|
|
20
|
+
* tests/<subject>.test.js.
|
|
21
|
+
*
|
|
22
|
+
* Run with --worklist for the machine-readable reorg work list (JSON on stdout).
|
|
23
|
+
* Run with no flag for a human summary; exits non-zero while any violation
|
|
24
|
+
* remains (so once the suite conforms this becomes a standing predeploy gate).
|
|
25
|
+
*/
|
|
26
|
+
const fs = require("node:fs");
|
|
27
|
+
const path = require("node:path");
|
|
28
|
+
const ROOT = path.resolve(__dirname, "..");
|
|
29
|
+
|
|
30
|
+
function camelKebab(s) { return s.replace(/([a-z0-9])([A-Z])/g, "$1-$2").replace(/_/g, "-").toLowerCase(); }
|
|
31
|
+
function read(p) { try { return fs.readFileSync(path.join(ROOT, p), "utf8"); } catch { return ""; } }
|
|
32
|
+
function ls(d) { try { return fs.readdirSync(path.join(ROOT, d), { withFileTypes: true }); } catch { return []; } }
|
|
33
|
+
|
|
34
|
+
function deriveSubjects() {
|
|
35
|
+
const subjects = new Map(); // name -> kind
|
|
36
|
+
const add = (s, kind) => { if (s && !subjects.has(s.toLowerCase())) subjects.set(s.toLowerCase(), kind); };
|
|
37
|
+
|
|
38
|
+
function walkSrc(d) {
|
|
39
|
+
for (const e of ls(d)) {
|
|
40
|
+
if (e.name === "node_modules") continue;
|
|
41
|
+
const rel = d + "/" + e.name;
|
|
42
|
+
if (e.isDirectory()) { walkSrc(rel); continue; }
|
|
43
|
+
if (!e.name.endsWith(".js")) continue;
|
|
44
|
+
const base = e.name.replace(/\.js$/, "");
|
|
45
|
+
// index.js is a directory entry point; the directory/canonical subject
|
|
46
|
+
// covers it, so treat the bare "index" basename as an alias, not a
|
|
47
|
+
// separately reverse-required module.
|
|
48
|
+
add(base, (base === "index" ? "alias:" : "module:") + rel);
|
|
49
|
+
const parent = path.basename(path.dirname(rel));
|
|
50
|
+
// parent-prefixed name (collectors-x, builders-x, validators-x) is an
|
|
51
|
+
// ALIAS of the canonical basename subject — a valid test target, but the
|
|
52
|
+
// canonical <base>.test.js already satisfies coverage, so don't double-
|
|
53
|
+
// count the alias as its own reverse gap.
|
|
54
|
+
if (!["lib", "scripts", "orchestrator", "bin"].includes(parent)) add(parent + "-" + base, "alias:" + rel);
|
|
55
|
+
const txt = read(rel);
|
|
56
|
+
for (const m of txt.matchAll(/(?:^|\n)\s*(?:async\s+)?(?:function|class)\s+([A-Za-z_$][\w$]*)/g)) add(camelKebab(m[1]), "fn:" + rel);
|
|
57
|
+
const exp = txt.match(/module\.exports\s*=\s*\{([\s\S]*?)\}/);
|
|
58
|
+
if (exp) for (const k of exp[1].matchAll(/([A-Za-z_$][\w$]*)\s*[,:}\n]/g)) add(camelKebab(k[1]), "fn:" + rel);
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
["lib", "orchestrator", "scripts", "bin", "sources/validators"].forEach(walkSrc);
|
|
62
|
+
add("orchestrator", "module:orchestrator/index.js");
|
|
63
|
+
add("cli", "module:bin/exceptd.js");
|
|
64
|
+
// Vendored (pinned third-party) modules are valid test SUBJECTS but are not
|
|
65
|
+
// reverse-required — we don't force a dedicated test per vendored file.
|
|
66
|
+
(function walkVendor(d) { for (const e of ls(d)) { const rel = d + "/" + e.name; if (e.isDirectory()) walkVendor(rel); else if (e.name.endsWith(".js")) add(e.name.replace(/\.js$/, ""), "vendor:" + rel); } })("vendor");
|
|
67
|
+
|
|
68
|
+
// CLI verbs — both the switch-case form and the dispatch-table form
|
|
69
|
+
// (verb: () => path.join(...)) that bin/exceptd.js uses for most subcommands.
|
|
70
|
+
const cliSrc = read("bin/exceptd.js");
|
|
71
|
+
for (const m of cliSrc.matchAll(/case\s+['"]([a-z][a-z0-9-]+)['"]/g)) { add("cli-" + m[1], "cli-verb"); add(m[1], "cli-verb"); }
|
|
72
|
+
for (const m of cliSrc.matchAll(/^\s*["']?([a-z][a-z0-9-]+)["']?:\s*\(\)\s*=>/gm)) { add("cli-" + m[1], "cli-verb"); add(m[1], "cli-verb"); }
|
|
73
|
+
|
|
74
|
+
// data catalog files
|
|
75
|
+
for (const e of ls("data")) if (e.isFile() && e.name.endsWith(".json")) add(e.name.replace(/\.json$/, ""), "data");
|
|
76
|
+
// data ENTRY primitives: every CVE id + every playbook
|
|
77
|
+
try { const cat = JSON.parse(read("data/cve-catalog.json")); for (const k of Object.keys(cat)) if (k !== "_meta") add(k.toLowerCase(), "cve-primitive"); } catch {}
|
|
78
|
+
for (const e of ls("data/playbooks")) if (e.isFile() && e.name.endsWith(".json")) { const b = e.name.replace(/\.json$/, ""); add(b, "playbook-primitive"); add("playbook-" + b, "alias:playbook"); }
|
|
79
|
+
// workflows
|
|
80
|
+
for (const e of ls(".github/workflows")) if (/\.ya?ml$/.test(e.name)) { const b = e.name.replace(/\.ya?ml$/, ""); add(b, "workflow"); add(b + "-workflow", "workflow"); }
|
|
81
|
+
|
|
82
|
+
// Repo-artifact subjects: shipped root config/doc files, the docker build
|
|
83
|
+
// context, the agents/ directory, and aggregate catalog directories. A test
|
|
84
|
+
// that pins one of these artifacts (its content, counts, or cross-references)
|
|
85
|
+
// is named after a durable subject, not a release — so these are valid test
|
|
86
|
+
// targets. Kind is not module/cve/playbook, so they are NOT reverse-required
|
|
87
|
+
// (we don't force a dedicated test per doc file).
|
|
88
|
+
for (const f of ["package.json", "manifest.json", "manifest-snapshot.json", "README.md", "AGENTS.md", "SECURITY.md", "ARCHITECTURE.md", "CONTEXT.md", "CHANGELOG.md", "CONTRIBUTING.md", "CODE_OF_CONDUCT.md", "LICENSE", "NOTICE"]) {
|
|
89
|
+
add(f.replace(/\.[^.]*$/, "").toLowerCase().replace(/_/g, "-"), "repo:" + f);
|
|
90
|
+
}
|
|
91
|
+
add("agents-md", "repo:AGENTS.md");
|
|
92
|
+
add("docker", "repo:docker/test.Dockerfile");
|
|
93
|
+
add("agents", "repo:agents/");
|
|
94
|
+
add("playbooks", "aggregate:data/playbooks");
|
|
95
|
+
add("workflows", "aggregate:.github/workflows");
|
|
96
|
+
add("governance", "repo:governance-files"); // LICENSE/NOTICE/FUNDING/CoC/gitignore/gitleaks presence + integrity
|
|
97
|
+
return subjects;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
function run() {
|
|
101
|
+
const subjects = deriveSubjects();
|
|
102
|
+
const testFiles = ls("tests").filter((e) => e.isFile() && e.name.endsWith(".test.js")).map((e) => e.name.replace(/\.test\.js$/, ""));
|
|
103
|
+
const testSet = new Set(testFiles.map((t) => t.toLowerCase()));
|
|
104
|
+
|
|
105
|
+
const suggest = (name) => {
|
|
106
|
+
const toks = name.toLowerCase().split("-");
|
|
107
|
+
for (let n = toks.length; n >= 1; n--) { const c = toks.slice(0, n).join("-"); if (subjects.has(c)) return c; }
|
|
108
|
+
return null;
|
|
109
|
+
};
|
|
110
|
+
const forward = [];
|
|
111
|
+
for (const t of testFiles) if (!subjects.has(t.toLowerCase())) forward.push({ file: "tests/" + t + ".test.js", suggested: suggest(t) });
|
|
112
|
+
const reverse = [];
|
|
113
|
+
for (const [s, kind] of subjects) if (!testSet.has(s)) reverse.push({ subject: s, kind });
|
|
114
|
+
return { subjects: subjects.size, forward, reverse };
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
if (require.main === module) {
|
|
118
|
+
const r = run();
|
|
119
|
+
if (process.argv.includes("--worklist")) { process.stdout.write(JSON.stringify(r) + "\n"); process.exitCode = (r.forward.length || r.reverse.length) ? 1 : 0; }
|
|
120
|
+
else {
|
|
121
|
+
const revMods = r.reverse.filter((x) => x.kind.startsWith("module:") || x.kind.startsWith("cve-primitive") || x.kind.startsWith("playbook-primitive"));
|
|
122
|
+
console.log(`[check-test-subjects] valid subjects=${r.subjects} | FORWARD violations=${r.forward.length} | REVERSE (module/cve/playbook) gaps=${revMods.length}`);
|
|
123
|
+
if (r.forward.length || revMods.length) { console.log("[check-test-subjects] FAIL — run with --worklist for the full list."); process.exitCode = 1; }
|
|
124
|
+
else console.log("[check-test-subjects] ok — every test maps to a subject and every subject has a test.");
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
module.exports = { deriveSubjects, run };
|
|
@@ -90,6 +90,13 @@ const COMMENT_EXEMPT = new Set([
|
|
|
90
90
|
// literals are load-bearing data, not sprinkled release tags.
|
|
91
91
|
"scripts/check-version-bump.js",
|
|
92
92
|
"tests/version-bump-cadence.test.js",
|
|
93
|
+
// The version-tag gate's own regression test asserts the trailing-period /
|
|
94
|
+
// IPv4 / longer-run boundaries and the PHASE_RESIDUE_RES / FILENAME_VERSION_RE
|
|
95
|
+
// / countLineViolations exports, so it MUST embed literal stamps like
|
|
96
|
+
// `0.18.9.`, `0.18.99`, `Pre-0.13.22`, and `foo-v0_13_2.test.js` as the inputs
|
|
97
|
+
// under test — load-bearing data for the detector's boundary cases, not
|
|
98
|
+
// sprinkled release tags.
|
|
99
|
+
"tests/check-version-tags.test.js",
|
|
93
100
|
]);
|
|
94
101
|
|
|
95
102
|
// Git-ignored files (a contributor's local-only working docs, scratch) are
|
|
@@ -115,11 +122,15 @@ function gitIgnoredSet(relPaths) {
|
|
|
115
122
|
// Pattern: project version like `v0.13.22` or bare `0.13.22`. Matches
|
|
116
123
|
// our pre-1.0 release range. External package versions like ATLAS
|
|
117
124
|
// `v5.6.0` or CycloneDX `1.6` don't match because the major is 0.
|
|
118
|
-
// The
|
|
119
|
-
//
|
|
120
|
-
//
|
|
121
|
-
//
|
|
122
|
-
|
|
125
|
+
// The trailing lookahead rejects a longer minor/patch digit (so `0.18.99`
|
|
126
|
+
// still matches, but the stamp can't be part of a wider number) and a
|
|
127
|
+
// dot-followed-by-digit (an IPv4 next octet / longer dotted-numeric run, e.g.
|
|
128
|
+
// `127.0.0.1`, whose `0.0.1` tail would otherwise register). A sentence-ending
|
|
129
|
+
// period after the patch (dot followed by non-digit / end-of-line, e.g.
|
|
130
|
+
// `// fixed in 0.18.9.`) is NOT excluded — that is exactly the version residue
|
|
131
|
+
// the gate must catch. The leading `(?<![\d.])` lookbehind keeps the IPv4
|
|
132
|
+
// suppression on the other side.
|
|
133
|
+
const VERSION_TAG_RE = /(?<![\d.])v?0\.\d+\.\d+(?!\d)(?!\.\d)/;
|
|
123
134
|
|
|
124
135
|
// Phase residue patterns — broader than just version tags.
|
|
125
136
|
const PHASE_RESIDUE_RES = [
|
|
@@ -314,5 +325,13 @@ function main() {
|
|
|
314
325
|
process.exitCode = 1;
|
|
315
326
|
}
|
|
316
327
|
|
|
328
|
+
module.exports = {
|
|
329
|
+
VERSION_TAG_RE,
|
|
330
|
+
PHASE_RESIDUE_RES,
|
|
331
|
+
FILENAME_VERSION_RE,
|
|
332
|
+
countLineViolations,
|
|
333
|
+
scanCurrent,
|
|
334
|
+
};
|
|
335
|
+
|
|
317
336
|
if (require.main === module) main();
|
|
318
337
|
|
package/scripts/predeploy.js
CHANGED
|
@@ -249,6 +249,19 @@ const GATES = [
|
|
|
249
249
|
args: [path.join(ROOT, "scripts", "check-codebase-patterns.js")],
|
|
250
250
|
ciJobName: "Data integrity (catalog + manifest snapshot)",
|
|
251
251
|
},
|
|
252
|
+
{
|
|
253
|
+
// Test-subject coverage gate. Bidirectional: every tests/<x>.test.js must
|
|
254
|
+
// be named after a real SUBJECT the codebase has (a module / CLI verb /
|
|
255
|
+
// CVE id / playbook / data primitive / repo artifact), and every such
|
|
256
|
+
// subject must have a test. Blocks the naming drift that lets a test be
|
|
257
|
+
// filed under a version/finding label (where downstream readers can't find
|
|
258
|
+
// it) and surfaces any module/playbook that ships without a test. Derived
|
|
259
|
+
// dynamically from the source tree, so the list is never hand-maintained.
|
|
260
|
+
name: "Test-subject coverage (every test maps to a subject; every subject has a test)",
|
|
261
|
+
command: process.execPath,
|
|
262
|
+
args: [path.join(ROOT, "scripts", "check-test-subjects.js")],
|
|
263
|
+
ciJobName: "Data integrity (catalog + manifest snapshot)",
|
|
264
|
+
},
|
|
252
265
|
{
|
|
253
266
|
// Release-notes extract + quality gate. Runs the same `## <version>`
|
|
254
267
|
// CHANGELOG extraction the release workflow publishes as the GitHub
|