@blamejs/exceptd-skills 0.18.9 → 0.18.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +36 -0
- package/bin/exceptd.js +204 -118
- package/data/_indexes/_meta.json +3 -3
- package/data/_indexes/frequency.json +2 -2
- package/data/d3fend-catalog.json +6 -6
- package/data/playbooks/identity-sso-compromise.json +2 -2
- package/data/playbooks/sbom.json +1 -1
- package/lib/citation-resolve.js +11 -0
- package/lib/collectors/containers.js +13 -0
- package/lib/collectors/cred-stores.js +18 -9
- package/lib/collectors/secrets.js +4 -2
- package/lib/cross-ref-api.js +29 -7
- package/lib/cve-regression-watcher.js +47 -15
- package/lib/framework-gap.js +52 -19
- package/lib/gap-detectors.js +8 -3
- package/lib/lint-skills.js +3 -2
- package/lib/playbook-runner.js +125 -7
- package/lib/refresh-external.js +58 -7
- package/lib/refresh-network.js +18 -5
- package/lib/rfc-cli.js +113 -18
- package/lib/schemas/playbook.schema.json +1 -1
- package/lib/scoring.js +71 -8
- package/lib/source-advisories.js +58 -9
- package/lib/ttp-mapper.js +31 -3
- package/lib/upstream-check-cli.js +13 -1
- package/lib/validate-catalog-meta.js +51 -7
- package/lib/validate-cve-catalog.js +10 -0
- package/lib/validate-playbooks.js +19 -1
- package/lib/verify.js +35 -34
- package/lib/xml-tokenizer.js +187 -25
- package/manifest.json +53 -53
- package/orchestrator/dispatcher.js +53 -9
- package/orchestrator/index.js +9 -7
- package/orchestrator/pipeline.js +62 -14
- package/orchestrator/scanner.js +60 -9
- package/package.json +1 -1
- package/sbom.cdx.json +115 -100
- package/scripts/build-indexes.js +21 -3
- package/scripts/builders/cwe-chains.js +5 -2
- package/scripts/builders/section-offsets.js +17 -8
- package/scripts/builders/summary-cards.js +12 -4
- package/scripts/check-catalog-gap-budget.js +3 -3
- package/scripts/check-codebase-patterns-currency.js +1 -0
- package/scripts/check-codebase-patterns.js +166 -11
- package/scripts/check-sbom-currency.js +69 -3
- package/scripts/check-test-count.js +28 -16
- package/scripts/check-test-subjects.js +148 -0
- package/scripts/check-version-tags.js +24 -5
- package/scripts/predeploy.js +32 -8
- package/scripts/refresh-upstream-catalogs.js +169 -44
- package/scripts/release.js +28 -11
package/scripts/build-indexes.js
CHANGED
|
@@ -74,9 +74,27 @@ function writeJson(name, obj) {
|
|
|
74
74
|
// is atomic on POSIX and Windows, so a reader (or the next build's
|
|
75
75
|
// parse-check) only ever sees the complete old or complete new file.
|
|
76
76
|
const abs = path.join(IDX, name);
|
|
77
|
-
|
|
77
|
+
// Random suffix (not just pid) so a Promise.all fan-out (--parallel) sharing
|
|
78
|
+
// a pid can't race the same tmp path, and the in-repo temp name isn't
|
|
79
|
+
// predictable. Mirrors lib/citation-resolve.js cachePut.
|
|
80
|
+
const tmp = `${abs}.tmp-${process.pid}.${crypto.randomBytes(4).toString("hex")}`;
|
|
78
81
|
fs.writeFileSync(tmp, JSON.stringify(obj, null, 2) + "\n", "utf8");
|
|
79
|
-
|
|
82
|
+
// rename-over-existing is atomic, but on Windows a sync client / AV / indexer
|
|
83
|
+
// (e.g. Dropbox holding the target open for a few ms) can make it transiently
|
|
84
|
+
// EPERM/EACCES/EBUSY. Retry with short backoff so a build doesn't fail on a
|
|
85
|
+
// lock that clears in milliseconds; on POSIX the first attempt always wins.
|
|
86
|
+
let lastErr;
|
|
87
|
+
for (let attempt = 0; attempt < 10; attempt++) {
|
|
88
|
+
try { fs.renameSync(tmp, abs); return; }
|
|
89
|
+
catch (e) {
|
|
90
|
+
lastErr = e;
|
|
91
|
+
if (e.code !== "EPERM" && e.code !== "EACCES" && e.code !== "EBUSY") break;
|
|
92
|
+
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 20 * (attempt + 1));
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
// Persistent failure: drop the temp sibling so it doesn't orphan, then surface.
|
|
96
|
+
try { fs.unlinkSync(tmp); } catch { /* best effort */ }
|
|
97
|
+
throw lastErr;
|
|
80
98
|
}
|
|
81
99
|
|
|
82
100
|
function readJson(p) {
|
|
@@ -795,7 +813,7 @@ async function main() {
|
|
|
795
813
|
for (const n of wanted) {
|
|
796
814
|
if (!OUTPUTS.find((o) => o.name === n)) {
|
|
797
815
|
console.error(`build-indexes: unknown output "${n}". Valid: ${OUTPUTS.map((o) => o.name).join(", ")}`);
|
|
798
|
-
process.
|
|
816
|
+
process.exitCode = 2; return;
|
|
799
817
|
}
|
|
800
818
|
}
|
|
801
819
|
chosen = withDependencyClosure(wanted);
|
|
@@ -81,11 +81,14 @@ function buildCweChains({ skills, cweCatalog, atlasTtps, cveCatalog, frameworkGa
|
|
|
81
81
|
};
|
|
82
82
|
|
|
83
83
|
// Related CVEs: walk evidence_cves on the framework_gaps that the
|
|
84
|
-
// referencing skills cite. Inner join via the skill graph.
|
|
84
|
+
// referencing skills cite. Inner join via the skill graph. Skip draft
|
|
85
|
+
// (_draft) CVEs so this CWE half agrees with the CVE half (build-indexes.js)
|
|
86
|
+
// and the reverse-ref index — all three must reflect the same curated,
|
|
87
|
+
// operator-queryable truth.
|
|
85
88
|
const relatedCves = new Set();
|
|
86
89
|
for (const gap of accum.framework_gaps) {
|
|
87
90
|
for (const ev of (frameworkGaps[gap]?.evidence_cves || [])) {
|
|
88
|
-
if (cveCatalog[ev]) relatedCves.add(ev);
|
|
91
|
+
if (cveCatalog[ev] && cveCatalog[ev]._draft !== true) relatedCves.add(ev);
|
|
89
92
|
}
|
|
90
93
|
}
|
|
91
94
|
|
|
@@ -68,13 +68,19 @@ function buildOne(absPath, relPath) {
|
|
|
68
68
|
const totalBytes = buf.length;
|
|
69
69
|
const text = buf.toString("utf8");
|
|
70
70
|
const lines = text.split(/\r?\n/);
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
71
|
+
// EOL-aware line-start byte offsets: walk the actual terminator bytes off the
|
|
72
|
+
// decoded text rather than assuming a fixed 1-byte newline. On a pure-LF body
|
|
73
|
+
// every terminator is 1 byte so the offsets are identical to the old `+ 1`
|
|
74
|
+
// accumulator; on a stray CRLF body the \r\n terminator is 2 bytes and the
|
|
75
|
+
// offsets stay correct (the old fixed-constant approach undercounted by 1
|
|
76
|
+
// byte per line and silently misaligned every token-budget slice). The split
|
|
77
|
+
// above discards terminator bytes, so the width can only be recovered by
|
|
78
|
+
// reading the terminators off the raw text — done here.
|
|
79
|
+
const lineByteOffsets = [0];
|
|
80
|
+
const eolRe = /\r?\n/g;
|
|
81
|
+
let m;
|
|
82
|
+
while ((m = eolRe.exec(text)) !== null) {
|
|
83
|
+
lineByteOffsets.push(Buffer.byteLength(text.slice(0, m.index + m[0].length), "utf8"));
|
|
78
84
|
}
|
|
79
85
|
|
|
80
86
|
// Frontmatter: lines between the first "---" and the second "---".
|
|
@@ -167,4 +173,7 @@ function buildSectionOffsets({ root, skills }) {
|
|
|
167
173
|
};
|
|
168
174
|
}
|
|
169
175
|
|
|
170
|
-
|
|
176
|
+
// buildOne is exported for regression testing of the EOL-aware byte offsets
|
|
177
|
+
// (a CRLF body must still produce byte_start values that point at the real
|
|
178
|
+
// "## " byte in the raw file).
|
|
179
|
+
module.exports = { buildSectionOffsets, buildOne };
|
|
@@ -106,10 +106,18 @@ function firstChunkAfterHeader(body, headerRegex, maxChars = 600) {
|
|
|
106
106
|
}
|
|
107
107
|
|
|
108
108
|
function handoffTargets(body, allSkillNames, selfName) {
|
|
109
|
-
// Look in the Hand-Off section; backtick-quoted skill names count as a
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
109
|
+
// Look ONLY in the Hand-Off section; backtick-quoted skill names count as a
|
|
110
|
+
// target. Bound the scan to [Hand-Off header, next real H2) so unrelated
|
|
111
|
+
// later sections are not mis-attributed as hand-off targets. Header
|
|
112
|
+
// detection and the section boundary are both fence-aware (a `## ` line
|
|
113
|
+
// inside a ```...``` block is not a real H2).
|
|
114
|
+
const lines = body.split(/\r?\n/);
|
|
115
|
+
const h2 = findRealH2Indices(lines);
|
|
116
|
+
const handoffIdx = h2.find((i) => /^## Hand-?Off/.test(lines[i]));
|
|
117
|
+
if (handoffIdx == null) return [];
|
|
118
|
+
const nextH2 = h2.find((i) => i > handoffIdx);
|
|
119
|
+
const sectionEnd = nextH2 != null ? nextH2 : lines.length;
|
|
120
|
+
const slice = lines.slice(handoffIdx, sectionEnd).join("\n");
|
|
113
121
|
const targets = new Set();
|
|
114
122
|
for (const name of allSkillNames) {
|
|
115
123
|
if (name === selfName) continue;
|
|
@@ -116,12 +116,12 @@ function main() {
|
|
|
116
116
|
console.error("Add an explicit budget entry in both:");
|
|
117
117
|
console.error(" scripts/check-catalog-gap-budget.js");
|
|
118
118
|
console.error(" tests/shipped-catalog-integrity.test.js");
|
|
119
|
-
process.
|
|
119
|
+
process.exitCode = 1; return;
|
|
120
120
|
}
|
|
121
121
|
if (missingBudget.length > 0) {
|
|
122
122
|
console.error("\n[check-catalog-gap-budget] BUDGET missing entries for declared classes:");
|
|
123
123
|
for (const c of missingBudget) console.error(` ${c}: declared by lib/gap-detectors.js DETECTOR_CLASSES, no BUDGET entry`);
|
|
124
|
-
process.
|
|
124
|
+
process.exitCode = 1; return;
|
|
125
125
|
}
|
|
126
126
|
if (regressions.length > 0) {
|
|
127
127
|
console.error("\n[check-catalog-gap-budget] REGRESSION beyond budget:");
|
|
@@ -131,7 +131,7 @@ function main() {
|
|
|
131
131
|
console.error("\nClose the gap in this PR (preferred) or update BUDGET in both:");
|
|
132
132
|
console.error(" scripts/check-catalog-gap-budget.js");
|
|
133
133
|
console.error(" tests/shipped-catalog-integrity.test.js");
|
|
134
|
-
process.
|
|
134
|
+
process.exitCode = 1; return;
|
|
135
135
|
}
|
|
136
136
|
console.log("[check-catalog-gap-budget] all classes within budget; every class is budgeted.");
|
|
137
137
|
}
|
|
@@ -47,6 +47,7 @@ const UPSTREAM_TRIAGED = Object.freeze([ // keep-sorted
|
|
|
47
47
|
"from-base64url-untrapped",
|
|
48
48
|
"fs-path-from-operator-identifier-without-traversal-refusal",
|
|
49
49
|
"gitleaks-entropy",
|
|
50
|
+
"hand-rolled-sql",
|
|
50
51
|
"handrolled-buffer-collect",
|
|
51
52
|
"handrolled-debounce",
|
|
52
53
|
"hostname-compare-trailing-dot",
|
|
@@ -49,6 +49,7 @@ const VALID_ALLOW_CLASSES = Object.freeze({
|
|
|
49
49
|
"bidi-codepoint-literal": true,
|
|
50
50
|
"unsorted-marked-array": true,
|
|
51
51
|
"misaligned-marked-run": true,
|
|
52
|
+
"hand-rolled-sql": true,
|
|
52
53
|
});
|
|
53
54
|
|
|
54
55
|
const EXCLUDE_DIRS = new Set([
|
|
@@ -148,6 +149,79 @@ function filterMarkers(hits, cls) {
|
|
|
148
149
|
|
|
149
150
|
// ---- require.main block ranges -------------------------------------------
|
|
150
151
|
|
|
152
|
+
// Count `{` / `}` in `line` that are in REAL CODE context, advancing a
|
|
153
|
+
// stateful tokenizer that tracks string / template / comment regions across
|
|
154
|
+
// lines. Braces inside a single/double/template string, a `//` line comment,
|
|
155
|
+
// or a `/* */` block comment do NOT affect depth — otherwise a `{` or `}`
|
|
156
|
+
// typed inside a string literal in the require.main block miscounts the brace
|
|
157
|
+
// balance and the computed block range slides onto an unrelated later function
|
|
158
|
+
// (whose process.exit() is then wrongly treated as a CLI-entry exit and not
|
|
159
|
+
// flagged). `inTemplate` and `inBlockComment` are the cross-line states a
|
|
160
|
+
// per-line stripper cannot model, so the tokenizer state object is threaded
|
|
161
|
+
// line-to-line by the caller.
|
|
162
|
+
//
|
|
163
|
+
// `state` is mutated in place: { inSingle, inDouble, inTemplate, inBlock,
|
|
164
|
+
// templateDepth } — `templateDepth` tracks `${ … }` interpolation nesting so
|
|
165
|
+
// the closing `}` of an interpolation is treated as template punctuation, not
|
|
166
|
+
// a code brace, while braces INSIDE the interpolation expression still count.
|
|
167
|
+
function countCodeBraces(line, state) {
|
|
168
|
+
let delta = 0;
|
|
169
|
+
for (let i = 0; i < line.length; i++) {
|
|
170
|
+
const ch = line[i];
|
|
171
|
+
const next = line[i + 1];
|
|
172
|
+
if (state.inBlock) {
|
|
173
|
+
if (ch === "*" && next === "/") { state.inBlock = false; i++; }
|
|
174
|
+
continue;
|
|
175
|
+
}
|
|
176
|
+
if (state.inSingle) {
|
|
177
|
+
if (ch === "\\") { i++; continue; }
|
|
178
|
+
if (ch === "'") state.inSingle = false;
|
|
179
|
+
continue;
|
|
180
|
+
}
|
|
181
|
+
if (state.inDouble) {
|
|
182
|
+
if (ch === "\\") { i++; continue; }
|
|
183
|
+
if (ch === '"') state.inDouble = false;
|
|
184
|
+
continue;
|
|
185
|
+
}
|
|
186
|
+
if (state.inTemplate) {
|
|
187
|
+
if (ch === "\\") { i++; continue; }
|
|
188
|
+
if (ch === "`") { state.inTemplate = false; continue; }
|
|
189
|
+
if (ch === "$" && next === "{") {
|
|
190
|
+
// Enter an interpolation expression: braces inside ARE code.
|
|
191
|
+
state.templateExpr.push(0);
|
|
192
|
+
state.inTemplate = false;
|
|
193
|
+
i++; // skip the `{`; the `${` opener is template punctuation
|
|
194
|
+
continue;
|
|
195
|
+
}
|
|
196
|
+
continue;
|
|
197
|
+
}
|
|
198
|
+
// Code context (possibly inside a template interpolation expression).
|
|
199
|
+
if (ch === "/" && next === "/") break; // `//` — rest of the line is a comment
|
|
200
|
+
if (ch === "/" && next === "*") { state.inBlock = true; i++; continue; }
|
|
201
|
+
if (ch === "'") { state.inSingle = true; continue; }
|
|
202
|
+
if (ch === '"') { state.inDouble = true; continue; }
|
|
203
|
+
if (ch === "`") { state.inTemplate = true; continue; }
|
|
204
|
+
if (ch === "{") {
|
|
205
|
+
if (state.templateExpr.length) state.templateExpr[state.templateExpr.length - 1]++;
|
|
206
|
+
delta++;
|
|
207
|
+
} else if (ch === "}") {
|
|
208
|
+
if (state.templateExpr.length && state.templateExpr[state.templateExpr.length - 1] === 0) {
|
|
209
|
+
// Closes the `${ … }` interpolation — back to template body.
|
|
210
|
+
state.templateExpr.pop();
|
|
211
|
+
state.inTemplate = true;
|
|
212
|
+
} else {
|
|
213
|
+
if (state.templateExpr.length) state.templateExpr[state.templateExpr.length - 1]--;
|
|
214
|
+
delta--;
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
return delta;
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
function newBraceState() {
|
|
222
|
+
return { inSingle: false, inDouble: false, inTemplate: false, inBlock: false, templateExpr: [] };
|
|
223
|
+
}
|
|
224
|
+
|
|
151
225
|
// Line ranges (1-based, inclusive) of `if (require.main === module) { ... }`
|
|
152
226
|
// blocks — the dual-mode CLI-entry section where synchronous-print-then-exit
|
|
153
227
|
// is correct. process.exit there is owned by tests/safe-exit-grep.test.js and
|
|
@@ -156,15 +230,16 @@ function requireMainRanges(lines) {
|
|
|
156
230
|
const ranges = [];
|
|
157
231
|
for (let i = 0; i < lines.length; i++) {
|
|
158
232
|
if (/\brequire\.main\s*===\s*module\b/.test(lines[i])) {
|
|
159
|
-
// Find the opening brace (same line or next few), then balance
|
|
233
|
+
// Find the opening brace (same line or next few), then balance —
|
|
234
|
+
// string/comment/template-aware so braces inside literals don't skew the
|
|
235
|
+
// depth (see countCodeBraces).
|
|
160
236
|
let depth = 0;
|
|
161
237
|
let started = false;
|
|
162
238
|
let j = i;
|
|
239
|
+
const state = newBraceState();
|
|
163
240
|
for (; j < lines.length; j++) {
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
else if (ch === "}") { depth--; }
|
|
167
|
-
}
|
|
241
|
+
depth += countCodeBraces(lines[j], state);
|
|
242
|
+
if (depth > 0) started = true;
|
|
168
243
|
if (started && depth <= 0) break;
|
|
169
244
|
}
|
|
170
245
|
if (started) ranges.push([i + 1, j + 1]);
|
|
@@ -181,8 +256,14 @@ function inRanges(ranges, lineNo) {
|
|
|
181
256
|
|
|
182
257
|
// A line that opens a new function body (so a backward stdout-write scan stops
|
|
183
258
|
// at the enclosing function and doesn't arm an exit from an unrelated earlier
|
|
184
|
-
// function).
|
|
185
|
-
|
|
259
|
+
// function). The bare-identifier (third) alternative matches a declaration /
|
|
260
|
+
// method-shorthand opener (`foo() {`, `async bar() {`), but it must REFUSE
|
|
261
|
+
// control-flow openers (`for (…) {`, `if (…) {`, `while/switch/catch (…) {`):
|
|
262
|
+
// a control-flow block sitting between a stdout write and a process.exit() is
|
|
263
|
+
// inside the SAME function, so stopping the backward scan there would wrongly
|
|
264
|
+
// leave the exit unflagged. The negative lookahead excludes the control-flow
|
|
265
|
+
// keywords; `function` and arrow alternatives are unchanged.
|
|
266
|
+
const FUNCTION_START = /(^|[^.\w])function\b|=>\s*\{?\s*$|^\s*(async\s+)?(?!(?:if|for|while|switch|catch|do|else|with|finally|return)\b)[A-Za-z_$][\w$]*\s*\([^)]*\)\s*\{/;
|
|
186
267
|
|
|
187
268
|
function detectProcessExitAfterStdout(files) {
|
|
188
269
|
const hits = [];
|
|
@@ -210,6 +291,15 @@ function detectProcessExitAfterStdout(files) {
|
|
|
210
291
|
return filterMarkers(hits, "process-exit-after-stdout-write");
|
|
211
292
|
}
|
|
212
293
|
|
|
294
|
+
// The first non-whitespace char of the first arg is `"`, `'`, or `/` => a
|
|
295
|
+
// string/regex literal => static, safe. Anything else (an identifier, a `(`,
|
|
296
|
+
// a backtick template) is operator-derivable and flagged. Backtick is NOT
|
|
297
|
+
// exempt — a template literal can interpolate operator input, so it must be
|
|
298
|
+
// flagged the same as a bare identifier.
|
|
299
|
+
function isStaticRegexFirstChar(ch) {
|
|
300
|
+
return ch === '"' || ch === "'" || ch === "/";
|
|
301
|
+
}
|
|
302
|
+
|
|
213
303
|
function detectDynamicRegex(files) {
|
|
214
304
|
const hits = [];
|
|
215
305
|
for (const rel of (files || filesUnder(["lib", "orchestrator", "bin/exceptd.js"]))) {
|
|
@@ -217,10 +307,29 @@ function detectDynamicRegex(files) {
|
|
|
217
307
|
for (let i = 0; i < lines.length; i++) {
|
|
218
308
|
const code = stripLineComment(lines[i]);
|
|
219
309
|
const m = code.match(/\bnew RegExp\s*\(\s*(.)/);
|
|
220
|
-
if (
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
310
|
+
if (m) {
|
|
311
|
+
if (isStaticRegexFirstChar(m[1])) continue;
|
|
312
|
+
hits.push({ file: rel, line: i + 1, content: lines[i].trim() });
|
|
313
|
+
continue;
|
|
314
|
+
}
|
|
315
|
+
// Multi-line form: `new RegExp(` ends the (comment-stripped) line with the
|
|
316
|
+
// open paren as the last token, and the pattern arg is on a following
|
|
317
|
+
// line. The single-line match above can't see the first-arg char, so it
|
|
318
|
+
// would silently pass a dynamic RegExp whose argument starts next line.
|
|
319
|
+
// Look ahead, skipping blank and comment-only lines (capped at 5), and
|
|
320
|
+
// inspect the first code line's first non-whitespace char.
|
|
321
|
+
if (!/\bnew RegExp\s*\(\s*$/.test(code)) continue;
|
|
322
|
+
let firstChar = null;
|
|
323
|
+
for (let k = i + 1; k <= i + 5 && k < lines.length; k++) {
|
|
324
|
+
const ahead = stripLineComment(lines[k]).replace(/^\s+/, "");
|
|
325
|
+
if (ahead === "") continue; // blank or comment-only — skip
|
|
326
|
+
firstChar = ahead[0];
|
|
327
|
+
break;
|
|
328
|
+
}
|
|
329
|
+
// A `new RegExp(` with nothing parseable after it within the cap is
|
|
330
|
+
// suspicious — flag conservatively. Otherwise apply the SAME literal
|
|
331
|
+
// exemption as the single-line path.
|
|
332
|
+
if (firstChar !== null && isStaticRegexFirstChar(firstChar)) continue;
|
|
224
333
|
hits.push({ file: rel, line: i + 1, content: lines[i].trim() });
|
|
225
334
|
}
|
|
226
335
|
}
|
|
@@ -378,6 +487,37 @@ function detectMisalignedMarkedRun(files) {
|
|
|
378
487
|
return hits;
|
|
379
488
|
}
|
|
380
489
|
|
|
490
|
+
// ---- hand-rolled SQL in a file that talks to a database ------------------
|
|
491
|
+
//
|
|
492
|
+
// exceptd ships no database today, so this is a FORWARD guard: the moment a
|
|
493
|
+
// file imports a SQL driver, any SQL STATEMENT (`"SELECT …"`) or CLAUSE built
|
|
494
|
+
// by string concatenation (`… + " WHERE " + …`) is a parameterization/injection
|
|
495
|
+
// sink and must use the driver's bound-parameter API instead. The SQL-driver
|
|
496
|
+
// import is the gate — a SQL-looking string in a file with no driver executes
|
|
497
|
+
// nothing, so prose that merely begins with "Update …" / "Delete …" in a
|
|
498
|
+
// non-DB file is never scanned (no false positives). A trusted static DDL
|
|
499
|
+
// string can opt out with `// allow:hand-rolled-sql — <reason>`.
|
|
500
|
+
const SQL_DRIVER_IMPORT = /require\(\s*["'](?:node:sqlite|better-sqlite3|sqlite3|sqlite|pg|mysql2?|knex|sequelize|drizzle-orm|postgres|@libsql\/[\w.-]+)(?:\/[^"']*)?["']\s*\)|\bfrom\s+["'](?:node:sqlite|better-sqlite3|pg|mysql2?|knex|sequelize|drizzle-orm)(?:\/[^"']*)?["']/;
|
|
501
|
+
const SQL_STMT_START = /(["'`])\s*(?:SELECT\b|INSERT\s+(?:INTO|OR)\b|REPLACE\s+INTO\b|UPDATE\s+["'`]?[A-Za-z_]|DELETE\s+FROM\b|CREATE\s+(?:TABLE|UNIQUE\s+INDEX|INDEX|TRIGGER|VIRTUAL\s+TABLE)\b|ALTER\s+TABLE\b|DROP\s+(?:TABLE|TRIGGER|INDEX)\b|MERGE\s+INTO\b)/i;
|
|
502
|
+
// Trailing-concat form tolerates an embedded SQL string-quote inside the
|
|
503
|
+
// clause (e.g. `" WHERE name = 'x' " + id`) by scanning to the first `+`
|
|
504
|
+
// concatenation operator rather than requiring the clause string to close on
|
|
505
|
+
// the same quote with no interior quotes.
|
|
506
|
+
const SQL_CLAUSE_FRAG = /(?:\+\s*["'`]\s*(?:SET|FROM|WHERE|VALUES|ORDER\s+BY|GROUP\s+BY|HAVING|RETURNING|LIMIT|OFFSET|ON\s+CONFLICT|(?:INNER\s+|LEFT\s+|RIGHT\s+|CROSS\s+)?JOIN)\b|["'`]\s*(?:SET|FROM|WHERE|VALUES\s*\(|ORDER\s+BY|GROUP\s+BY|HAVING|RETURNING|ON\s+CONFLICT|(?:INNER\s+|LEFT\s+|RIGHT\s+|CROSS\s+)?JOIN)\b[^+]*\+)/i;
|
|
507
|
+
function detectHandRolledSql(files) {
|
|
508
|
+
const hits = [];
|
|
509
|
+
for (const rel of (files || filesUnder(["lib", "orchestrator", "bin/exceptd.js", "scripts"]))) {
|
|
510
|
+
const lines = readLines(rel);
|
|
511
|
+
if (!SQL_DRIVER_IMPORT.test(lines.join("\n"))) continue; // not a DB file — never scan prose
|
|
512
|
+
for (let i = 0; i < lines.length; i++) {
|
|
513
|
+
const code = stripLineComment(lines[i]);
|
|
514
|
+
if (SQL_STMT_START.test(code)) { hits.push({ file: rel, line: i + 1, content: lines[i].trim().slice(0, 110), why: "SQL statement in a string literal" }); continue; }
|
|
515
|
+
if (SQL_CLAUSE_FRAG.test(code)) { hits.push({ file: rel, line: i + 1, content: lines[i].trim().slice(0, 110), why: "SQL clause built by concatenation" }); }
|
|
516
|
+
}
|
|
517
|
+
}
|
|
518
|
+
return filterMarkers(hits, "hand-rolled-sql");
|
|
519
|
+
}
|
|
520
|
+
|
|
381
521
|
const CLASSES = [
|
|
382
522
|
{
|
|
383
523
|
id: "process-exit-after-stdout-write",
|
|
@@ -415,6 +555,12 @@ const CLASSES = [
|
|
|
415
555
|
warnOnly: false,
|
|
416
556
|
hint: "a typo'd or reason-less `// allow:<class>` suppresses nothing — fix the class id or add `— <reason>`",
|
|
417
557
|
},
|
|
558
|
+
{
|
|
559
|
+
id: "hand-rolled-sql",
|
|
560
|
+
run: detectHandRolledSql,
|
|
561
|
+
warnOnly: false,
|
|
562
|
+
hint: "a SQL statement/clause assembled as a string in a file that imports a SQL driver is an injection sink — use bound parameters, or `// allow:hand-rolled-sql — <reason>` for a trusted static DDL string",
|
|
563
|
+
},
|
|
418
564
|
];
|
|
419
565
|
|
|
420
566
|
function main() {
|
|
@@ -451,8 +597,17 @@ module.exports = {
|
|
|
451
597
|
detectOrphanAllowClass,
|
|
452
598
|
detectUnsortedMarkedArray,
|
|
453
599
|
detectMisalignedMarkedRun,
|
|
600
|
+
detectHandRolledSql,
|
|
601
|
+
SQL_DRIVER_IMPORT,
|
|
602
|
+
SQL_STMT_START,
|
|
603
|
+
SQL_CLAUSE_FRAG,
|
|
454
604
|
scanUnsortedMarkedArray,
|
|
455
605
|
scanMisalignedMarkedRun,
|
|
606
|
+
requireMainRanges,
|
|
607
|
+
countCodeBraces,
|
|
608
|
+
newBraceState,
|
|
609
|
+
isStaticRegexFirstChar,
|
|
610
|
+
FUNCTION_START,
|
|
456
611
|
filesUnder,
|
|
457
612
|
};
|
|
458
613
|
|
|
@@ -50,6 +50,66 @@ function catalogEntryCount(dataDir, file) {
|
|
|
50
50
|
return 0;
|
|
51
51
|
}
|
|
52
52
|
|
|
53
|
+
// Files/prefixes that refresh-sbom's expandAllowlist excludes from the shipped
|
|
54
|
+
// file: component inventory. Kept in sync with scripts/refresh-sbom.js
|
|
55
|
+
// (SELF_EXCLUDED + DERIVABLE_PREFIXES): the SBOM never hashes itself, and the
|
|
56
|
+
// pre-computed index cache under data/_indexes/ is regenerated/test-mutated, so
|
|
57
|
+
// it carries no per-file component. The completeness check below must apply the
|
|
58
|
+
// SAME exclusions or it would demand a component for a file the generator never
|
|
59
|
+
// emits one for.
|
|
60
|
+
const SBOM_SELF_EXCLUDED = new Set(["sbom.cdx.json"]);
|
|
61
|
+
const SBOM_DERIVABLE_PREFIXES = ["data/_indexes/"];
|
|
62
|
+
|
|
63
|
+
function sbomIsDerivable(rel) {
|
|
64
|
+
return SBOM_DERIVABLE_PREFIXES.some(
|
|
65
|
+
(p) => rel === p.replace(/\/$/, "") || rel.startsWith(p)
|
|
66
|
+
);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
// Recursively list every regular file under absDir, returned as absolute paths.
|
|
70
|
+
// Mirrors refresh-sbom's walkFiles (which is root-agnostic — it walks whatever
|
|
71
|
+
// absolute dir it is given).
|
|
72
|
+
function walkFilesAbs(absDir) {
|
|
73
|
+
const out = [];
|
|
74
|
+
let entries;
|
|
75
|
+
try { entries = fs.readdirSync(absDir, { withFileTypes: true }); }
|
|
76
|
+
catch { return out; }
|
|
77
|
+
for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) {
|
|
78
|
+
const abs = path.join(absDir, entry.name);
|
|
79
|
+
if (entry.isDirectory()) out.push(...walkFilesAbs(abs));
|
|
80
|
+
else if (entry.isFile()) out.push(abs);
|
|
81
|
+
}
|
|
82
|
+
return out;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
// Root-aware allowlist expansion: the shipped-file set that must each have a
|
|
86
|
+
// file: component, computed against the TARGET tree (`root`), not the running
|
|
87
|
+
// script's source repo. refresh-sbom's exported expandAllowlist is bound to its
|
|
88
|
+
// own REPO_ROOT (it joins/relativizes against __dirname/..), so under a `--root`
|
|
89
|
+
// target the completeness check would otherwise validate the SOURCE repo's file
|
|
90
|
+
// list against the TARGET SBOM — the wrong tree. Replicating the expansion here
|
|
91
|
+
// (same SELF_EXCLUDED + DERIVABLE_PREFIXES exclusions, same dedupe+sort) keeps
|
|
92
|
+
// the gate honest under `--root` without reaching across into the generator's
|
|
93
|
+
// module-level root.
|
|
94
|
+
function expandAllowlistAt(allowlist, root) {
|
|
95
|
+
const abs = [];
|
|
96
|
+
for (const entry of allowlist) {
|
|
97
|
+
const full = path.join(root, entry);
|
|
98
|
+
let stat;
|
|
99
|
+
try { stat = fs.statSync(full); }
|
|
100
|
+
catch { continue; } // tolerate a stale entry; presence checks elsewhere flag
|
|
101
|
+
if (stat.isDirectory()) abs.push(...walkFilesAbs(full));
|
|
102
|
+
else if (stat.isFile()) abs.push(full);
|
|
103
|
+
}
|
|
104
|
+
const rel = Array.from(
|
|
105
|
+
new Set(abs.map((a) => path.relative(root, a).split(path.sep).join("/")))
|
|
106
|
+
)
|
|
107
|
+
.filter((r) => !SBOM_SELF_EXCLUDED.has(r))
|
|
108
|
+
.filter((r) => !sbomIsDerivable(r))
|
|
109
|
+
.sort();
|
|
110
|
+
return rel;
|
|
111
|
+
}
|
|
112
|
+
|
|
53
113
|
// The description string embeds per-catalog ENTRY counts as free text, e.g.
|
|
54
114
|
// "11 catalogs (439 CVEs / 177 CWEs / 805 ATT&CK + ICS / 170 ATLAS /
|
|
55
115
|
// 468 D3FEND / 8888 RFCs)". Each token maps to one data/*.json catalog whose
|
|
@@ -332,9 +392,15 @@ function checkSbomCurrency(root) {
|
|
|
332
392
|
// refresh-sbom's exact allowlist expansion + digest so the gate can't drift
|
|
333
393
|
// from the generator.
|
|
334
394
|
try {
|
|
335
|
-
|
|
395
|
+
// bundleDigest operates purely on the file: component objects (no tree
|
|
396
|
+
// walk), so it is root-agnostic and safe to reuse from the generator. The
|
|
397
|
+
// allowlist expansion, by contrast, MUST run against the target `root`
|
|
398
|
+
// (expandAllowlistAt below) — refresh-sbom's exported expandAllowlist is
|
|
399
|
+
// pinned to its own source-repo REPO_ROOT and would validate the wrong tree
|
|
400
|
+
// under `--root`.
|
|
401
|
+
const { bundleDigest } = require("./refresh-sbom");
|
|
336
402
|
const pkg = JSON.parse(fs.readFileSync(path.join(root, "package.json"), "utf8"));
|
|
337
|
-
const expected =
|
|
403
|
+
const expected = expandAllowlistAt(pkg.files || [], root);
|
|
338
404
|
const fileComps = components.filter(
|
|
339
405
|
(c) => typeof c["bom-ref"] === "string" && c["bom-ref"].startsWith("file:")
|
|
340
406
|
);
|
|
@@ -402,6 +468,6 @@ function main() {
|
|
|
402
468
|
);
|
|
403
469
|
}
|
|
404
470
|
|
|
405
|
-
module.exports = { checkSbomCurrency, resolveRoot, DESCRIPTION_ENTRY_TOKENS, catalogEntryCount };
|
|
471
|
+
module.exports = { checkSbomCurrency, resolveRoot, DESCRIPTION_ENTRY_TOKENS, catalogEntryCount, expandAllowlistAt };
|
|
406
472
|
|
|
407
473
|
if (require.main === module) main();
|
|
@@ -11,18 +11,24 @@
|
|
|
11
11
|
* catches test-set shrinkage.
|
|
12
12
|
*
|
|
13
13
|
* Scope + blind spot. This counts test DECLARATIONS, so it detects deleted
|
|
14
|
-
* files / removed `test(` calls / glob-exclusions. It does NOT detect a
|
|
15
|
-
* neutered in place: `test('name', { skip: true }, fn)
|
|
16
|
-
* still count as one declaration, so flipping a running
|
|
17
|
-
* skipped leaves the count unchanged. Guarding against
|
|
18
|
-
* need runnable-vs-skipped tracking; that is out of scope
|
|
14
|
+
* files / removed `test(`/`it(` calls / glob-exclusions. It does NOT detect a
|
|
15
|
+
* test neutered in place: `test('name', { skip: true }, fn)`, `test.skip(`,
|
|
16
|
+
* and `it.skip(` all still count as one declaration, so flipping a running
|
|
17
|
+
* test to permanently skipped leaves the count unchanged. Guarding against
|
|
18
|
+
* skip-in-place would need runnable-vs-skipped tracking; that is out of scope
|
|
19
|
+
* for this gate.
|
|
19
20
|
*
|
|
20
|
-
* Mechanism: count `test(
|
|
21
|
-
* across `tests/*.test.js` via static analysis (faster than
|
|
22
|
-
*
|
|
23
|
-
*
|
|
24
|
-
*
|
|
25
|
-
*
|
|
21
|
+
* Mechanism: count `test(`/`it(` declarations (including the `.only`/`.skip`
|
|
22
|
+
* variants) across `tests/*.test.js` via static analysis (faster than
|
|
23
|
+
* running). node:test supports both `test()` and the BDD-style `it()`/
|
|
24
|
+
* `describe()` aliases as first-class declarations, and the suite mixes both,
|
|
25
|
+
* so counting only `test(` was blind to every `it(` test. `describe(` is NOT
|
|
26
|
+
* counted — those are containers, not tests, so counting them would inflate
|
|
27
|
+
* the total and conflate removing a grouping block with removing a test.
|
|
28
|
+
* Compare to a baseline pinned in `tests/.test-count-baseline.json`. Fail if
|
|
29
|
+
* the observed count drops MORE than the configured tolerance (default 1)
|
|
30
|
+
* below the baseline. Growth above baseline is fine; if the count grows by
|
|
31
|
+
* more than `update_baseline_when_growth_exceeds`, surface a notice that the
|
|
26
32
|
* baseline file should be refreshed (operator commits the refresh as part
|
|
27
33
|
* of the release that added the tests).
|
|
28
34
|
*
|
|
@@ -71,7 +77,7 @@ function countTests(filePath) {
|
|
|
71
77
|
// Drop a trailing line comment too (`test('x'); // disabled`).
|
|
72
78
|
const stripped = noStrings.replace(/\/\/.*$/, '').trim();
|
|
73
79
|
if (!stripped) continue;
|
|
74
|
-
if (/(?<![A-Za-z0-9_$.])test(?:\.only|\.skip)?\s*\(/.test(stripped)) count++;
|
|
80
|
+
if (/(?<![A-Za-z0-9_$.])(?:test|it)(?:\.only|\.skip)?\s*\(/.test(stripped)) count++;
|
|
75
81
|
}
|
|
76
82
|
return count;
|
|
77
83
|
}
|
|
@@ -165,14 +171,20 @@ function main() {
|
|
|
165
171
|
|
|
166
172
|
if (status === 'shrunk_beyond_tolerance') {
|
|
167
173
|
console.error(`[check-test-count] FAIL - test count dropped from ${baseline} to ${observed} (delta ${delta}, tolerance -${tolerance}).`);
|
|
168
|
-
console.error('[check-test-count] Either a test file was accidentally removed, a test() invocation was deleted, OR the baseline is stale.');
|
|
174
|
+
console.error('[check-test-count] Either a test file was accidentally removed, a test()/it() invocation was deleted, OR the baseline is stale.');
|
|
169
175
|
console.error('[check-test-count] If the drop is intentional, run: node scripts/check-test-count.js --update-baseline');
|
|
170
|
-
process.exit
|
|
176
|
+
// exitCode + return (not process.exit) so the buffered stdout write above
|
|
177
|
+
// (the --json result / one-line summary) drains before the event loop ends
|
|
178
|
+
// — process.exit() can truncate piped output.
|
|
179
|
+
process.exitCode = 1;
|
|
180
|
+
return;
|
|
171
181
|
}
|
|
172
182
|
if (status === 'grew_beyond_threshold_consider_bump') {
|
|
173
183
|
console.error(`[check-test-count] NOTICE - test count grew by ${delta} (above the ${updateThreshold} notice threshold). Consider refreshing the baseline: node scripts/check-test-count.js --update-baseline`);
|
|
174
184
|
}
|
|
175
|
-
process.
|
|
185
|
+
process.exitCode = 0;
|
|
176
186
|
}
|
|
177
187
|
|
|
178
|
-
|
|
188
|
+
module.exports = { countTests, listTestFiles };
|
|
189
|
+
|
|
190
|
+
if (require.main === module) main();
|