@blamejs/exceptd-skills 0.19.33 → 0.19.35
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/bin/exceptd.js +895 -2828
- package/data/_indexes/_meta.json +2 -2
- package/lib/auto-discovery.js +56 -286
- package/lib/canonical-eq.js +7 -40
- package/lib/citation-resolve.js +22 -70
- package/lib/collectors/ai-api.js +170 -76
- package/lib/collectors/cicd-pipeline-compromise.js +113 -136
- package/lib/collectors/citation-hygiene.js +72 -210
- package/lib/collectors/containers.js +41 -130
- package/lib/collectors/cred-stores.js +31 -115
- package/lib/collectors/crypto-codebase.js +55 -138
- package/lib/collectors/crypto.js +24 -54
- package/lib/collectors/hardening.js +20 -78
- package/lib/collectors/kernel.js +16 -46
- package/lib/collectors/library-author.js +198 -211
- package/lib/collectors/mcp.js +24 -70
- package/lib/collectors/runtime.js +24 -86
- package/lib/collectors/sbom.js +130 -118
- package/lib/collectors/scan-excludes.js +33 -139
- package/lib/collectors/secrets.js +62 -178
- package/lib/cross-ref-api.js +39 -123
- package/lib/currency-severity.js +8 -27
- package/lib/cve-batch.js +13 -21
- package/lib/cve-cli.js +13 -20
- package/lib/cve-curation.js +72 -239
- package/lib/cve-regression-watcher.js +29 -155
- package/lib/cvss.js +13 -54
- package/lib/doctor-bucketing.js +3 -19
- package/lib/exit-codes.js +10 -42
- package/lib/flag-suggest.js +7 -25
- package/lib/framework-gap.js +39 -113
- package/lib/gap-detectors.js +37 -159
- package/lib/id-validation.js +9 -30
- package/lib/job-queue.js +13 -36
- package/lib/lint-skills.js +88 -236
- package/lib/playbook-runner.js +759 -2107
- package/lib/prefetch.js +101 -376
- package/lib/refresh-external.js +199 -633
- package/lib/refresh-network.js +78 -311
- package/lib/rfc-cli.js +23 -68
- package/lib/scoring.js +85 -146
- package/lib/sign.js +43 -229
- package/lib/source-advisories.js +43 -194
- package/lib/source-ghsa.js +37 -120
- package/lib/source-osv.js +94 -266
- package/lib/ttp-mapper.js +28 -27
- package/lib/upstream-check-cli.js +36 -29
- package/lib/upstream-check.js +19 -44
- package/lib/validate-catalog-meta.js +17 -61
- package/lib/validate-cve-catalog.js +52 -121
- package/lib/validate-indexes.js +25 -76
- package/lib/validate-package.js +16 -62
- package/lib/validate-playbooks.js +78 -286
- package/lib/validate-vendor.js +16 -49
- package/lib/verify.js +56 -286
- package/lib/version-pins.js +5 -34
- package/lib/worker-pool.js +11 -30
- package/lib/xml-tokenizer.js +47 -152
- package/manifest.json +53 -53
- package/orchestrator/dispatcher.js +17 -68
- package/orchestrator/event-bus.js +11 -74
- package/orchestrator/index.js +138 -413
- package/orchestrator/pipeline.js +28 -85
- package/orchestrator/scanner.js +34 -138
- package/orchestrator/scheduler.js +20 -84
- package/package.json +1 -1
- package/sbom.cdx.json +242 -242
- package/scripts/audit-catalog-gaps.js +9 -62
- package/scripts/audit-cross-skill.js +5 -31
- package/scripts/audit-perf.js +29 -28
- package/scripts/backfill-theater-test.js +7 -64
- package/scripts/bootstrap.js +12 -44
- package/scripts/build-indexes.js +40 -154
- package/scripts/builders/activity-feed.js +4 -14
- package/scripts/builders/catalog-summaries.js +3 -10
- package/scripts/builders/currency.js +7 -20
- package/scripts/builders/cwe-chains.js +7 -30
- package/scripts/builders/did-ladders.js +6 -13
- package/scripts/builders/frequency.js +5 -19
- package/scripts/builders/jurisdiction-clocks.js +6 -25
- package/scripts/builders/recipes.js +6 -14
- package/scripts/builders/section-offsets.js +13 -51
- package/scripts/builders/stale-content.js +7 -28
- package/scripts/builders/summary-cards.js +8 -29
- package/scripts/builders/theater-fingerprints.js +21 -31
- package/scripts/builders/token-budget.js +4 -31
- package/scripts/check-agents-md-collectors.js +26 -57
- package/scripts/check-catalog-gap-budget.js +15 -32
- package/scripts/check-changelog-extract.js +18 -48
- package/scripts/check-codebase-patterns-currency.js +6 -22
- package/scripts/check-codebase-patterns.js +63 -143
- package/scripts/check-epss-consistency.js +9 -64
- package/scripts/check-framework-gap-coverage.js +13 -31
- package/scripts/check-manifest-snapshot.js +62 -81
- package/scripts/check-sbom-currency.js +44 -142
- package/scripts/check-test-count.js +15 -52
- package/scripts/check-test-coverage.js +83 -198
- package/scripts/check-test-subjects.js +21 -62
- package/scripts/check-ttp-references.js +14 -38
- package/scripts/check-ttp-upstream.js +8 -40
- package/scripts/check-version-bump.js +9 -61
- package/scripts/check-version-tags.js +20 -121
- package/scripts/predeploy.js +38 -184
- package/scripts/refresh-manifest-snapshot.js +16 -38
- package/scripts/refresh-mitre-atlas.js +7 -8
- package/scripts/refresh-mitre-attack.js +1 -8
- package/scripts/refresh-mitre-d3fend.js +3 -9
- package/scripts/refresh-mitre-ics-attack.js +7 -8
- package/scripts/refresh-reverse-refs.js +27 -94
- package/scripts/refresh-rfc-index.js +7 -10
- package/scripts/refresh-sbom.js +31 -161
- package/scripts/refresh-upstream-catalogs.js +63 -148
- package/scripts/release.js +69 -234
- package/scripts/run-e2e-scenarios.js +26 -73
- package/scripts/sync-manifest-metadata.js +10 -34
- package/scripts/sync-package-description.js +8 -17
- package/scripts/validate-vendor-online.js +13 -44
- package/scripts/verify-shipped-tarball.js +35 -141
|
@@ -2,41 +2,15 @@
|
|
|
2
2
|
'use strict';
|
|
3
3
|
|
|
4
4
|
/**
|
|
5
|
-
*
|
|
5
|
+
* Canonical-test-count predeploy gate: catches test-set shrinkage the lint and
|
|
6
|
+
* diff-coverage gates cannot see.
|
|
6
7
|
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
* noticing. The lint + diff-coverage gates catch source changes; this gate
|
|
11
|
-
* catches test-set shrinkage.
|
|
8
|
+
* Counts DECLARATIONS statically across `tests/*.test.js` — `test(` and `it(`
|
|
9
|
+
* with their `.only` / `.skip` variants. `describe(` is NOT counted: a container
|
|
10
|
+
* is not a test. Blind spot: a test neutered in place still counts as one.
|
|
12
11
|
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
* test neutered in place: `test('name', { skip: true }, fn)`, `test.skip(`,
|
|
16
|
-
* and `it.skip(` all still count as one declaration, so flipping a running
|
|
17
|
-
* test to permanently skipped leaves the count unchanged. Guarding against
|
|
18
|
-
* skip-in-place would need runnable-vs-skipped tracking; that is out of scope
|
|
19
|
-
* for this gate.
|
|
20
|
-
*
|
|
21
|
-
* Mechanism: count `test(`/`it(` declarations (including the `.only`/`.skip`
|
|
22
|
-
* variants) across `tests/*.test.js` via static analysis (faster than
|
|
23
|
-
* running). node:test supports both `test()` and the BDD-style `it()`/
|
|
24
|
-
* `describe()` aliases as first-class declarations, and the suite mixes both,
|
|
25
|
-
* so counting only `test(` was blind to every `it(` test. `describe(` is NOT
|
|
26
|
-
* counted — those are containers, not tests, so counting them would inflate
|
|
27
|
-
* the total and conflate removing a grouping block with removing a test.
|
|
28
|
-
* Compare to a baseline pinned in `tests/.test-count-baseline.json`. Fail if
|
|
29
|
-
* the observed count drops MORE than the configured tolerance (default 1)
|
|
30
|
-
* below the baseline. Growth above baseline is fine; if the count grows by
|
|
31
|
-
* more than `update_baseline_when_growth_exceeds`, surface a notice that the
|
|
32
|
-
* baseline file should be refreshed (operator commits the refresh as part
|
|
33
|
-
* of the release that added the tests).
|
|
34
|
-
*
|
|
35
|
-
* Output:
|
|
36
|
-
* stdout: structured JSON when --json, else a one-line summary
|
|
37
|
-
* exit 0: observed count is at or above baseline minus tolerance
|
|
38
|
-
* exit 1: observed count dropped beyond tolerance — fail predeploy
|
|
39
|
-
* exit 2: baseline file missing or malformed
|
|
12
|
+
* exit 0 at or above baseline minus tolerance, 1 when it drops further, 2 when
|
|
13
|
+
* the baseline file is missing or malformed.
|
|
40
14
|
*/
|
|
41
15
|
|
|
42
16
|
const fs = require('fs');
|
|
@@ -63,16 +37,13 @@ function listTestFiles(dir) {
|
|
|
63
37
|
|
|
64
38
|
function countTests(filePath) {
|
|
65
39
|
let text = fs.readFileSync(filePath, 'utf8');
|
|
66
|
-
// Strip block comments first
|
|
67
|
-
//
|
|
68
|
-
// defeated this gate's entire purpose (it only stripped whole-line `//`).
|
|
40
|
+
// Strip block comments first: commenting a test out is the usual way to
|
|
41
|
+
// disable one, and counting it anyway defeats the gate.
|
|
69
42
|
text = text.replace(/\/\*[\s\S]*?\*\//g, '');
|
|
70
43
|
let count = 0;
|
|
71
44
|
for (const rawLine of text.split('\n')) {
|
|
72
|
-
// Blank
|
|
73
|
-
//
|
|
74
|
-
// file's docstring examples) is not miscounted as a declaration — that
|
|
75
|
-
// phantom inflated the baseline and could mask a real test deletion.
|
|
45
|
+
// Blank string and template bodies first, so a `test(` inside a string
|
|
46
|
+
// literal is not read as a declaration — a phantom inflates the baseline.
|
|
76
47
|
const noStrings = rawLine.replace(/'(?:[^'\\]|\\.)*'|"(?:[^"\\]|\\.)*"|`(?:[^`\\]|\\.)*`/g, "''");
|
|
77
48
|
// Drop a trailing line comment too (`test('x'); // disabled`).
|
|
78
49
|
const stripped = noStrings.replace(/\/\/.*$/, '').trim();
|
|
@@ -86,11 +57,8 @@ function main() {
|
|
|
86
57
|
const wantJson = process.argv.includes('--json');
|
|
87
58
|
const wantUpdate = process.argv.includes('--update-baseline');
|
|
88
59
|
|
|
89
|
-
//
|
|
90
|
-
//
|
|
91
|
-
// check-then-use window (CodeQL js/file-system-race) where the file the
|
|
92
|
-
// gate decides about is not the file it later reads. ENOENT from the single
|
|
93
|
-
// read IS the "missing" signal — no second path access needed.
|
|
60
|
+
// Branch on the read RESULT, never on a prior existsSync probe: ENOENT from
|
|
61
|
+
// this single read IS the "missing" signal.
|
|
94
62
|
let baselineRaw = null;
|
|
95
63
|
try {
|
|
96
64
|
baselineRaw = fs.readFileSync(BASELINE_PATH, 'utf8');
|
|
@@ -99,13 +67,10 @@ function main() {
|
|
|
99
67
|
console.error(`[check-test-count] cannot read baseline: ${e.message}`);
|
|
100
68
|
process.exit(2);
|
|
101
69
|
}
|
|
102
|
-
// ENOENT — baseline absent.
|
|
103
70
|
if (wantUpdate) {
|
|
104
71
|
const files = listTestFiles(TESTS_DIR);
|
|
105
72
|
const observed = files.reduce((n, f) => n + countTests(f), 0);
|
|
106
|
-
// Exclusive create
|
|
107
|
-
// between the read above and this write, so we never clobber a baseline
|
|
108
|
-
// a concurrent run just produced — atomic, no check-then-write window.
|
|
73
|
+
// Exclusive create: EEXIST rather than clobbering a concurrent run's baseline.
|
|
109
74
|
fs.writeFileSync(BASELINE_PATH, JSON.stringify({
|
|
110
75
|
baseline: observed,
|
|
111
76
|
tolerance: 1,
|
|
@@ -173,9 +138,7 @@ function main() {
|
|
|
173
138
|
console.error(`[check-test-count] FAIL - test count dropped from ${baseline} to ${observed} (delta ${delta}, tolerance -${tolerance}).`);
|
|
174
139
|
console.error('[check-test-count] Either a test file was accidentally removed, a test()/it() invocation was deleted, OR the baseline is stale.');
|
|
175
140
|
console.error('[check-test-count] If the drop is intentional, run: node scripts/check-test-count.js --update-baseline');
|
|
176
|
-
// exitCode
|
|
177
|
-
// (the --json result / one-line summary) drains before the event loop ends
|
|
178
|
-
// — process.exit() can truncate piped output.
|
|
141
|
+
// `process.exitCode`, not `process.exit()`: the buffered stdout write must drain.
|
|
179
142
|
process.exitCode = 1;
|
|
180
143
|
return;
|
|
181
144
|
}
|
|
@@ -1,38 +1,17 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
"use strict";
|
|
3
3
|
/**
|
|
4
|
-
*
|
|
4
|
+
* Diff-aware test-coverage gate. Compares the changed surface in the working
|
|
5
|
+
* tree (or a staged set, or any --base..HEAD range) against the tests/ tree
|
|
6
|
+
* and reports any surface change that lacks a covering test.
|
|
5
7
|
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
8
|
+
* Surfaces: CLI verbs and flags in bin/exceptd.js, exported functions in
|
|
9
|
+
* lib / orchestrator / scripts, playbook detect-indicator and look-artifact
|
|
10
|
+
* ids, and CVE entries whose iocs changed. Docs, tooling dotfiles, tests/
|
|
11
|
+
* itself and derived indexes are allowlisted; workflows, manifests, schemas,
|
|
12
|
+
* SBOM and unclassified files are surfaced as manual-review, never auto-green.
|
|
9
13
|
*
|
|
10
|
-
*
|
|
11
|
-
* - bin/exceptd.js CLI verbs / flags (COMMANDS / PLAYBOOK_VERBS)
|
|
12
|
-
* - lib/*.js, orchestrator/*.js,
|
|
13
|
-
* scripts/*.js exported functions (module.exports = {...})
|
|
14
|
-
* - data/playbooks/*.json detect.indicators[].id + look.artifacts[].id
|
|
15
|
-
* - data/cve-catalog.json CVE entries whose iocs field changed
|
|
16
|
-
*
|
|
17
|
-
* Categorization (no test required):
|
|
18
|
-
* - *.md outside data/, .gitignore, .npmrc, .editorconfig
|
|
19
|
-
* - CHANGELOG.md / README.md / CONTRIBUTING.md / SECURITY.md
|
|
20
|
-
* - whitespace-only diffs (re-run with --ignore-all-space)
|
|
21
|
-
* - tests/** changes (no recursion)
|
|
22
|
-
* - .github/workflows/*.yml surfaced as manual-review-required
|
|
23
|
-
* - skills/<name>/skill.md satisfied by Ed25519 verify gate
|
|
24
|
-
*
|
|
25
|
-
* Exit codes:
|
|
26
|
-
* 0 no uncovered surface (or --warn-only)
|
|
27
|
-
* 1 uncovered surface detected
|
|
28
|
-
* 2 runner error (bad flag, git failure, etc.)
|
|
29
|
-
*
|
|
30
|
-
* Flags:
|
|
31
|
-
* --base <ref> compare HEAD against <ref> (default: origin/main)
|
|
32
|
-
* --staged use the staged index against HEAD
|
|
33
|
-
* --json emit machine-readable report on stdout
|
|
34
|
-
* --warn-only print but never exit non-zero
|
|
35
|
-
* --help, -h this help
|
|
14
|
+
* Exit codes: 0 clean or --warn-only, 1 uncovered surface, 2 runner error.
|
|
36
15
|
*/
|
|
37
16
|
|
|
38
17
|
const fs = require("fs");
|
|
@@ -41,8 +20,6 @@ const childProc = require("child_process");
|
|
|
41
20
|
|
|
42
21
|
const ROOT = path.resolve(__dirname, "..");
|
|
43
22
|
|
|
44
|
-
// --- Flag parsing -----------------------------------------------------------
|
|
45
|
-
|
|
46
23
|
function parseArgs(argv) {
|
|
47
24
|
const out = { base: "origin/main", staged: false, json: false, warnOnly: false };
|
|
48
25
|
for (let i = 0; i < argv.length; i++) {
|
|
@@ -60,15 +37,29 @@ function parseArgs(argv) {
|
|
|
60
37
|
|
|
61
38
|
function printHelp() {
|
|
62
39
|
const banner =
|
|
40
|
+
"check-test-coverage — report changed surface that no test covers.\n" +
|
|
41
|
+
"\n" +
|
|
63
42
|
"Usage: node scripts/check-test-coverage.js [--base <ref>] [--staged]\n" +
|
|
64
43
|
" [--json] [--warn-only]\n" +
|
|
65
44
|
"\n" +
|
|
66
|
-
"
|
|
45
|
+
"Flags:\n" +
|
|
46
|
+
" --base <ref> compare against <ref>..HEAD (default: origin/main)\n" +
|
|
47
|
+
" --staged compare the staged set instead of a ref range\n" +
|
|
48
|
+
" --json emit the result as JSON instead of a text report\n" +
|
|
49
|
+
" --warn-only report findings but always exit 0\n" +
|
|
50
|
+
"\n" +
|
|
51
|
+
"Surfaces checked: CLI verbs and flags in bin/exceptd.js, exported functions\n" +
|
|
52
|
+
"in lib / orchestrator / scripts, playbook detect-indicator and look-artifact\n" +
|
|
53
|
+
"ids, and CVE entries whose iocs changed.\n" +
|
|
54
|
+
"\n" +
|
|
55
|
+
"Categorization: docs, tooling dotfiles, tests/ itself and derived indexes are\n" +
|
|
56
|
+
"allowlisted; workflows, manifests, schemas, SBOM and unclassified files are\n" +
|
|
57
|
+
"surfaced as manual-review, never auto-green.\n" +
|
|
58
|
+
"\n" +
|
|
59
|
+
"Exit codes: 0 clean or --warn-only, 1 uncovered surface, 2 runner error.\n";
|
|
67
60
|
process.stdout.write(banner);
|
|
68
61
|
}
|
|
69
62
|
|
|
70
|
-
// --- Git plumbing -----------------------------------------------------------
|
|
71
|
-
|
|
72
63
|
function git(args, cwd) {
|
|
73
64
|
const r = childProc.spawnSync("git", args, { cwd, encoding: "utf8" });
|
|
74
65
|
if (r.status !== 0) {
|
|
@@ -79,29 +70,14 @@ function git(args, cwd) {
|
|
|
79
70
|
return r.stdout;
|
|
80
71
|
}
|
|
81
72
|
|
|
82
|
-
//
|
|
83
|
-
//
|
|
84
|
-
//
|
|
85
|
-
// `opts.base` ref — so if origin/main advanced past the merge-base between
|
|
86
|
-
// the file-list call and the per-file diff calls, the analyzer compared
|
|
87
|
-
// per-file content against a newer upstream tree than the file list itself
|
|
88
|
-
// was derived from. Result: false "added/removed" surface findings or real
|
|
89
|
-
// findings masked. Codex P1 flag on PR #2 of v0.12.8.
|
|
73
|
+
// The diff anchor resolves ONCE and the resolved SHA threads through every
|
|
74
|
+
// per-file call: passing the raw ref lets origin/main advance mid-run, comparing
|
|
75
|
+
// content against a newer tree than the file list came from.
|
|
90
76
|
function resolveBaseRef(opts, cwd) {
|
|
91
77
|
if (opts.staged) return null; // staged mode uses --cached / HEAD throughout
|
|
92
|
-
//
|
|
93
|
-
//
|
|
94
|
-
//
|
|
95
|
-
// subsequent git invocation, surfacing as a runner-level error. In CI
|
|
96
|
-
// (full clones) the original ref usually resolves; on a developer
|
|
97
|
-
// laptop without `origin/main` configured (fresh clone, detached
|
|
98
|
-
// worktree, alternative remote name) the gate would fail entirely.
|
|
99
|
-
//
|
|
100
|
-
// Order of preference:
|
|
101
|
-
// 1. merge-base against the requested base
|
|
102
|
-
// 2. requested base verbatim, if `git rev-parse --verify` resolves it
|
|
103
|
-
// 3. local `main` HEAD if it exists
|
|
104
|
-
// 4. HEAD~1 as a last resort (single-commit diff)
|
|
78
|
+
// origin/main is not always reachable — fresh clone, detached worktree, remote
|
|
79
|
+
// under another name — and an unresolvable ref fails every later git call as a
|
|
80
|
+
// runner error rather than a coverage result.
|
|
105
81
|
const tryResolve = (ref) => {
|
|
106
82
|
try {
|
|
107
83
|
git(["merge-base", "HEAD", ref], cwd).trim();
|
|
@@ -160,12 +136,9 @@ function fileDiff(opts, file, cwd, ignoreWs, resolvedBase) {
|
|
|
160
136
|
}
|
|
161
137
|
|
|
162
138
|
function fileAtRef(file, ref, cwd) {
|
|
163
|
-
//
|
|
164
|
-
//
|
|
165
|
-
//
|
|
166
|
-
// return is the documented "missing" sentinel — silent truncation
|
|
167
|
-
// would make every entry in the live file appear as a fresh add and
|
|
168
|
-
// generate hundreds of bogus diff-coverage findings.
|
|
139
|
+
// maxBuffer far above Node's 1 MiB default: an ENOBUFS-truncated read of a
|
|
140
|
+
// multi-MiB catalog makes every entry in the live file look freshly added.
|
|
141
|
+
// Null is the "missing" sentinel, so a failure never returns partial content.
|
|
169
142
|
const r = childProc.spawnSync("git", ["show", ref + ":" + file], {
|
|
170
143
|
cwd, encoding: "utf8", maxBuffer: 64 * 1024 * 1024
|
|
171
144
|
});
|
|
@@ -190,23 +163,14 @@ function readMaybe(p) {
|
|
|
190
163
|
try { return fs.readFileSync(p, "utf8"); } catch { return null; }
|
|
191
164
|
}
|
|
192
165
|
|
|
193
|
-
//
|
|
194
|
-
|
|
195
|
-
// Mechanical / contributor-only docs the gate auto-allows: their content
|
|
196
|
-
// has no operator-facing semantic surface (CONTRIBUTING is for PRs;
|
|
197
|
-
// LICENSE / NOTICE / CODE_OF_CONDUCT are boilerplate; .gitignore / .npmrc
|
|
198
|
-
// / .editorconfig are tooling). Edits here never need a regression test.
|
|
166
|
+
// Contributor-only docs and tooling dotfiles: no semantic surface to test.
|
|
199
167
|
const DOCS_ALWAYS_GREEN = new Set([
|
|
200
168
|
"CONTRIBUTING.md", "LICENSE", "NOTICE", "CODE_OF_CONDUCT.md",
|
|
201
169
|
"SUPPORT.md", ".gitignore", ".npmrc", ".editorconfig",
|
|
202
170
|
]);
|
|
203
171
|
|
|
204
|
-
// Operator-facing docs
|
|
205
|
-
//
|
|
206
|
-
// auto-green — a PR could otherwise land deceptive copy here without any
|
|
207
|
-
// reviewer signal. Downgrade to manual-review so the diff surfaces in the
|
|
208
|
-
// gate output — a human (or the maintainer reviewing the bot summary) at
|
|
209
|
-
// least sees the change exists.
|
|
172
|
+
// Operator-facing docs must not auto-green — a PR could otherwise land
|
|
173
|
+
// deceptive copy with no reviewer signal — so they downgrade to manual-review.
|
|
210
174
|
const DOCS_MANUAL_REVIEW = new Set([
|
|
211
175
|
"CHANGELOG.md", "README.md", "SECURITY.md", "MIGRATING.md", "AGENTS.md",
|
|
212
176
|
]);
|
|
@@ -226,20 +190,14 @@ function categorize(file) {
|
|
|
226
190
|
if (norm.startsWith("scripts/") && norm.endsWith(".js")) return "lib";
|
|
227
191
|
if (norm.startsWith("data/playbooks/") && norm.endsWith(".json")) return "playbook";
|
|
228
192
|
if (norm === "data/cve-catalog.json") return "cve-catalog";
|
|
229
|
-
//
|
|
230
|
-
//
|
|
231
|
-
// schemas/*, data/*.json, sbom.cdx.json, manifest-snapshot.*) can carry
|
|
232
|
-
// semantic surface but the analyzer has no syntactic surface extractor
|
|
233
|
-
// for them — humans should look.
|
|
193
|
+
// Shapes carrying semantic surface the analyzer has no extractor for: a human
|
|
194
|
+
// looks, instead of an allowlist.
|
|
234
195
|
if (norm === "manifest.json") return "manual-review";
|
|
235
196
|
if (norm === "manifest-snapshot.json") return "manual-review";
|
|
236
197
|
if (norm === "manifest-snapshot.sha256") return "manual-review";
|
|
237
198
|
if (norm === "sbom.cdx.json") return "manual-review";
|
|
238
199
|
if (norm.startsWith("lib/schemas/")) return "manual-review";
|
|
239
|
-
//
|
|
240
|
-
// `npm run build-indexes`; the source-of-truth diff is in the data/
|
|
241
|
-
// files themselves. Allowlist the derived index files so they don't
|
|
242
|
-
// perpetually surface as manual-review on every release commit.
|
|
200
|
+
// data/_indexes/ is regenerated; the reviewable diff is in the data/ sources.
|
|
243
201
|
if (norm.startsWith("data/_indexes/")) return "allowlist-derived";
|
|
244
202
|
if (norm.startsWith("data/") && norm.endsWith(".json")) return "manual-review";
|
|
245
203
|
if (norm === "package.json") return "manual-review";
|
|
@@ -252,14 +210,11 @@ function isWhitespaceOnly(opts, file, cwd, resolvedBase) {
|
|
|
252
210
|
.filter(l => !l.startsWith("+++") && !l.startsWith("---")).length === 0;
|
|
253
211
|
}
|
|
254
212
|
|
|
255
|
-
// --- Surface extraction -----------------------------------------------------
|
|
256
|
-
|
|
257
213
|
function extractCliSurface(content) {
|
|
258
214
|
if (!content) return { verbs: new Set(), flags: new Set() };
|
|
259
215
|
const verbs = new Set();
|
|
260
216
|
const flags = new Set();
|
|
261
|
-
// Only
|
|
262
|
-
// picking up arbitrary keys from elsewhere.
|
|
217
|
+
// Only the COMMANDS block and PLAYBOOK_VERBS Set, not arbitrary keys elsewhere.
|
|
263
218
|
const cmdBlock = content.match(/const COMMANDS = \{([\s\S]*?)\n\};/);
|
|
264
219
|
if (cmdBlock) {
|
|
265
220
|
const re = /^\s*"?([a-zA-Z][\w-]+)"?\s*:/gm;
|
|
@@ -272,18 +227,22 @@ function extractCliSurface(content) {
|
|
|
272
227
|
let m;
|
|
273
228
|
while ((m = re.exec(playbookBlock[1])) !== null) verbs.add(m[1]);
|
|
274
229
|
}
|
|
275
|
-
// REMOVED_VERBS keys are still
|
|
276
|
-
//
|
|
277
|
-
//
|
|
278
|
-
// dropped — otherwise removing dead COMMANDS table rows for already-retired
|
|
279
|
-
// verbs reads as a fresh "removed-but-test-remains" against the refusal test.
|
|
230
|
+
// REMOVED_VERBS keys are still CLI surface: invoking one returns a structured
|
|
231
|
+
// refusal that tests cover. Counting them stops a dropped COMMANDS row reading
|
|
232
|
+
// as a fresh "removed-but-test-remains" against the refusal test.
|
|
280
233
|
const removedBlock = content.match(/const REMOVED_VERBS = \{([\s\S]*?)\n\};/);
|
|
281
234
|
if (removedBlock) {
|
|
282
235
|
for (const m of removedBlock[1].matchAll(/^\s*"?([a-zA-Z][\w-]+)"?\s*:/gm)) verbs.add(m[1]);
|
|
283
236
|
}
|
|
237
|
+
// Scans the whole file, prose included, so a flag named in a comment counts as
|
|
238
|
+
// surface. A trailing hyphen means the match stopped at a line break mid-name
|
|
239
|
+
// (`--attest-` wrapping to `ownership`), which is never a flag — admitting one
|
|
240
|
+
// makes deleting that comment read as a removed flag.
|
|
284
241
|
const flagRe = /(--[a-zA-Z][\w-]+)/g;
|
|
285
242
|
let m;
|
|
286
|
-
while ((m = flagRe.exec(content)) !== null)
|
|
243
|
+
while ((m = flagRe.exec(content)) !== null) {
|
|
244
|
+
if (!m[1].endsWith("-")) flags.add(m[1]);
|
|
245
|
+
}
|
|
287
246
|
for (const f of ["--help", "--version"]) flags.delete(f);
|
|
288
247
|
return { verbs, flags };
|
|
289
248
|
}
|
|
@@ -299,27 +258,18 @@ function diffSets(before, after) {
|
|
|
299
258
|
function extractLibExports(content) {
|
|
300
259
|
if (!content) return new Set();
|
|
301
260
|
const out = new Set();
|
|
302
|
-
//
|
|
303
|
-
//
|
|
304
|
-
// block does not shadow the real exports lower in the file. Pre-fix, the
|
|
305
|
-
// analyzer's own file matched a 3-char doc-comment fragment first and
|
|
306
|
-
// returned an empty export set — any source that mentions `module.exports`
|
|
307
|
-
// in a JSDoc/banner block hit the same bug. After stripping comments,
|
|
308
|
-
// the `module.exports = {...}` match runs against real code only.
|
|
261
|
+
// Strip comments first: a `module.exports = {...}` inside a doc comment
|
|
262
|
+
// otherwise shadows the real exports and the export set comes back empty.
|
|
309
263
|
const stripped = content
|
|
310
264
|
.replace(/\/\*[\s\S]*?\*\//g, "")
|
|
311
265
|
.replace(/^\s*\/\/.*$/gm, "");
|
|
312
|
-
//
|
|
313
|
-
//
|
|
314
|
-
// truncate the export list at the first inner `}` and hide later exports —
|
|
315
|
-
// which would let an uncovered new export ship green (the gate's blind spot).
|
|
266
|
+
// Brace-balanced so a nested member (`{ CONFIG: { a: 1 }, x, y }`) does not
|
|
267
|
+
// truncate the list at the first inner `}` and hide later exports.
|
|
316
268
|
const objStart = stripped.search(/module\.exports\s*=\s*\{/);
|
|
317
269
|
if (objStart !== -1) {
|
|
318
270
|
const openIdx = stripped.indexOf("{", objStart);
|
|
319
|
-
// String-aware
|
|
320
|
-
//
|
|
321
|
-
// hide the exports that follow — that blind spot let an uncovered export
|
|
322
|
-
// ship green.
|
|
271
|
+
// String-aware: a `}` inside a value (`{ PATTERN: "a}b", realExport }`)
|
|
272
|
+
// must not close the object early and hide the exports that follow.
|
|
323
273
|
let depth = 0, end = -1, inStr = null;
|
|
324
274
|
for (let i = openIdx; i < stripped.length; i++) {
|
|
325
275
|
const ch = stripped[i];
|
|
@@ -334,8 +284,7 @@ function extractLibExports(content) {
|
|
|
334
284
|
}
|
|
335
285
|
if (end !== -1) {
|
|
336
286
|
const body = stripped.slice(openIdx + 1, end);
|
|
337
|
-
// String-aware
|
|
338
|
-
// not split a member or skew the bracket depth.
|
|
287
|
+
// String-aware split: a `,` or bracket inside a string must not split a member.
|
|
339
288
|
let d = 0, cur = "", sInStr = null;
|
|
340
289
|
const members = [];
|
|
341
290
|
for (let i = 0; i < body.length; i++) {
|
|
@@ -356,12 +305,9 @@ function extractLibExports(content) {
|
|
|
356
305
|
for (const tok of members) {
|
|
357
306
|
const id = tok.split(":")[0].trim();
|
|
358
307
|
if (/^[a-zA-Z_$][\w$]*$/.test(id)) { out.add(id); continue; }
|
|
359
|
-
// Method-shorthand member (`fn(a){
|
|
360
|
-
//
|
|
361
|
-
//
|
|
362
|
-
// new exported method would ship with no diff-coverage requirement.
|
|
363
|
-
// Recover the name as the identifier immediately before the first `(`,
|
|
364
|
-
// after any leading modifier keyword (async/get/set) or generator `*`.
|
|
308
|
+
// Method-shorthand member (`fn(a){}`, `async load(){}`, `*gen(){}`): the
|
|
309
|
+
// colon split leaves `name(...)`, so recover the name before the first
|
|
310
|
+
// `(`, past any modifier keyword or generator `*`.
|
|
365
311
|
const beforeParen = tok.split("(")[0].trim();
|
|
366
312
|
const parts = beforeParen.split(/\s+/);
|
|
367
313
|
const cand = parts[parts.length - 1].replace(/^\*/, "").trim();
|
|
@@ -369,12 +315,8 @@ function extractLibExports(content) {
|
|
|
369
315
|
}
|
|
370
316
|
}
|
|
371
317
|
}
|
|
372
|
-
//
|
|
373
|
-
//
|
|
374
|
-
// extractor above matches nothing, so without this the file's only export is
|
|
375
|
-
// invisible to the gate and a change to it requires no test. Capture the bare
|
|
376
|
-
// identifier on the RHS of `module.exports =` when it is not an object/array/
|
|
377
|
-
// function-expression/arrow (those are handled elsewhere or have no name).
|
|
318
|
+
// Whole-module export (`module.exports = mainFn;`): the object extractor above
|
|
319
|
+
// matches nothing, so without this the file's only export is invisible.
|
|
378
320
|
const singleIdent = stripped.match(/module\.exports\s*=\s*([a-zA-Z_$][\w$]*)\s*;/);
|
|
379
321
|
if (singleIdent && !/^(function|async|class)$/.test(singleIdent[1])) {
|
|
380
322
|
out.add(singleIdent[1]);
|
|
@@ -401,13 +343,8 @@ function extractPlaybookIds(content) {
|
|
|
401
343
|
return { indicators: ind, artifacts: arts };
|
|
402
344
|
}
|
|
403
345
|
|
|
404
|
-
// Canonical
|
|
405
|
-
//
|
|
406
|
-
// JSON.stringify(after.iocs) — non-canonical: key order, trailing
|
|
407
|
-
// whitespace, and numeric format differences all flagged as "changed"
|
|
408
|
-
// when the operator made no semantic change. Symptoms were patched
|
|
409
|
-
// twice with skip rules (_auto_imported, _iocs_stub) instead of fixing
|
|
410
|
-
// the comparator. v0.13.20 fixes the root cause.
|
|
346
|
+
// Canonical equality, not JSON.stringify: key order, whitespace and numeric
|
|
347
|
+
// formatting are not semantic changes to an iocs block.
|
|
411
348
|
const { canonicalEqual } = require("../lib/canonical-eq");
|
|
412
349
|
|
|
413
350
|
function extractCveIocChanges(beforeStr, afterStr) {
|
|
@@ -417,10 +354,8 @@ function extractCveIocChanges(beforeStr, afterStr) {
|
|
|
417
354
|
const ids = new Set([...Object.keys(before), ...Object.keys(after)]);
|
|
418
355
|
for (const id of ids) {
|
|
419
356
|
if (!/^CVE-\d{4}-\d+/.test(id)) continue;
|
|
420
|
-
//
|
|
421
|
-
//
|
|
422
|
-
// operator curation. Removing this would surface every fresh KEV
|
|
423
|
-
// bulk-import as a per-CVE iocs-modified finding.
|
|
357
|
+
// Rows auto-imported on BOTH sides hold stub IoCs by design; without the
|
|
358
|
+
// skip, every fresh KEV bulk-import surfaces as a per-CVE finding.
|
|
424
359
|
const beforeAuto = !!(before[id] && before[id]._auto_imported);
|
|
425
360
|
const afterAuto = !!(after[id] && after[id]._auto_imported);
|
|
426
361
|
if (beforeAuto && afterAuto) continue;
|
|
@@ -433,8 +368,6 @@ function extractCveIocChanges(beforeStr, afterStr) {
|
|
|
433
368
|
|
|
434
369
|
function safeParse(s) { try { return s ? JSON.parse(s) : null; } catch { return null; } }
|
|
435
370
|
|
|
436
|
-
// --- Test corpus + coverage probes ------------------------------------------
|
|
437
|
-
|
|
438
371
|
function loadTestCorpus(cwd) {
|
|
439
372
|
const root = path.join(cwd, "tests");
|
|
440
373
|
if (!fs.existsSync(root)) return { joined: "", files: [] };
|
|
@@ -473,32 +406,21 @@ function coversCliFlag(corpus, flag) {
|
|
|
473
406
|
return corpus.includes(flag);
|
|
474
407
|
}
|
|
475
408
|
|
|
476
|
-
//
|
|
477
|
-
//
|
|
478
|
-
//
|
|
479
|
-
//
|
|
480
|
-
// Pre-fix: an `assert.equal(...)` mention in one test file plus a stray
|
|
481
|
-
// `require('../lib/x')` in a completely different test file counted as
|
|
482
|
-
// coverage. That's not coverage — it's textual coincidence.
|
|
483
|
-
//
|
|
484
|
-
// `corpus` may be either a string (legacy joined corpus, used by
|
|
485
|
-
// CLI/playbook/CVE coverage probes) or the structured shape
|
|
486
|
-
// `{ joined, files }` produced by loadTestCorpus().
|
|
409
|
+
// Coverage needs same-file context: the identifier must appear inside a real
|
|
410
|
+
// test block in the SAME file that requires the module, or a stray require() in
|
|
411
|
+
// one file plus a mention in another reads as coverage. `corpus` is either the
|
|
412
|
+
// structured `{ joined, files }` from loadTestCorpus or a legacy joined string.
|
|
487
413
|
function coversLibExport(corpus, libRel, ident) {
|
|
488
414
|
const baseName = path.basename(libRel).replace(/\.js$/, "");
|
|
489
|
-
const baseFile = path.basename(libRel);
|
|
415
|
+
const baseFile = path.basename(libRel);
|
|
490
416
|
const identRe = new RegExp("\\b" + escapeRe(ident) + "\\b");
|
|
491
417
|
const requireRe = new RegExp("require\\([^)]*" + escapeRe(baseName) + "[^)]*\\)");
|
|
492
|
-
// Accept the structured shape (preferred). Walk files individually.
|
|
493
418
|
if (corpus && Array.isArray(corpus.files)) {
|
|
494
419
|
for (const f of corpus.files) {
|
|
495
420
|
const hasRequire = requireRe.test(f.content);
|
|
496
421
|
const mentionsSpawnPath = f.content.includes(baseFile);
|
|
497
422
|
if (!hasRequire && !mentionsSpawnPath) continue;
|
|
498
423
|
if (!identRe.test(f.content)) continue;
|
|
499
|
-
// F10 — require the identifier appears inside a test block in this
|
|
500
|
-
// file. Recognise `test(`, `it(`, `describe(`, or `assert(` (or any
|
|
501
|
-
// `assert.<member>(`) bracketed argument that mentions the ident.
|
|
502
424
|
if (mentionsIdentInTestContext(f.content, ident)) return true;
|
|
503
425
|
}
|
|
504
426
|
return false;
|
|
@@ -510,16 +432,11 @@ function coversLibExport(corpus, libRel, ident) {
|
|
|
510
432
|
return false;
|
|
511
433
|
}
|
|
512
434
|
|
|
513
|
-
//
|
|
514
|
-
// `test(
|
|
515
|
-
//
|
|
516
|
-
// the call" by finding the opening paren after the keyword, then walking
|
|
517
|
-
// matched parens until the call closes. This is a syntactic-enough check
|
|
518
|
-
// for vanilla JavaScript tests; the goal is to refuse "ident only appears
|
|
519
|
-
// in a top-level comment" while still accepting `assert.deepEqual(foo, ...)`.
|
|
435
|
+
// True when `ident` appears as a token inside the parenthesised body of a
|
|
436
|
+
// `test(`, `it(`, `describe(`, `assert(` or `assert.<member>(` call. The paren
|
|
437
|
+
// walk is approximate by design.
|
|
520
438
|
function mentionsIdentInTestContext(content, ident) {
|
|
521
439
|
const tokenRe = new RegExp("\\b" + escapeRe(ident) + "\\b");
|
|
522
|
-
// Quick reject: file does not mention the identifier at all.
|
|
523
440
|
if (!tokenRe.test(content)) return false;
|
|
524
441
|
const callRe = /\b(test|it|describe|assert(?:\.[A-Za-z_$][\w$]*)?)\s*\(/g;
|
|
525
442
|
let m;
|
|
@@ -556,33 +473,15 @@ function coversCveIoc(corpus, cveId) {
|
|
|
556
473
|
return /\biocs\b/i.test(corpus);
|
|
557
474
|
}
|
|
558
475
|
|
|
559
|
-
//
|
|
560
|
-
|
|
561
|
-
//
|
|
562
|
-
//
|
|
563
|
-
// Anti-coincidence rule: every exit-code assertion must pin the
|
|
564
|
-
// EXACT code. `assert.notEqual(r.status, 0)` silently passes when an
|
|
565
|
-
// unrelated failure produces ANY non-zero exit, hiding the regression the
|
|
566
|
-
// test was meant to catch. This lint walks tests/*.test.js and rejects the
|
|
567
|
-
// pattern outright. The `// allow-notEqual: <reason>` opt-out on the same
|
|
568
|
-
// line is the escape hatch for genuine refusal-pins (asserting NOT a
|
|
569
|
-
// specific code) — those must justify themselves inline.
|
|
570
|
-
//
|
|
571
|
-
// Pattern hits any of:
|
|
572
|
-
// assert.notEqual(r.status, 0)
|
|
573
|
-
// assert.notEqual(result.status, 0, '...')
|
|
574
|
-
// assert.notEqual(foo.status, 2, 'must not be unknown-cmd') ← also refused
|
|
575
|
-
// unless the same line ends with `// allow-notEqual: <reason>`.
|
|
576
|
-
//
|
|
577
|
-
// Structural lint replaces a per-instance hunt across 25+ test sites — keeps
|
|
578
|
-
// new tests / new ports from regressing into coincidence-passing assertions.
|
|
579
|
-
// Fix the class, not the instance.
|
|
476
|
+
// Every exit-code assertion must pin the EXACT code: `assert.notEqual(r.status,
|
|
477
|
+
// 0)` passes on any non-zero exit, hiding the regression the test was written
|
|
478
|
+
// for. The only opt-out is `// allow-notEqual: <reason>` on the same line, for a
|
|
479
|
+
// genuine refusal-pin that asserts NOT a specific code.
|
|
580
480
|
function scanForCoincidenceAsserts(cwd) {
|
|
581
481
|
const out = [];
|
|
582
482
|
const testsDir = path.join(cwd, "tests");
|
|
583
483
|
if (!fs.existsSync(testsDir)) return out;
|
|
584
|
-
//
|
|
585
|
-
// (r, r1, result, child, etc.) but the .status access is the signal.
|
|
484
|
+
// The receiver name varies (r, r1, result, child); the `.status` access is the signal.
|
|
586
485
|
const banRe = /assert\.notEqual\s*\(\s*[A-Za-z_$][\w$]*\.status\b/;
|
|
587
486
|
const allowRe = /\/\/\s*allow-notEqual\s*:/;
|
|
588
487
|
const skipPrefix = "_helpers"; // helpers may legitimately reference the pattern
|
|
@@ -611,10 +510,6 @@ function scanForCoincidenceAsserts(cwd) {
|
|
|
611
510
|
|
|
612
511
|
function analyze(opts) {
|
|
613
512
|
const cwd = opts.repo || ROOT;
|
|
614
|
-
// v0.12.8: resolve the diff anchor ONCE and thread it through every
|
|
615
|
-
// per-file call so listChangedFiles + fileDiff + fileBefore all agree on
|
|
616
|
-
// the same SHA. Otherwise origin/main advancing past the merge-base
|
|
617
|
-
// between calls produces false add/remove findings.
|
|
618
513
|
const resolvedBase = resolveBaseRef(opts, cwd);
|
|
619
514
|
const changed = listChangedFiles(opts, cwd, resolvedBase);
|
|
620
515
|
const corpusObj = loadTestCorpus(cwd);
|
|
@@ -632,10 +527,7 @@ function analyze(opts) {
|
|
|
632
527
|
}
|
|
633
528
|
if (cat === "skill") { allowlisted.push({ file: ch.file, reason: "skill-signed" }); continue; }
|
|
634
529
|
if (cat === "workflow") { manualReview.push({ file: ch.file, reason: "workflow" }); continue; }
|
|
635
|
-
// F11 — data catalogs, schemas, manifests, SBOM go to manual review
|
|
636
|
-
// instead of being silently allowlisted. They show up in CI output.
|
|
637
530
|
if (cat === "manual-review") { manualReview.push({ file: ch.file, reason: "manual-review" }); continue; }
|
|
638
|
-
// v0.12.14: derived index files allowlist (auto-regenerated artifacts).
|
|
639
531
|
if (cat === "allowlist-derived") { allowlisted.push({ file: ch.file, reason: "derived-artifact" }); continue; }
|
|
640
532
|
if (cat === "other") { manualReview.push({ file: ch.file, reason: "unclassified" }); continue; }
|
|
641
533
|
if (ch.status !== "D" && isWhitespaceOnly(opts, ch.file, cwd, resolvedBase)) {
|
|
@@ -663,8 +555,6 @@ function analyze(opts) {
|
|
|
663
555
|
const b = extractLibExports(before);
|
|
664
556
|
const a = extractLibExports(after);
|
|
665
557
|
const d = diffSets(b, a);
|
|
666
|
-
// F10 — pass the structured corpus so coversLibExport can enforce
|
|
667
|
-
// same-file require()+identifier-in-test-context coverage.
|
|
668
558
|
for (const id of d.added) if (!coversLibExport(corpusObj, ch.file, id))
|
|
669
559
|
findings.push({ file: ch.file, kind: "lib-export", surface: id, change: "added" });
|
|
670
560
|
for (const id of d.removed) if (coversLibExport(corpusObj, ch.file, id))
|
|
@@ -687,11 +577,8 @@ function analyze(opts) {
|
|
|
687
577
|
}
|
|
688
578
|
}
|
|
689
579
|
|
|
690
|
-
//
|
|
691
|
-
//
|
|
692
|
-
// a coincidence-passing assert that lands via a non-test-coverage
|
|
693
|
-
// path (someone hand-edits a tests/ file in a docs-only commit) is
|
|
694
|
-
// still a regression vector the gate must catch.
|
|
580
|
+
// Runs irrespective of the diff: a coincidence-passing assert can land by a
|
|
581
|
+
// path this analyzer never inspects.
|
|
695
582
|
const coincidenceFindings = scanForCoincidenceAsserts(cwd);
|
|
696
583
|
for (const f of coincidenceFindings) {
|
|
697
584
|
findings.push({
|
|
@@ -705,8 +592,6 @@ function analyze(opts) {
|
|
|
705
592
|
return { findings, allowlisted, manualReview, totalChanged: changed.length };
|
|
706
593
|
}
|
|
707
594
|
|
|
708
|
-
// --- Output -----------------------------------------------------------------
|
|
709
|
-
|
|
710
595
|
function emitHuman(report) {
|
|
711
596
|
const out = [];
|
|
712
597
|
out.push("Diff coverage analyzer — " + report.totalChanged + " changed file(s)");
|