@blamejs/exceptd-skills 0.19.32 → 0.19.34
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +22 -0
- package/bin/exceptd.js +896 -2824
- package/data/_indexes/_meta.json +8 -8
- package/data/_indexes/activity-feed.json +2 -2
- package/data/_indexes/catalog-summaries.json +7 -7
- package/data/_indexes/chains.json +60118 -0
- package/data/attack-techniques.json +267 -7
- package/data/cve-catalog.json +9991 -3
- package/data/cwe-catalog.json +109 -2
- package/data/framework-control-gaps.json +578 -3
- package/data/zeroday-lessons.json +8330 -1
- package/lib/auto-discovery.js +56 -286
- package/lib/canonical-eq.js +7 -40
- package/lib/citation-resolve.js +22 -70
- package/lib/collectors/ai-api.js +20 -54
- package/lib/collectors/cicd-pipeline-compromise.js +40 -108
- package/lib/collectors/citation-hygiene.js +72 -210
- package/lib/collectors/containers.js +41 -130
- package/lib/collectors/cred-stores.js +31 -115
- package/lib/collectors/crypto-codebase.js +55 -138
- package/lib/collectors/crypto.js +24 -54
- package/lib/collectors/hardening.js +20 -78
- package/lib/collectors/kernel.js +16 -46
- package/lib/collectors/library-author.js +57 -206
- package/lib/collectors/mcp.js +24 -70
- package/lib/collectors/runtime.js +24 -86
- package/lib/collectors/sbom.js +34 -106
- package/lib/collectors/scan-excludes.js +31 -138
- package/lib/collectors/secrets.js +62 -178
- package/lib/cross-ref-api.js +39 -123
- package/lib/currency-severity.js +8 -27
- package/lib/cve-batch.js +13 -21
- package/lib/cve-cli.js +13 -20
- package/lib/cve-curation.js +72 -239
- package/lib/cve-regression-watcher.js +29 -152
- package/lib/cvss.js +13 -54
- package/lib/doctor-bucketing.js +3 -19
- package/lib/exit-codes.js +10 -42
- package/lib/flag-suggest.js +7 -25
- package/lib/framework-gap.js +35 -114
- package/lib/gap-detectors.js +37 -159
- package/lib/id-validation.js +9 -30
- package/lib/job-queue.js +13 -36
- package/lib/lint-skills.js +64 -232
- package/lib/playbook-runner.js +693 -2095
- package/lib/prefetch.js +100 -376
- package/lib/refresh-external.js +199 -627
- package/lib/refresh-network.js +75 -307
- package/lib/rfc-cli.js +23 -68
- package/lib/scoring.js +77 -145
- package/lib/sign.js +43 -229
- package/lib/source-advisories.js +43 -194
- package/lib/source-ghsa.js +37 -120
- package/lib/source-osv.js +94 -266
- package/lib/ttp-mapper.js +14 -24
- package/lib/upstream-check-cli.js +10 -28
- package/lib/upstream-check.js +19 -44
- package/lib/validate-catalog-meta.js +17 -61
- package/lib/validate-cve-catalog.js +43 -119
- package/lib/validate-indexes.js +25 -76
- package/lib/validate-package.js +16 -62
- package/lib/validate-playbooks.js +69 -275
- package/lib/validate-vendor.js +16 -49
- package/lib/verify.js +56 -286
- package/lib/version-pins.js +5 -34
- package/lib/worker-pool.js +11 -30
- package/lib/xml-tokenizer.js +47 -152
- package/manifest.json +53 -53
- package/orchestrator/dispatcher.js +17 -68
- package/orchestrator/event-bus.js +11 -74
- package/orchestrator/index.js +138 -412
- package/orchestrator/pipeline.js +28 -85
- package/orchestrator/scanner.js +34 -138
- package/orchestrator/scheduler.js +20 -84
- package/package.json +2 -2
- package/sbom.cdx.json +253 -253
- package/scripts/audit-catalog-gaps.js +9 -62
- package/scripts/audit-cross-skill.js +5 -31
- package/scripts/audit-perf.js +6 -16
- package/scripts/backfill-theater-test.js +7 -64
- package/scripts/bootstrap.js +12 -44
- package/scripts/build-indexes.js +40 -154
- package/scripts/builders/activity-feed.js +4 -14
- package/scripts/builders/catalog-summaries.js +3 -10
- package/scripts/builders/currency.js +7 -20
- package/scripts/builders/cwe-chains.js +7 -30
- package/scripts/builders/did-ladders.js +6 -13
- package/scripts/builders/frequency.js +5 -19
- package/scripts/builders/jurisdiction-clocks.js +6 -25
- package/scripts/builders/recipes.js +6 -14
- package/scripts/builders/section-offsets.js +13 -51
- package/scripts/builders/stale-content.js +7 -28
- package/scripts/builders/summary-cards.js +8 -29
- package/scripts/builders/theater-fingerprints.js +12 -27
- package/scripts/builders/token-budget.js +4 -31
- package/scripts/check-agents-md-collectors.js +11 -54
- package/scripts/check-catalog-gap-budget.js +15 -32
- package/scripts/check-changelog-extract.js +18 -48
- package/scripts/check-codebase-patterns-currency.js +6 -22
- package/scripts/check-codebase-patterns.js +50 -143
- package/scripts/check-epss-consistency.js +9 -64
- package/scripts/check-framework-gap-coverage.js +13 -31
- package/scripts/check-manifest-snapshot.js +13 -73
- package/scripts/check-sbom-currency.js +44 -142
- package/scripts/check-test-count.js +15 -52
- package/scripts/check-test-coverage.js +66 -197
- package/scripts/check-test-subjects.js +21 -62
- package/scripts/check-ttp-references.js +14 -38
- package/scripts/check-ttp-upstream.js +8 -40
- package/scripts/check-version-bump.js +9 -61
- package/scripts/check-version-tags.js +20 -121
- package/scripts/predeploy.js +38 -184
- package/scripts/refresh-manifest-snapshot.js +16 -38
- package/scripts/refresh-mitre-atlas.js +3 -8
- package/scripts/refresh-mitre-attack.js +1 -8
- package/scripts/refresh-mitre-d3fend.js +3 -9
- package/scripts/refresh-mitre-ics-attack.js +3 -8
- package/scripts/refresh-reverse-refs.js +27 -94
- package/scripts/refresh-rfc-index.js +2 -10
- package/scripts/refresh-sbom.js +31 -161
- package/scripts/refresh-upstream-catalogs.js +40 -137
- package/scripts/release.js +69 -232
- package/scripts/run-e2e-scenarios.js +24 -71
- package/scripts/sync-manifest-metadata.js +10 -34
- package/scripts/sync-package-description.js +8 -17
- package/scripts/validate-vendor-online.js +13 -44
- package/scripts/verify-shipped-tarball.js +35 -140
|
@@ -1,24 +1,10 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
/*
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
* Also enforces rule #10 (no placeholder data) by validating each CVE entry
|
|
10
|
-
* against the structural rules expressed in
|
|
11
|
-
* lib/schemas/cve-catalog.schema.json — required fields, types, enums,
|
|
12
|
-
* patterns, range constraints. Implemented as an inline validator (no AJV /
|
|
13
|
-
* external deps; Node 24 stdlib only) covering the schema features that
|
|
14
|
-
* file actually uses.
|
|
15
|
-
*
|
|
16
|
-
* Usage:
|
|
17
|
-
* node lib/validate-cve-catalog.js validate the full catalog
|
|
18
|
-
* node lib/validate-cve-catalog.js --quiet only print failures + summary
|
|
19
|
-
*
|
|
20
|
-
* Exit code: 0 if every CVE validates and has a matching zeroday lesson,
|
|
21
|
-
* 1 otherwise, 2 on argv error.
|
|
3
|
+
* CVE catalog validator. Every CVE in data/cve-catalog.json must carry a matching
|
|
4
|
+
* key in data/zeroday-lessons.json (rule #6) and satisfy
|
|
5
|
+
* lib/schemas/cve-catalog.schema.json, checked by the inline validator below.
|
|
6
|
+
* Exits 0 when every CVE validates and pairs with a lesson, 1 otherwise, 2 on an
|
|
7
|
+
* argv error. --strict promotes the advisory warnings to errors.
|
|
22
8
|
*/
|
|
23
9
|
|
|
24
10
|
'use strict';
|
|
@@ -38,16 +24,10 @@ const ATTACK_PATH = path.join(REPO_ROOT, 'data', 'attack-techniques.json');
|
|
|
38
24
|
const D3FEND_PATH = path.join(REPO_ROOT, 'data', 'd3fend-catalog.json');
|
|
39
25
|
const FRAMEWORK_GAPS_PATH = path.join(REPO_ROOT, 'data', 'framework-control-gaps.json');
|
|
40
26
|
|
|
41
|
-
//
|
|
42
|
-
//
|
|
43
|
-
//
|
|
44
|
-
//
|
|
45
|
-
// break patch-class compatibility; promoted to an error under --strict.
|
|
46
|
-
// The host-based entries anchor on scheme + host so a lookalike that merely
|
|
47
|
-
// embeds the host as a label (`exploit-db.com.attacker.example`,
|
|
48
|
-
// `not-exploit-db.com`, `evil.com/?ref=packetstormsecurity.com`) is no longer
|
|
49
|
-
// misclassified as a public-exploit source. Verified zero classification
|
|
50
|
-
// change across the live catalog's verification URLs.
|
|
27
|
+
// A verification_sources URL matching one of these marks a public exploit or PoC
|
|
28
|
+
// location; with poc_available true the entry must then carry an `iocs` block
|
|
29
|
+
// (Hard Rule #14). The host patterns anchor on scheme + host, so a lookalike
|
|
30
|
+
// embedding the host as a label does not match.
|
|
51
31
|
const PUBLIC_EXPLOIT_URL_PATTERNS = [
|
|
52
32
|
/^https?:\/\/(?:[\w-]+\.)*github\.com\/.+\/(?:exploits?|pocs?)\b/i,
|
|
53
33
|
/^https?:\/\/(?:[\w-]+\.)*exploit-?db\.com(?:[/:?#]|$)/i,
|
|
@@ -57,14 +37,11 @@ const PUBLIC_EXPLOIT_URL_PATTERNS = [
|
|
|
57
37
|
/-poc\b/i,
|
|
58
38
|
];
|
|
59
39
|
|
|
60
|
-
//
|
|
61
|
-
//
|
|
62
|
-
// versions (2.0 / 3.0 / 3.1 / 4.0). Emitted as a warning by default;
|
|
63
|
-
// promoted to an error under --strict.
|
|
40
|
+
// The schema's own pattern accepts any "CVSS:<digits>/"; this admits only known
|
|
41
|
+
// CVSS versions. A warning by default, an error under --strict.
|
|
64
42
|
const STRICT_CVSS_PATTERN = /^CVSS:(2\.0|3\.[01]|4\.0)\//;
|
|
65
43
|
|
|
66
|
-
//
|
|
67
|
-
// (typos like 1014 or 20262) without rejecting legitimate ISO dates.
|
|
44
|
+
// Wide enough for any legitimate ISO date, narrow enough to reject a year typo.
|
|
68
45
|
const MIN_VALID_YEAR = 1990;
|
|
69
46
|
const MAX_VALID_YEAR = 2100;
|
|
70
47
|
const DATE_FIELDS = [
|
|
@@ -116,10 +93,9 @@ function typeMatches(value, expected) {
|
|
|
116
93
|
return actual === expected;
|
|
117
94
|
}
|
|
118
95
|
|
|
119
|
-
/*
|
|
120
|
-
*
|
|
121
|
-
*
|
|
122
|
-
* minProperties, enum, format=uri (loose check). */
|
|
96
|
+
/* Handles the JSON Schema subset cve-catalog.schema.json uses: type, required,
|
|
97
|
+
* properties, additionalProperties, items, pattern, minLength, minimum, maximum,
|
|
98
|
+
* minItems, minProperties, enum, format=uri. Any other keyword is ignored. */
|
|
123
99
|
function validate(value, schema, schemaName, pathStr) {
|
|
124
100
|
const errors = [];
|
|
125
101
|
const here = pathStr || schemaName;
|
|
@@ -220,11 +196,8 @@ function isUsableDate(value) {
|
|
|
220
196
|
}
|
|
221
197
|
const d = new Date(value + 'T00:00:00Z');
|
|
222
198
|
if (Number.isNaN(d.getTime())) return { ok: false, reason: 'unparseable' };
|
|
223
|
-
//
|
|
224
|
-
//
|
|
225
|
-
// rolls over to March 2 and reports a valid getTime() — so a malformed KEV
|
|
226
|
-
// deadline would pass this, the only date-validity guard. Require the parsed
|
|
227
|
-
// Y-M-D to round-trip back to the input components.
|
|
199
|
+
// `new Date('2026-02-30T00:00:00Z')` does not throw — it rolls over to March 2 —
|
|
200
|
+
// so a malformed KEV deadline would pass the only date-validity guard.
|
|
228
201
|
const [yStr, mStr, dStr] = value.split('-');
|
|
229
202
|
if (d.getUTCFullYear() !== Number(yStr) ||
|
|
230
203
|
d.getUTCMonth() + 1 !== Number(mStr) ||
|
|
@@ -244,17 +217,13 @@ function isUsableDate(value) {
|
|
|
244
217
|
function additionalChecks(key, entry, ctx) {
|
|
245
218
|
const warnings = [];
|
|
246
219
|
|
|
247
|
-
//
|
|
248
|
-
//
|
|
249
|
-
// the uncaught `entry.poc_available` TypeError on a null/array entry into a
|
|
250
|
-
// clean no-op so main() still prints that FAIL and continues to later
|
|
251
|
-
// entries instead of aborting the whole gate. The FAIL is preserved — it
|
|
252
|
-
// originates in validate(), not here.
|
|
220
|
+
// validate() already emits the type error, so the FAIL survives this no-op;
|
|
221
|
+
// without it, entry.* throws and aborts the gate.
|
|
253
222
|
if (!entry || typeof entry !== 'object' || Array.isArray(entry)) {
|
|
254
223
|
return [];
|
|
255
224
|
}
|
|
256
225
|
|
|
257
|
-
//
|
|
226
|
+
// Hard Rule #14: poc_available plus a public-exploit URL requires iocs.
|
|
258
227
|
if (entry.poc_available === true) {
|
|
259
228
|
const sources = Array.isArray(entry.verification_sources)
|
|
260
229
|
? entry.verification_sources
|
|
@@ -272,10 +241,7 @@ function additionalChecks(key, entry, ctx) {
|
|
|
272
241
|
}
|
|
273
242
|
}
|
|
274
243
|
|
|
275
|
-
//
|
|
276
|
-
// by default; promoted to hard failures under --strict. V2 expansion
|
|
277
|
-
// extends the walk from cwe_refs only to attack_refs, atlas_refs,
|
|
278
|
-
// d3fend_refs, AND framework_control_gaps.
|
|
244
|
+
// Cross-catalog refs: an unresolved id is a warning, an error under --strict.
|
|
279
245
|
const REF_FIELDS = [
|
|
280
246
|
{ field: 'atlas_refs', set: ctx.atlasKeys, file: 'data/atlas-ttps.json' },
|
|
281
247
|
{ field: 'cwe_refs', set: ctx.cweKeys, file: 'data/cwe-catalog.json' },
|
|
@@ -289,22 +255,16 @@ function additionalChecks(key, entry, ctx) {
|
|
|
289
255
|
];
|
|
290
256
|
for (const { field, set, file } of REF_FIELDS) {
|
|
291
257
|
if (!set) {
|
|
292
|
-
//
|
|
293
|
-
//
|
|
294
|
-
// and an orphaned id would ship unchecked. Surface it (promoted to a hard
|
|
295
|
-
// error under --strict, like an unresolved ref) for any entry that
|
|
296
|
-
// actually carries the field, rather than passing silently.
|
|
258
|
+
// An absent catalog means the family cannot be validated, not that it
|
|
259
|
+
// passed: surface it so an orphaned id cannot ship unchecked.
|
|
297
260
|
if (entry[field] && (Array.isArray(entry[field]) ? entry[field].length > 0 : typeof entry[field] === 'object' && Object.keys(entry[field]).length > 0)) {
|
|
298
261
|
warnings.push(`${key}: cannot validate ${field} — ${file} catalog is absent, so its ids go unchecked (an error under --strict).`);
|
|
299
262
|
}
|
|
300
263
|
continue;
|
|
301
264
|
}
|
|
302
265
|
const refs = entry[field];
|
|
303
|
-
// Most ref fields are arrays of
|
|
304
|
-
//
|
|
305
|
-
// Array.isArray-only guard skipped the map form entirely, so orphaned
|
|
306
|
-
// control ids never surfaced. Derive the candidate ids from whichever
|
|
307
|
-
// shape the field uses: array elements, or the keys of a plain object.
|
|
266
|
+
// Most ref fields are arrays of ids, but framework_control_gaps is a map keyed
|
|
267
|
+
// BY control id — an Array.isArray-only guard lets orphaned ids through.
|
|
308
268
|
let ids;
|
|
309
269
|
if (Array.isArray(refs)) {
|
|
310
270
|
ids = refs;
|
|
@@ -323,11 +283,8 @@ function additionalChecks(key, entry, ctx) {
|
|
|
323
283
|
}
|
|
324
284
|
}
|
|
325
285
|
|
|
326
|
-
//
|
|
327
|
-
//
|
|
328
|
-
// jurisdiction-clock SLAs key off the KEV listing date). cisa_kev=true with a
|
|
329
|
-
// null/missing/invalid cisa_kev_date is flagged; promoted to a hard error
|
|
330
|
-
// under --strict (predeploy). 0 violations in the shipped catalog today.
|
|
286
|
+
// Hard Rule #1: RWEP scoring and the jurisdiction-clock SLAs key off the KEV
|
|
287
|
+
// listing date, so a KEV flag without one is an incomplete claim.
|
|
331
288
|
if (entry.cisa_kev === true) {
|
|
332
289
|
const d = entry.cisa_kev_date;
|
|
333
290
|
const dateOk = typeof d === 'string' && isUsableDate(d).ok;
|
|
@@ -336,12 +293,9 @@ function additionalChecks(key, entry, ctx) {
|
|
|
336
293
|
`${key}: cisa_kev=true but cisa_kev_date is ${JSON.stringify(d)} (KEV status must carry a valid listing date — Hard Rule #1)`,
|
|
337
294
|
);
|
|
338
295
|
}
|
|
339
|
-
//
|
|
340
|
-
//
|
|
341
|
-
//
|
|
342
|
-
// the field is indistinguishable from one CISA has not. Both readings are
|
|
343
|
-
// available only while the field can be absent, so on a KEV-listed entry it
|
|
344
|
-
// may not be.
|
|
296
|
+
// An omitted boolean reads to every consumer as false, so a designation CISA
|
|
297
|
+
// made is indistinguishable from one it did not make. On a KEV entry the
|
|
298
|
+
// field may not be absent.
|
|
345
299
|
if (typeof entry.known_ransomware_use !== 'boolean') {
|
|
346
300
|
warnings.push(
|
|
347
301
|
`${key}: cisa_kev=true but known_ransomware_use is ${JSON.stringify(entry.known_ransomware_use)} (an absent boolean is read as false, which asserts a designation CISA may not agree with)`,
|
|
@@ -349,7 +303,6 @@ function additionalChecks(key, entry, ctx) {
|
|
|
349
303
|
}
|
|
350
304
|
}
|
|
351
305
|
|
|
352
|
-
// V4 — Impossible-date guard.
|
|
353
306
|
for (const f of DATE_FIELDS) {
|
|
354
307
|
const v = entry[f];
|
|
355
308
|
if (v === undefined || v === null) continue;
|
|
@@ -359,8 +312,6 @@ function additionalChecks(key, entry, ctx) {
|
|
|
359
312
|
}
|
|
360
313
|
}
|
|
361
314
|
|
|
362
|
-
// Sch1 — strict CVSS-vector prefix (warning-only for v0.12.12). The schema
|
|
363
|
-
// pattern stays loose; this admits only known CVSS versions.
|
|
364
315
|
if (typeof entry.cvss_vector === 'string' && entry.cvss_vector.length > 0) {
|
|
365
316
|
if (!STRICT_CVSS_PATTERN.test(entry.cvss_vector)) {
|
|
366
317
|
warnings.push(
|
|
@@ -372,10 +323,8 @@ function additionalChecks(key, entry, ctx) {
|
|
|
372
323
|
return warnings;
|
|
373
324
|
}
|
|
374
325
|
|
|
375
|
-
//
|
|
376
|
-
//
|
|
377
|
-
// _meta.entry_count (or absent/null catalogs) are skipped, so any loaded catalog
|
|
378
|
-
// that adopts the field is covered without editing this function.
|
|
326
|
+
// Catalogs without a numeric _meta.entry_count, and absent ones, are skipped, so
|
|
327
|
+
// one that adopts the field is covered without editing here.
|
|
379
328
|
function entryCountMismatches(catalogs) {
|
|
380
329
|
const failures = [];
|
|
381
330
|
for (const { name, catalog: cat } of catalogs) {
|
|
@@ -395,14 +344,8 @@ function main() {
|
|
|
395
344
|
const schema = readJson(SCHEMA_PATH);
|
|
396
345
|
const catalog = readJson(CATALOG_PATH);
|
|
397
346
|
const lessons = readJson(LESSONS_PATH);
|
|
398
|
-
//
|
|
399
|
-
//
|
|
400
|
-
// uniformly for every ref family. atlas/cwe previously defaulted to {},
|
|
401
|
-
// which built an EMPTY Set (never null) — so an absent atlas/cwe catalog
|
|
402
|
-
// silently FLAGGED every ref as unresolved (false positive) instead of
|
|
403
|
-
// surfacing the catalog-absent warning, the asymmetric opposite of how
|
|
404
|
-
// attack/d3fend/framework behaved. Uniform null makes the absence
|
|
405
|
-
// observable rather than misclassified.
|
|
347
|
+
// null when absent, never {}: an empty object builds an empty Set that is never
|
|
348
|
+
// null, so every ref would flag unresolved instead of firing the absent branch.
|
|
406
349
|
const atlas = fs.existsSync(ATLAS_PATH) ? readJson(ATLAS_PATH) : null;
|
|
407
350
|
const cwe = fs.existsSync(CWE_PATH) ? readJson(CWE_PATH) : null;
|
|
408
351
|
const attack = fs.existsSync(ATTACK_PATH) ? readJson(ATTACK_PATH) : null;
|
|
@@ -432,13 +375,8 @@ function main() {
|
|
|
432
375
|
const cveKeys = Object.keys(catalog).filter((k) => !k.startsWith('_'));
|
|
433
376
|
const lessonKeys = new Set(Object.keys(lessons).filter((k) => !k.startsWith('_')));
|
|
434
377
|
|
|
435
|
-
//
|
|
436
|
-
//
|
|
437
|
-
// the two could drift independently (the v0.12.41 _meta block had 5
|
|
438
|
-
// values, the schema enum had 4 — operators reading the _meta vocab
|
|
439
|
-
// got a list that the validator wouldn't accept). v0.13.5 reconciled
|
|
440
|
-
// by adding "theoretical" to the schema; this cross-check refuses any
|
|
441
|
-
// future drift.
|
|
378
|
+
// The schema's active_exploitation enum and _meta.active_exploitation_vocabulary
|
|
379
|
+
// must agree exactly, or operators read a vocabulary the validator rejects.
|
|
442
380
|
const schemaEnum = ((schema.properties || {}).active_exploitation || {}).enum;
|
|
443
381
|
const metaVocab = (catalog._meta && catalog._meta.active_exploitation_vocabulary && catalog._meta.active_exploitation_vocabulary.values) || null;
|
|
444
382
|
if (Array.isArray(schemaEnum) && Array.isArray(metaVocab)) {
|
|
@@ -459,13 +397,7 @@ function main() {
|
|
|
459
397
|
}
|
|
460
398
|
}
|
|
461
399
|
|
|
462
|
-
//
|
|
463
|
-
// framework-control-gaps counter once declared 184 while the file held 192
|
|
464
|
-
// and nothing caught it; the zeroday-lessons counter drifted to 68 while the
|
|
465
|
-
// file held 422 because only framework-control-gaps was gated. This checks
|
|
466
|
-
// EVERY loaded catalog that declares a numeric _meta.entry_count, so a new
|
|
467
|
-
// catalog with the field is covered automatically. The covered set is derived
|
|
468
|
-
// from the catalogs main() already loaded, not a fixed allowlist.
|
|
400
|
+
// Guards the hand-maintained _meta.entry_count fields against silent drift.
|
|
469
401
|
const ENTRY_COUNT_CATALOGS = [
|
|
470
402
|
{ name: 'cve-catalog', catalog },
|
|
471
403
|
{ name: 'zeroday-lessons', catalog: lessons },
|
|
@@ -490,7 +422,6 @@ function main() {
|
|
|
490
422
|
let drafts = 0;
|
|
491
423
|
let warned = 0;
|
|
492
424
|
|
|
493
|
-
// V3 — Duplicate-name detection across all non-_meta entries.
|
|
494
425
|
const nameToKeys = new Map();
|
|
495
426
|
for (const k of cveKeys) {
|
|
496
427
|
const n = catalog[k] && catalog[k].name;
|
|
@@ -510,11 +441,8 @@ function main() {
|
|
|
510
441
|
|
|
511
442
|
for (const key of cveKeys) {
|
|
512
443
|
const entry = catalog[key];
|
|
513
|
-
//
|
|
514
|
-
//
|
|
515
|
-
// exit-failing) so the nightly auto-PR pipeline can ship them while
|
|
516
|
-
// editorial fields await human or AI-assisted enrichment via
|
|
517
|
-
// `exceptd run cve-curation --advisory <id>`.
|
|
444
|
+
// Drafts validate as warnings, never failures, so the nightly auto-PR ships
|
|
445
|
+
// them while their editorial fields await enrichment.
|
|
518
446
|
const isDraft = entry && (entry._auto_imported === true || entry._draft === true);
|
|
519
447
|
const errors = validate(entry, schema, 'cve', key);
|
|
520
448
|
let warnings = additionalChecks(key, entry, ctx);
|
|
@@ -523,8 +451,7 @@ function main() {
|
|
|
523
451
|
`${key}: missing matching entry in data/zeroday-lessons.json (rule #6: zero-day learning is live)`,
|
|
524
452
|
);
|
|
525
453
|
}
|
|
526
|
-
//
|
|
527
|
-
// exempt (drafts already exit non-fail).
|
|
454
|
+
// --strict promotes per-CVE warnings to errors; drafts stay exempt.
|
|
528
455
|
if (opts.strict && !isDraft) {
|
|
529
456
|
errors.push(...warnings);
|
|
530
457
|
warnings = [];
|
|
@@ -536,7 +463,6 @@ function main() {
|
|
|
536
463
|
for (const e of errors) console.log(` - [warn] ${e}`);
|
|
537
464
|
for (const w of warnings) console.log(` - [warn] ${w}`);
|
|
538
465
|
}
|
|
539
|
-
// Drafts don't increment `failed` — they're warnings, not errors.
|
|
540
466
|
continue;
|
|
541
467
|
}
|
|
542
468
|
if (errors.length === 0 && warnings.length === 0) {
|
|
@@ -553,9 +479,8 @@ function main() {
|
|
|
553
479
|
}
|
|
554
480
|
}
|
|
555
481
|
|
|
556
|
-
/*
|
|
557
|
-
*
|
|
558
|
-
* the placeholder propagation rule #6 wants to catch. */
|
|
482
|
+
/* The other direction: a lesson with no catalog entry is the placeholder
|
|
483
|
+
* propagation rule #6 exists to catch. */
|
|
559
484
|
for (const lessonKey of lessonKeys) {
|
|
560
485
|
if (!cveKeys.includes(lessonKey)) {
|
|
561
486
|
failed++;
|
|
@@ -566,7 +491,6 @@ function main() {
|
|
|
566
491
|
}
|
|
567
492
|
}
|
|
568
493
|
|
|
569
|
-
// V3 — emit duplicate-name warnings as a catalog-wide tail block.
|
|
570
494
|
for (const w of dupNameWarnings) {
|
|
571
495
|
console.log(`WARN catalog`);
|
|
572
496
|
console.log(` - [warn] ${w}`);
|
|
@@ -579,7 +503,7 @@ function main() {
|
|
|
579
503
|
(warned ? `, ${warned} with warnings` : '') +
|
|
580
504
|
(failed ? `, ${failed} failed` : '') + '.';
|
|
581
505
|
console.log(summary);
|
|
582
|
-
// process.exitCode
|
|
506
|
+
// process.exitCode, not process.exit() — the exit can truncate a piped write.
|
|
583
507
|
process.exitCode = failed === 0 ? 0 : 1;
|
|
584
508
|
}
|
|
585
509
|
|
package/lib/validate-indexes.js
CHANGED
|
@@ -1,26 +1,8 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
/**
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
* against the canonical sources (manifest.json + every data/*.json
|
|
7
|
-
* minus _indexes/* + every skill body).
|
|
8
|
-
*
|
|
9
|
-
* Strategy:
|
|
10
|
-
* 1. Load `data/_indexes/_meta.json` for the source SHA-256 table.
|
|
11
|
-
* 2. Re-hash every listed source file.
|
|
12
|
-
* 3. Fail if any hash diverges (the indexes are stale).
|
|
13
|
-
* 4. Fail if a new source file exists that's not in the index (the
|
|
14
|
-
* index doesn't reflect current state).
|
|
15
|
-
* 5. Fail if source_hashes is empty (build-indexes never ran).
|
|
16
|
-
* 6. Fail if any data/*.json or listed source is a symlink.
|
|
17
|
-
*
|
|
18
|
-
* Exit 0 on success, 1 on staleness.
|
|
19
|
-
*
|
|
20
|
-
* Run as: node lib/validate-indexes.js
|
|
21
|
-
* Or as predeploy gate via scripts/predeploy.js.
|
|
22
|
-
*
|
|
23
|
-
* Re-build with: npm run build-indexes
|
|
3
|
+
* Predeploy gate. Confirms `data/_indexes/*.json` is current against the
|
|
4
|
+
* canonical sources by re-hashing each against the SHA-256 table in
|
|
5
|
+
* data/_indexes/_meta.json. Exit 0 when current, 1 when stale.
|
|
24
6
|
*/
|
|
25
7
|
|
|
26
8
|
const fs = require("fs");
|
|
@@ -37,17 +19,14 @@ function sha256(buf) {
|
|
|
37
19
|
}
|
|
38
20
|
|
|
39
21
|
/**
|
|
40
|
-
* Confirm every derived index OUTPUT exists, is a real file, and parses as
|
|
41
|
-
*
|
|
42
|
-
* Source-hash drift alone does NOT catch a deleted/truncated/corrupted output,
|
|
43
|
-
* so this runs in addition to the source-hash comparison.
|
|
22
|
+
* Confirm every derived index OUTPUT exists, is a real file, and parses as JSON.
|
|
23
|
+
* Returns human-readable issue strings (empty = all good).
|
|
44
24
|
*/
|
|
45
25
|
function verifyOutputs(idxDir, outputs) {
|
|
46
26
|
const issues = [];
|
|
47
|
-
// O_NOFOLLOW
|
|
48
|
-
// index path (ELOOP) and every
|
|
49
|
-
//
|
|
50
|
-
// O_NOFOLLOW is unavailable the flag degrades to 0 (plain open).
|
|
27
|
+
// O_NOFOLLOW plus a single descriptor: the open atomically refuses a symlinked
|
|
28
|
+
// index path (ELOOP) and every read rides the same fd, so there is no
|
|
29
|
+
// existsSync->lstat->readFileSync TOCTOU. Where unavailable the flag is 0.
|
|
51
30
|
const O_NOFOLLOW = fs.constants.O_NOFOLLOW || 0;
|
|
52
31
|
for (const file of outputs) {
|
|
53
32
|
const abs = path.join(idxDir, file);
|
|
@@ -65,10 +44,8 @@ function verifyOutputs(idxDir, outputs) {
|
|
|
65
44
|
continue;
|
|
66
45
|
}
|
|
67
46
|
try {
|
|
68
|
-
// readFileSync(fd) loops read() to EOF — a single readSync
|
|
69
|
-
//
|
|
70
|
-
// leaving the tail NUL-filled and truncating the JSON. Reading via the
|
|
71
|
-
// open fd keeps the open→fstat ordering TOCTOU-free.
|
|
47
|
+
// readFileSync(fd) loops read() to EOF — a single readSync can return fewer
|
|
48
|
+
// than st.size bytes on a network fd, NUL-filling the tail.
|
|
72
49
|
JSON.parse(fs.readFileSync(fd, "utf8"));
|
|
73
50
|
} catch (e) {
|
|
74
51
|
issues.push(`derived index file does not parse: data/_indexes/${file} (${e.message})`);
|
|
@@ -80,14 +57,8 @@ function verifyOutputs(idxDir, outputs) {
|
|
|
80
57
|
}
|
|
81
58
|
|
|
82
59
|
/**
|
|
83
|
-
* Compare a live source set against the recorded source_hashes table
|
|
84
|
-
*
|
|
85
|
-
* absFn + fs reads of the named sources) so it is unit-testable for the
|
|
86
|
-
* fail-closed edge cases:
|
|
87
|
-
* - a source that vanished between discovery and hashing (TOCTOU) is REPORTED
|
|
88
|
-
* missing, never an unhandled ENOENT crash;
|
|
89
|
-
* - a non-string recorded hash (corrupted _meta.json) is REPORTED as drift,
|
|
90
|
-
* never a TypeError from .slice() of a non-string.
|
|
60
|
+
* Compare a live source set against the recorded source_hashes table. Fail-closed:
|
|
61
|
+
* a vanished source and a non-string recorded hash are both REPORTED, never thrown.
|
|
91
62
|
*
|
|
92
63
|
* @param {Set<string>|string[]} liveSources source paths discovered live
|
|
93
64
|
* @param {Record<string,string>} recorded the _meta.json source_hashes table
|
|
@@ -104,15 +75,9 @@ function checkSourceHashes(liveSources, recorded, absFn) {
|
|
|
104
75
|
continue;
|
|
105
76
|
}
|
|
106
77
|
const abs = absFn(p);
|
|
107
|
-
// F16 — also check listed-but-symlinked sources. A source can disappear or
|
|
108
|
-
// be swapped between discovery and hashing (TOCTOU); resolve its type and
|
|
109
|
-
// read it defensively so a vanished or newly-symlinked file is REPORTED as
|
|
110
|
-
// missing, never an unhandled ENOENT that crashes the whole validation.
|
|
111
78
|
// Open through a single O_NOFOLLOW descriptor and hash the bytes from THAT
|
|
112
|
-
// descriptor
|
|
113
|
-
//
|
|
114
|
-
// file (ENOENT) is reported missing instead of crashing; the fstat type
|
|
115
|
-
// check refuses a dir/junction/fifo on every platform.
|
|
79
|
+
// descriptor: no path stat-then-read, so no check-then-use window. The fstat
|
|
80
|
+
// type check refuses a dir / junction / fifo on every platform.
|
|
116
81
|
let live;
|
|
117
82
|
{
|
|
118
83
|
let fd;
|
|
@@ -136,9 +101,6 @@ function checkSourceHashes(liveSources, recorded, absFn) {
|
|
|
136
101
|
}
|
|
137
102
|
const recordedHash = recorded[p];
|
|
138
103
|
if (typeof recordedHash !== 'string') {
|
|
139
|
-
// A null/non-string recorded hash means a corrupted _meta.json
|
|
140
|
-
// source_hashes map — report it as drift rather than crashing on
|
|
141
|
-
// .slice() of a non-string (fail-closed, not fail-open).
|
|
142
104
|
drift.push(`hash drift: ${p} (recorded entry is not a string: ${JSON.stringify(recordedHash)}, live ${live.slice(0, 12)}…)`);
|
|
143
105
|
} else if (live !== recordedHash) {
|
|
144
106
|
drift.push(`hash drift: ${p} (recorded ${recordedHash.slice(0, 12)}…, live ${live.slice(0, 12)}…)`);
|
|
@@ -155,7 +117,8 @@ function checkSourceHashes(liveSources, recorded, absFn) {
|
|
|
155
117
|
function main() {
|
|
156
118
|
if (!fs.existsSync(META)) {
|
|
157
119
|
console.error("[validate-indexes] data/_indexes/_meta.json missing — run `npm run build-indexes`.");
|
|
158
|
-
//
|
|
120
|
+
// `process.exitCode` + return, never process.exit(): a piped stderr write
|
|
121
|
+
// can be truncated by the exit. Same for every other exit path below.
|
|
159
122
|
process.exitCode = 1;
|
|
160
123
|
return;
|
|
161
124
|
}
|
|
@@ -163,12 +126,8 @@ function main() {
|
|
|
163
126
|
const meta = JSON.parse(fs.readFileSync(META, "utf8"));
|
|
164
127
|
const recorded = meta.source_hashes || {};
|
|
165
128
|
|
|
166
|
-
//
|
|
167
|
-
//
|
|
168
|
-
// because the for-loop body never executed; the resulting "0 sources" pass
|
|
169
|
-
// banner falsely advertised the indexes as current. An empty source-hash
|
|
170
|
-
// table means build-indexes was never run, or was run against an empty
|
|
171
|
-
// repo, and the index files themselves are not trustworthy.
|
|
129
|
+
// An empty source_hashes table is a hard fail, not a "0 sources" pass: the
|
|
130
|
+
// comparison loop never runs, so an empty table would advertise them current.
|
|
172
131
|
if (Object.keys(recorded).length === 0) {
|
|
173
132
|
console.error(
|
|
174
133
|
"[validate-indexes] data/_indexes/_meta.json source_hashes is empty — " +
|
|
@@ -179,19 +138,14 @@ function main() {
|
|
|
179
138
|
return;
|
|
180
139
|
}
|
|
181
140
|
|
|
182
|
-
// Discover the current canonical source set.
|
|
183
141
|
const manifest = JSON.parse(fs.readFileSync(ABS("manifest.json"), "utf8"));
|
|
184
142
|
const liveSources = new Set();
|
|
185
143
|
liveSources.add("manifest.json");
|
|
186
|
-
//
|
|
187
|
-
//
|
|
188
|
-
// source set in sync — otherwise the hashed README looks like a removed
|
|
189
|
-
// file here. (Mirrors liveSourceSet() in scripts/build-indexes.js.)
|
|
144
|
+
// This set mirrors liveSourceSet() in scripts/build-indexes.js, which hashes
|
|
145
|
+
// README.md for the stale-content index; out of sync, README reads as removed.
|
|
190
146
|
if (fs.existsSync(ABS("README.md"))) liveSources.add("README.md");
|
|
191
|
-
//
|
|
192
|
-
//
|
|
193
|
-
// (or a misconfigured filesystem) to swap data origin without tripping the
|
|
194
|
-
// gate. Reject symlinks outright.
|
|
147
|
+
// lstat, not stat: a symlinked .json under data/ would be hashed through its
|
|
148
|
+
// target, letting a checkout swap the data origin without tripping the gate.
|
|
195
149
|
for (const f of fs.readdirSync(ABS("data"))) {
|
|
196
150
|
if (!f.endsWith(".json")) continue;
|
|
197
151
|
const abs = ABS("data/" + f);
|
|
@@ -210,14 +164,9 @@ function main() {
|
|
|
210
164
|
|
|
211
165
|
const { drift, missing } = checkSourceHashes(liveSources, recorded, ABS);
|
|
212
166
|
|
|
213
|
-
//
|
|
214
|
-
//
|
|
215
|
-
//
|
|
216
|
-
// _meta.outputs is the canonical list build-indexes records; an absent list
|
|
217
|
-
// FAILS (rather than silently skipping the check) so an old _meta can't pass.
|
|
218
|
-
// Absent `outputs` list (an older build that predates this field) is itself a
|
|
219
|
-
// staleness condition — fold it into the unified STALE report rather than
|
|
220
|
-
// early-returning, so a concurrent source-hash drift still surfaces too.
|
|
167
|
+
// Source-hash drift alone does not catch a deleted or corrupted OUTPUT.
|
|
168
|
+
// _meta.outputs is the canonical list build-indexes records; an absent list is
|
|
169
|
+
// itself staleness, folded into the STALE report so hash drift still surfaces.
|
|
221
170
|
const outputs = Array.isArray(meta.outputs) ? meta.outputs : null;
|
|
222
171
|
if (!outputs || outputs.length === 0) {
|
|
223
172
|
missing.push(
|