@blamejs/exceptd-skills 0.19.33 → 0.19.35

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (119) hide show
  1. package/CHANGELOG.md +28 -0
  2. package/bin/exceptd.js +895 -2828
  3. package/data/_indexes/_meta.json +2 -2
  4. package/lib/auto-discovery.js +56 -286
  5. package/lib/canonical-eq.js +7 -40
  6. package/lib/citation-resolve.js +22 -70
  7. package/lib/collectors/ai-api.js +170 -76
  8. package/lib/collectors/cicd-pipeline-compromise.js +113 -136
  9. package/lib/collectors/citation-hygiene.js +72 -210
  10. package/lib/collectors/containers.js +41 -130
  11. package/lib/collectors/cred-stores.js +31 -115
  12. package/lib/collectors/crypto-codebase.js +55 -138
  13. package/lib/collectors/crypto.js +24 -54
  14. package/lib/collectors/hardening.js +20 -78
  15. package/lib/collectors/kernel.js +16 -46
  16. package/lib/collectors/library-author.js +198 -211
  17. package/lib/collectors/mcp.js +24 -70
  18. package/lib/collectors/runtime.js +24 -86
  19. package/lib/collectors/sbom.js +130 -118
  20. package/lib/collectors/scan-excludes.js +33 -139
  21. package/lib/collectors/secrets.js +62 -178
  22. package/lib/cross-ref-api.js +39 -123
  23. package/lib/currency-severity.js +8 -27
  24. package/lib/cve-batch.js +13 -21
  25. package/lib/cve-cli.js +13 -20
  26. package/lib/cve-curation.js +72 -239
  27. package/lib/cve-regression-watcher.js +29 -155
  28. package/lib/cvss.js +13 -54
  29. package/lib/doctor-bucketing.js +3 -19
  30. package/lib/exit-codes.js +10 -42
  31. package/lib/flag-suggest.js +7 -25
  32. package/lib/framework-gap.js +39 -113
  33. package/lib/gap-detectors.js +37 -159
  34. package/lib/id-validation.js +9 -30
  35. package/lib/job-queue.js +13 -36
  36. package/lib/lint-skills.js +88 -236
  37. package/lib/playbook-runner.js +759 -2107
  38. package/lib/prefetch.js +101 -376
  39. package/lib/refresh-external.js +199 -633
  40. package/lib/refresh-network.js +78 -311
  41. package/lib/rfc-cli.js +23 -68
  42. package/lib/scoring.js +85 -146
  43. package/lib/sign.js +43 -229
  44. package/lib/source-advisories.js +43 -194
  45. package/lib/source-ghsa.js +37 -120
  46. package/lib/source-osv.js +94 -266
  47. package/lib/ttp-mapper.js +28 -27
  48. package/lib/upstream-check-cli.js +36 -29
  49. package/lib/upstream-check.js +19 -44
  50. package/lib/validate-catalog-meta.js +17 -61
  51. package/lib/validate-cve-catalog.js +52 -121
  52. package/lib/validate-indexes.js +25 -76
  53. package/lib/validate-package.js +16 -62
  54. package/lib/validate-playbooks.js +78 -286
  55. package/lib/validate-vendor.js +16 -49
  56. package/lib/verify.js +56 -286
  57. package/lib/version-pins.js +5 -34
  58. package/lib/worker-pool.js +11 -30
  59. package/lib/xml-tokenizer.js +47 -152
  60. package/manifest.json +53 -53
  61. package/orchestrator/dispatcher.js +17 -68
  62. package/orchestrator/event-bus.js +11 -74
  63. package/orchestrator/index.js +138 -413
  64. package/orchestrator/pipeline.js +28 -85
  65. package/orchestrator/scanner.js +34 -138
  66. package/orchestrator/scheduler.js +20 -84
  67. package/package.json +1 -1
  68. package/sbom.cdx.json +242 -242
  69. package/scripts/audit-catalog-gaps.js +9 -62
  70. package/scripts/audit-cross-skill.js +5 -31
  71. package/scripts/audit-perf.js +29 -28
  72. package/scripts/backfill-theater-test.js +7 -64
  73. package/scripts/bootstrap.js +12 -44
  74. package/scripts/build-indexes.js +40 -154
  75. package/scripts/builders/activity-feed.js +4 -14
  76. package/scripts/builders/catalog-summaries.js +3 -10
  77. package/scripts/builders/currency.js +7 -20
  78. package/scripts/builders/cwe-chains.js +7 -30
  79. package/scripts/builders/did-ladders.js +6 -13
  80. package/scripts/builders/frequency.js +5 -19
  81. package/scripts/builders/jurisdiction-clocks.js +6 -25
  82. package/scripts/builders/recipes.js +6 -14
  83. package/scripts/builders/section-offsets.js +13 -51
  84. package/scripts/builders/stale-content.js +7 -28
  85. package/scripts/builders/summary-cards.js +8 -29
  86. package/scripts/builders/theater-fingerprints.js +21 -31
  87. package/scripts/builders/token-budget.js +4 -31
  88. package/scripts/check-agents-md-collectors.js +26 -57
  89. package/scripts/check-catalog-gap-budget.js +15 -32
  90. package/scripts/check-changelog-extract.js +18 -48
  91. package/scripts/check-codebase-patterns-currency.js +6 -22
  92. package/scripts/check-codebase-patterns.js +63 -143
  93. package/scripts/check-epss-consistency.js +9 -64
  94. package/scripts/check-framework-gap-coverage.js +13 -31
  95. package/scripts/check-manifest-snapshot.js +62 -81
  96. package/scripts/check-sbom-currency.js +44 -142
  97. package/scripts/check-test-count.js +15 -52
  98. package/scripts/check-test-coverage.js +83 -198
  99. package/scripts/check-test-subjects.js +21 -62
  100. package/scripts/check-ttp-references.js +14 -38
  101. package/scripts/check-ttp-upstream.js +8 -40
  102. package/scripts/check-version-bump.js +9 -61
  103. package/scripts/check-version-tags.js +20 -121
  104. package/scripts/predeploy.js +38 -184
  105. package/scripts/refresh-manifest-snapshot.js +16 -38
  106. package/scripts/refresh-mitre-atlas.js +7 -8
  107. package/scripts/refresh-mitre-attack.js +1 -8
  108. package/scripts/refresh-mitre-d3fend.js +3 -9
  109. package/scripts/refresh-mitre-ics-attack.js +7 -8
  110. package/scripts/refresh-reverse-refs.js +27 -94
  111. package/scripts/refresh-rfc-index.js +7 -10
  112. package/scripts/refresh-sbom.js +31 -161
  113. package/scripts/refresh-upstream-catalogs.js +63 -148
  114. package/scripts/release.js +69 -234
  115. package/scripts/run-e2e-scenarios.js +26 -73
  116. package/scripts/sync-manifest-metadata.js +10 -34
  117. package/scripts/sync-package-description.js +8 -17
  118. package/scripts/validate-vendor-online.js +13 -44
  119. package/scripts/verify-shipped-tarball.js +35 -141
@@ -1,38 +1,10 @@
1
1
  #!/usr/bin/env node
2
2
  "use strict";
3
3
  /**
4
- * scripts/refresh-upstream-catalogs.js
5
- *
6
- * Unified entrypoint + library for refreshing the four canonical upstream
7
- * catalogs from their official sources. Each refresher is idempotent and
8
- * never overwrites operator-curated entries (rows that lack the
9
- * `_auto_imported: true` flag are preserved verbatim).
10
- *
11
- * rfc ietf-rfc-index data/rfc-references.json
12
- * attack mitre-attack-stix data/attack-techniques.json
13
- * atlas mitre-atlas-stix data/atlas-ttps.json
14
- * d3fend mitre-d3fend-owl data/d3fend-catalog.json
15
- *
16
- * CLI usage:
17
- *
18
- * node scripts/refresh-upstream-catalogs.js # all four
19
- * node scripts/refresh-upstream-catalogs.js --source rfc # one
20
- * node scripts/refresh-upstream-catalogs.js --source rfc,atlas # two
21
- * node scripts/refresh-upstream-catalogs.js --dry-run # report only
22
- * CAP=200 node scripts/refresh-upstream-catalogs.js --source attack
23
- *
24
- * Module usage (per-type wrappers under scripts/refresh-{rfc,attack,atlas,
25
- * d3fend}.js import the corresponding refreshX function from this file):
26
- *
27
- * const { refreshRfc } = require("./refresh-upstream-catalogs.js");
28
- * await refreshRfc({ dry: false });
29
- *
30
- * npm aliases (package.json scripts):
31
- * refresh-upstream-catalogs runs all four
32
- * refresh-rfc-index --source rfc
33
- * refresh-mitre-attack --source attack
34
- * refresh-mitre-atlas --source atlas
35
- * refresh-mitre-d3fend --source d3fend
4
+ * Refreshes the upstream catalogs (IETF RFC index, MITRE ATT&CK, ICS-ATT&CK,
5
+ * ATLAS, D3FEND) from their official sources. Refreshers are idempotent and
6
+ * additive: a populated field is never overwritten, and only `_auto_imported`
7
+ * rows take a status bump.
36
8
  */
37
9
 
38
10
  const fs = require("fs");
@@ -42,30 +14,11 @@ const path = require("path");
42
14
  const ROOT = path.join(__dirname, "..");
43
15
  const TODAY = new Date().toISOString().slice(0, 10);
44
16
 
45
- // v0.13.20 class-3.11 fix: refreshers read their required-context list
46
- // from the audit SPEC. Eliminates the parallel hardcoded field arrays
47
- // that v0.13.17→19 carried (and forgot to keep in sync — the v0.13.19
48
- // audit found 106 ATT&CK rows missing `description` + `tactic` because
49
- // the v0.13.18 backfill list omitted those fields). One source of truth
50
- // = the audit-catalog-gaps SPEC.
51
- const AUDIT_SPEC = require("./audit-catalog-gaps.js").SPEC;
52
- function specRequiredFields(catalogKey) {
53
- const spec = AUDIT_SPEC[catalogKey];
54
- if (!spec || !Array.isArray(spec.required_context)) return [];
55
- return spec.required_context.map((r) => r.field);
56
- }
57
-
58
17
  const MAX_REDIRECTS = 5;
59
18
 
60
- // Hardened fetch helper. Three properties the hand-rolled follower lacked:
61
- // 1. Redirect-depth cap + base-URL resolution + response drain, so a
62
- // redirect loop rejects within the cap (rather than recursing/hanging
63
- // unbounded) and a relative Location resolves against the current URL.
64
- // 2. 4xx/5xx (and a missing statusCode edge) reject instead of resolving an
65
- // error body as a "successful" empty result — every consumer fails
66
- // closed on an HTTP error rather than stamping _meta on a non-fetch.
67
- // 3. A 3xx with no Location header rejects with a clear message rather than
68
- // throwing an opaque ERR_INVALID_URL on `new URL(undefined, url)`.
19
+ // Rejects on anything but a 2xx, so an error body can never reach a consumer as
20
+ // a "successful" empty result and get _meta stamped on a non-fetch. A redirect
21
+ // resolves Location against the current URL and rejects past MAX_REDIRECTS.
69
22
  function fetchUrl(url, depth = 0) {
70
23
  return new Promise((resolve, reject) => {
71
24
  https.get(url, { headers: { "User-Agent": "exceptd-refresh-upstream-catalogs" } }, (r) => {
@@ -99,11 +52,8 @@ function loadCatalog(rel) {
99
52
  return JSON.parse(fs.readFileSync(path.join(ROOT, "data", rel), "utf8"));
100
53
  }
101
54
 
102
- // Atomic write: a crash / disk-full / SIGKILL mid-write would otherwise leave a
103
- // truncated JSON catalog on disk. Write to a temp sibling and rename — rename is
104
- // atomic on POSIX and on same-volume Windows renames (the .tmp sibling is
105
- // adjacent to the target, same volume), so a reader / the next run only ever
106
- // sees the complete old or complete new file. Mirrors build-indexes#writeJson.
55
+ // Temp sibling + atomic rename, so a crash mid-write cannot leave a truncated
56
+ // catalog behind. Mirrors build-indexes#writeJson.
107
57
  function writeCatalog(rel, obj) {
108
58
  const abs = path.join(ROOT, "data", rel);
109
59
  const tmp = `${abs}.tmp-${process.pid}`;
@@ -117,8 +67,6 @@ function getTag(blk, tag) {
117
67
  return m ? m[1].trim() : null;
118
68
  }
119
69
 
120
- // ---------------- RFC ----------------
121
-
122
70
  const RFC_SRC = "https://www.rfc-editor.org/rfc-index.xml";
123
71
  const RFC_STATUS_MAP = {
124
72
  "INTERNET STANDARD": "Internet Standard",
@@ -136,8 +84,7 @@ const RFC_MONTHS = {
136
84
  September: "09", October: "10", November: "11", December: "12"
137
85
  };
138
86
 
139
- // Extract every doc-id reference inside a parent tag like
140
- // <obsoleted-by><doc-id>RFC123</doc-id><doc-id>RFC456</doc-id></obsoleted-by>.
87
+ // Every <doc-id> inside a parent tag: <obsoleted-by><doc-id>RFC123</doc-id>…
141
88
  function getDocIdList(blk, parentTag) {
142
89
  const re = new RegExp(`<${parentTag}>([\\s\\S]*?)<\\/${parentTag}>`);
143
90
  const m = blk.match(re);
@@ -166,7 +113,6 @@ function getAbstract(blk) {
166
113
  return paras.length ? paras.join(" ") : null;
167
114
  }
168
115
 
169
- // <keywords><kw>k1</kw><kw>k2</kw></keywords>
170
116
  function getKeywords(blk) {
171
117
  const m = blk.match(/<keywords>([\s\S]*?)<\/keywords>/);
172
118
  if (!m) return [];
@@ -211,10 +157,6 @@ function parseRfcEntry(blk) {
211
157
  const published = year && month ? `${year}-${month}` : (year || "unknown");
212
158
  const hasErrata = /<errata-url>/.test(blk);
213
159
  const obsoleted = /<obsoleted-by>/.test(blk);
214
- // New context-search fields (v0.13.18+): the AI needs more than the
215
- // title to locate an RFC by topic. abstract + keywords + area +
216
- // wg_acronym + stream + authors + obsoletes/updates relationships are
217
- // all present in the IETF index — we were only extracting title before.
218
160
  const abstract = getAbstract(blk);
219
161
  const keywords = getKeywords(blk);
220
162
  const area = getTag(blk, "area");
@@ -235,7 +177,7 @@ function parseRfcEntry(blk) {
235
177
  };
236
178
  }
237
179
 
238
- async function refreshRfc({ dry = false, _deps = {} } = {}) {
180
+ async function refreshRfc({ dry = false, cap = Infinity, _deps = {} } = {}) {
239
181
  const _fetchUrl = _deps.fetchUrl || fetchUrl;
240
182
  const _loadCatalog = _deps.loadCatalog || loadCatalog;
241
183
  const _writeCatalog = _deps.writeCatalog || writeCatalog;
@@ -253,12 +195,8 @@ async function refreshRfc({ dry = false, _deps = {} } = {}) {
253
195
  if (e.obsoleted || e.status === "HISTORIC" || e.status === "UNKNOWN") continue;
254
196
  entries.push(e);
255
197
  }
256
- // Sanity floor: the IETF index has ~9000+ RFCs, so a successful fetch can
257
- // never parse to zero entries. A zero count means the fetch returned an
258
- // error/empty/soft-error body (a 200 with a CDN error page, a captive portal,
259
- // or a truncated body the HTTP-status guard can't see) — refuse to stamp
260
- // _meta or write rfc-references.json, matching the JSON-parse failures the
261
- // STIX sources surface for free (an empty index is never a legitimate result).
198
+ // A successful fetch of a ~9000-RFC index can never parse to zero. Zero means a
199
+ // soft-error body — a CDN error page under a 200 — the status guard cannot see.
262
200
  if (backfillable.length === 0) {
263
201
  throw new Error("RFC index parsed 0 entries (fetch likely returned an error/empty body) — refusing to stamp _meta or write rfc-references.json");
264
202
  }
@@ -266,19 +204,16 @@ async function refreshRfc({ dry = false, _deps = {} } = {}) {
266
204
  const cat = _loadCatalog("rfc-references.json");
267
205
  const existing = new Set(Object.keys(cat).filter((k) => k !== "_meta"));
268
206
  let added = 0, statusBumped = 0, backfilledCount = 0;
269
- // First pass: backfill ALL existing rows from the broader entry set
270
- // (including obsoleted historics). Operator may have curated an
271
- // obsoleted RFC in for documentation; we still want abstract/authors.
207
+ // Backfill every existing row from the broader entry set, obsoleted historics
208
+ // included: a curated obsoleted RFC still wants its abstract and authors.
272
209
  for (const e of backfillable) {
273
210
  const id = `RFC-${e.num}`;
274
211
  if (!existing.has(id)) continue;
275
212
  const cur = cat[id];
276
213
  if (!cur) continue;
277
214
  let touched = false;
278
- // Mirror the new-add path's `|| e.status` fallback: an upstream status
279
- // outside RFC_STATUS_MAP must NOT write `undefined` (which would drop the
280
- // status field on an existing curated row). Fall back to the raw upstream
281
- // status, and only bump when the mapped value actually differs.
215
+ // An upstream status outside RFC_STATUS_MAP falls back to the raw status
216
+ // rather than writing `undefined` and dropping the field on an existing row.
282
217
  const mapped = RFC_STATUS_MAP[e.status] || e.status;
283
218
  if (cur._auto_imported && mapped && cur.status !== mapped) {
284
219
  cur.status = mapped;
@@ -302,18 +237,17 @@ async function refreshRfc({ dry = false, _deps = {} } = {}) {
302
237
  if (!cur.html_url) { cur.html_url = `https://www.rfc-editor.org/rfc/rfc${e.num}.html`; touched = true; }
303
238
  if (touched) { cur.last_verified = TODAY; backfilledCount++; }
304
239
  }
305
- // Second pass: add new "current" entries that weren't in the catalog.
306
- // Add new rows from the FULL index, not just the current series. Obsoleted
307
- // and historic RFCs were previously excluded, so "is RFC N still current?"
308
- // had no offline answer and forced a datatracker lookup. They are added here
309
- // marked `_obsoleted` (with obsoleted_by populated) so the resolver can say
310
- // "Historic, superseded by RFC X" offline. UNKNOWN-status index rows
311
- // (placeholders / not-issued numbers) are still skipped.
240
+ // New rows come from the full index, not just the current series: obsoleted and
241
+ // historic RFCs land marked `_obsoleted` with obsoleted_by populated, so the
242
+ // resolver answers "Historic, superseded by RFC X" offline. UNKNOWN is skipped.
312
243
  for (const e of backfillable) {
313
244
  const id = `RFC-${e.num}`;
314
245
  // Existing rows handled in the first-pass backfill above.
315
246
  if (existing.has(id)) continue;
316
247
  if (e.status === "UNKNOWN") continue;
248
+ // The cap bounds new adds only; the backfill pass above is unbounded, so a
249
+ // capped run still completes context on every row already curated.
250
+ if (added >= cap) continue;
317
251
  const obsoleted = !!e.obsoleted || e.status === "HISTORIC";
318
252
  cat[id] = {
319
253
  number: e.num,
@@ -347,10 +281,8 @@ async function refreshRfc({ dry = false, _deps = {} } = {}) {
347
281
  existing.add(id);
348
282
  added++;
349
283
  }
350
- // Only restamp _meta + write when something actually changed. A genuine
351
- // no-op leaves the file byte-identical so the daily refresh doesn't emit a
352
- // spurious _meta-only diff (and so the freshness gates stay honest — a
353
- // wall-clock restamp on an unchanged catalog masks real staleness).
284
+ // Restamp _meta and write only on a real change: restamping an unchanged
285
+ // catalog masks real staleness from the freshness gates.
354
286
  const changed = added > 0 || backfilledCount > 0 || statusBumped > 0;
355
287
  if (dry) {
356
288
  console.log(`[refresh-upstream:rfc] DRY-RUN: +${added} new, ${backfilledCount} backfilled, ${statusBumped} status bumps.`);
@@ -369,8 +301,6 @@ async function refreshRfc({ dry = false, _deps = {} } = {}) {
369
301
  return { added, statusBumped, backfilled: backfilledCount };
370
302
  }
371
303
 
372
- // ---------------- ATT&CK ----------------
373
-
374
304
  const ATTACK_SRC = "https://raw.githubusercontent.com/mitre/cti/master/enterprise-attack/enterprise-attack.json";
375
305
  const ATTACK_TACTIC_NAME = {
376
306
  "reconnaissance": "Reconnaissance", "resource-development": "Resource Development",
@@ -383,10 +313,8 @@ const ATTACK_TACTIC_NAME = {
383
313
  "exfiltration": "Exfiltration", "impact": "Impact"
384
314
  };
385
315
 
386
- // Extract the full STIX-bundle context fields the AI needs to find
387
- // techniques by topic (not just by ID). description_full preserves the
388
- // MITRE description; description (short) is the first-sentence
389
- // extractive summary used by token-budgeted consumers.
316
+ // description_full keeps MITRE's text; description is the first-sentence
317
+ // summary that token-budgeted consumers read.
390
318
  function attackEntryFromStix(t, extRef) {
391
319
  const id = extRef.external_id;
392
320
  const tactics = (t.kill_chain_phases || [])
@@ -429,13 +357,9 @@ function backfillAttack(cur, fresh) {
429
357
  if (!cur[key] && val) { cur[key] = val; touched = true; }
430
358
  }
431
359
  };
432
- // v0.13.19: include description (short) + tactic in the backfill set.
433
- // Existing rows from the original 110-entry catalog often have only
434
- // {name, version} — they need tactic + short-description too, not just
435
- // the v0.13.18 description_full / platforms / detection additions.
436
360
  fillIfEmpty("description", fresh.description);
437
- // tactic: arrays only (existing rows may have a string tactic; do
438
- // not overwrite a stringified tactic with an array form).
361
+ // Arrays only: an existing row may carry a string tactic, which must not be
362
+ // overwritten with the array form.
439
363
  if ((!cur.tactic || (Array.isArray(cur.tactic) && cur.tactic.length === 0)) && Array.isArray(fresh.tactic) && fresh.tactic.length) {
440
364
  cur.tactic = fresh.tactic;
441
365
  touched = true;
@@ -462,12 +386,9 @@ async function refreshAttack({ dry = false, cap = Infinity, _deps = {} } = {}) {
462
386
  console.log("[refresh-upstream:attack] fetching MITRE ATT&CK STIX...");
463
387
  const body = await _fetchUrl(ATTACK_SRC);
464
388
  const stix = JSON.parse(body);
465
- // For NEW adds: live techniques only (skip revoked / deprecated).
466
- // For BACKFILL on existing rows: include revoked too — an operator-
467
- // curated row that references a now-revoked MITRE ID may still want
468
- // the context fields (name / description / platforms) from the
469
- // pre-revocation STIX record. Same logic as the RFC obsoleted-but-
470
- // backfillable two-pass design.
389
+ // New adds take live techniques only; the backfill pass includes revoked and
390
+ // deprecated ones, so a curated row on a revoked MITRE ID still gets its
391
+ // context fields from the pre-revocation record.
471
392
  const liveTechs = (stix.objects || []).filter(
472
393
  (o) => o.type === "attack-pattern" && !o.revoked && !o.x_mitre_deprecated
473
394
  );
@@ -488,8 +409,6 @@ async function refreshAttack({ dry = false, cap = Infinity, _deps = {} } = {}) {
488
409
  });
489
410
  let added = 0;
490
411
  let backfilled = 0;
491
- // First pass: backfill existing rows against the FULL technique set
492
- // (including revoked) so operator-curated rows still get context.
493
412
  for (const t of backfillTechs) {
494
413
  const extRef = (t.external_references || []).find((r) => r.source_name === "mitre-attack");
495
414
  if (!extRef || !extRef.external_id) continue;
@@ -502,7 +421,6 @@ async function refreshAttack({ dry = false, cap = Infinity, _deps = {} } = {}) {
502
421
  backfilled++;
503
422
  }
504
423
  }
505
- // Second pass: add new entries from live techniques only.
506
424
  for (const t of techs) {
507
425
  const extRef = (t.external_references || []).find((r) => r.source_name === "mitre-attack");
508
426
  if (!extRef || !extRef.external_id) continue;
@@ -525,8 +443,6 @@ async function refreshAttack({ dry = false, cap = Infinity, _deps = {} } = {}) {
525
443
  return { added, backfilled };
526
444
  }
527
445
 
528
- // ---------------- ICS-ATT&CK ----------------
529
-
530
446
  const ICS_ATTACK_SRC = "https://raw.githubusercontent.com/mitre/cti/master/ics-attack/ics-attack.json";
531
447
  const ICS_TACTIC_NAME = {
532
448
  "initial-access": "Initial Access (ICS)",
@@ -557,6 +473,7 @@ async function refreshIcsAttack({ dry = false, cap = Infinity, _deps = {} } = {}
557
473
  const local = _loadCatalog("attack-techniques.json");
558
474
  const existing = new Set(Object.keys(local).filter((k) => k !== "_meta"));
559
475
  let added = 0, backfilled = 0;
476
+ const skippedNoIcsTactic = [];
560
477
  for (const t of techs) {
561
478
  const extRef = (t.external_references || []).find((r) => r.source_name === "mitre-ics-attack" || r.source_name === "mitre-attack");
562
479
  if (!extRef || !extRef.external_id) continue;
@@ -564,6 +481,17 @@ async function refreshIcsAttack({ dry = false, cap = Infinity, _deps = {} } = {}
564
481
  const tactics = (t.kill_chain_phases || [])
565
482
  .filter((p) => (p.kill_chain_name || "").includes("ics"))
566
483
  .map((p) => ICS_TACTIC_NAME[p.phase_name] || `${p.phase_name} (ICS)`);
484
+ // The external-reference match above accepts a cross-listed enterprise
485
+ // reference, while the tactic map keeps ICS kill-chain phases only. An
486
+ // object matched by the first and not the second would land with
487
+ // `tactic: []`, which audit-catalog-gaps counts as a missing-context gap
488
+ // the moment it is written — the import spending gap budget on itself.
489
+ // Enterprise-only techniques belong to refreshAttack, so skip them here,
490
+ // and report the count so the omission is observable rather than silent.
491
+ if (tactics.length === 0) {
492
+ skippedNoIcsTactic.push(id);
493
+ continue;
494
+ }
567
495
  const fullDesc = String(t.description || "").replace(/\s+/g, " ").trim();
568
496
  let shortDesc = fullDesc.split(/\.\s/)[0];
569
497
  if (shortDesc.length > 500) shortDesc = shortDesc.slice(0, 497) + "...";
@@ -592,7 +520,10 @@ async function refreshIcsAttack({ dry = false, cap = Infinity, _deps = {} } = {}
592
520
  existing.add(id);
593
521
  added++;
594
522
  }
595
- if (dry) { console.log(`[refresh-upstream:ics-attack] DRY-RUN: +${added} new, ${backfilled} backfills`); return { added, backfilled }; }
523
+ if (skippedNoIcsTactic.length) {
524
+ console.log(`[refresh-upstream:ics-attack] skipped ${skippedNoIcsTactic.length} technique(s) with no ICS kill-chain phase (enterprise-only, handled by the ATT&CK refresher): ${skippedNoIcsTactic.slice(0, 10).join(", ")}${skippedNoIcsTactic.length > 10 ? ", ..." : ""}`);
525
+ }
526
+ if (dry) { console.log(`[refresh-upstream:ics-attack] DRY-RUN: +${added} new, ${backfilled} backfills`); return { added, backfilled, skipped_no_ics_tactic: skippedNoIcsTactic.length }; }
596
527
  const changed = added > 0 || backfilled > 0;
597
528
  if (changed) {
598
529
  if (local._meta) { local._meta.last_updated = TODAY; local._meta.last_threat_review = TODAY; }
@@ -601,11 +532,9 @@ async function refreshIcsAttack({ dry = false, cap = Infinity, _deps = {} } = {}
601
532
  } else {
602
533
  console.log("[ok] attack-techniques.json: no upstream ICS changes — file unchanged");
603
534
  }
604
- return { added, backfilled };
535
+ return { added, backfilled, skipped_no_ics_tactic: skippedNoIcsTactic.length };
605
536
  }
606
537
 
607
- // ---------------- ATLAS ----------------
608
-
609
538
  const ATLAS_SRC = "https://raw.githubusercontent.com/mitre-atlas/atlas-navigator-data/main/dist/stix-atlas.json";
610
539
 
611
540
  function atlasTactic(phases) {
@@ -659,14 +588,10 @@ function backfillAtlas(cur, fresh) {
659
588
  if (!cur[key] && val) { cur[key] = val; touched = true; }
660
589
  }
661
590
  };
662
- // Parity with backfillAttack: include the short description + tactic in the
663
- // backfill set. Existing curated ATLAS rows often carry only {name} and need
664
- // the short description + tactic too, not just description_full/platforms/etc.
591
+ // Mirrors backfillAttack's set, short description and tactic included.
665
592
  fillIfEmpty("description", fresh.description);
666
- // tactic: atlasEntryFromStix() emits a STRING for a single-tactic technique
667
- // and an array for multi-tactic, so backfill both forms. fillIfEmpty only
668
- // writes when cur is empty, so an existing (string OR array) tactic is never
669
- // overwritten — the common single-tactic case is no longer left unhydrated.
593
+ // atlasEntryFromStix emits a string for a single-tactic technique and an array
594
+ // for multi; fillIfEmpty writes only into an empty field, so both survive.
670
595
  fillIfEmpty("tactic", fresh.tactic);
671
596
  fillIfEmpty("description_full", fresh.description_full);
672
597
  fillIfEmpty("platforms", fresh.platforms);
@@ -679,7 +604,7 @@ function backfillAtlas(cur, fresh) {
679
604
  return touched;
680
605
  }
681
606
 
682
- async function refreshAtlas({ dry = false, _deps = {} } = {}) {
607
+ async function refreshAtlas({ dry = false, cap = Infinity, _deps = {} } = {}) {
683
608
  const _fetchUrl = _deps.fetchUrl || fetchUrl;
684
609
  const _loadCatalog = _deps.loadCatalog || loadCatalog;
685
610
  const _writeCatalog = _deps.writeCatalog || writeCatalog;
@@ -721,14 +646,16 @@ async function refreshAtlas({ dry = false, _deps = {} } = {}) {
721
646
  if (backfillAtlas(cur, fresh)) { cur.last_verified = TODAY; backfilled++; }
722
647
  continue;
723
648
  }
649
+ // Bounds new adds only, matching every other refresher; backfill above runs
650
+ // on the full upstream set regardless of the cap.
651
+ if (added >= cap) continue;
724
652
  local[id] = atlasEntryFromStix(t, ext);
725
653
  existing.add(id);
726
654
  added++;
727
655
  }
728
656
  if (dry) { console.log(`[refresh-upstream:atlas] DRY-RUN: +${added} new, ${backfilled} backfills${atlasVersion ? `, v${atlasVersion}` : ""}`); return { added, backfilled, atlasVersion }; }
729
- // A newly-detected ATLAS matrix version that differs from the recorded one
730
- // is itself a change (the catalog should bump atlas_version + last_updated
731
- // together), independent of any added/backfilled rows.
657
+ // A matrix version differing from the recorded one is itself a change, so
658
+ // atlas_version and last_updated bump together with no row edits at all.
732
659
  const versionChanged = !!(atlasVersion && local._meta && local._meta.atlas_version !== atlasVersion);
733
660
  const changed = added > 0 || backfilled > 0 || versionChanged;
734
661
  if (changed) {
@@ -745,8 +672,6 @@ async function refreshAtlas({ dry = false, _deps = {} } = {}) {
745
672
  return { added, backfilled, atlasVersion };
746
673
  }
747
674
 
748
- // ---------------- D3FEND ----------------
749
-
750
675
  const D3FEND_SRC = "https://d3fend.mitre.org/ontologies/d3fend.json";
751
676
 
752
677
  function d3fendTactic(parent) {
@@ -772,12 +697,9 @@ function d3fendIdList(t, field) {
772
697
  return arr.map((x) => (x && x["@id"]) ? String(x["@id"]).replace(/^d3f:/, "") : null).filter(Boolean);
773
698
  }
774
699
 
775
- // Strip a trailing period from an OWL d3fend-id: a few upstream artifact ids
776
- // (e.g. "D3A-C4.") carry a spurious terminal dot that no id token regex can
777
- // round-trip, leaving the entry unmatchable by the orphan/cross-ref scanners.
778
- // No legitimate d3fend technique id ends in a period. The SAME normalization
779
- // must key the catalog (existing.has / local[id]) as well as the entry payload,
780
- // or a refresh re-adds the period-keyed row every run (duplicate / churn).
700
+ // A few upstream ids ("D3A-C4.") carry a terminal dot that no id-token regex
701
+ // round-trips, leaving the entry unmatchable by the orphan and cross-ref
702
+ // scanners. It must key the catalog too, or every run re-adds the dotted row.
781
703
  function normD3fendId(rawId) {
782
704
  return typeof rawId === "string" ? rawId.replace(/\.$/, "") : rawId;
783
705
  }
@@ -804,9 +726,6 @@ function d3fendEntryFromOwl(t) {
804
726
  description: desc || `D3FEND defensive technique ${id}. Reference: https://d3fend.mitre.org/technique/${id}/`,
805
727
  description_full: fullDesc || null,
806
728
  synonyms,
807
- // Relationship fields — what offensive techniques this counters,
808
- // what defensive technique it enables / falls under, the parent
809
- // narrower/broader classes for hierarchical lookup.
810
729
  defends_against: d3fendIdList(t, "d3f:defends-against"),
811
730
  counters: d3fendIdList(t, "d3f:counters"),
812
731
  enables: d3fendIdList(t, "d3f:enables"),
@@ -885,8 +804,6 @@ async function refreshD3fend({ dry = false, cap = Infinity, _deps = {} } = {}) {
885
804
  return { added, backfilled };
886
805
  }
887
806
 
888
- // ---------------- CLI dispatcher ----------------
889
-
890
807
  const SOURCES = {
891
808
  rfc: { name: "ietf-rfc-index", run: refreshRfc },
892
809
  attack: { name: "mitre-attack-stix", run: refreshAttack },
@@ -939,11 +856,9 @@ module.exports = {
939
856
  refreshD3fend,
940
857
  SOURCES,
941
858
  runCli,
942
- // Exported for regression tests: fetchUrl's status/redirect handling and
943
- // writeCatalog's atomicity are load-bearing fail-closed properties.
859
+ // Exported for tests: fail-closed fetch behaviour and the atomic write.
944
860
  fetchUrl,
945
861
  writeCatalog,
946
- // Exported for regression tests: backfillAtlas must mirror backfillAttack's
947
- // description + array-tactic backfill on curated rows.
862
+ // Exported for tests: backfillAtlas must keep mirroring backfillAttack.
948
863
  backfillAtlas
949
864
  };