@blamejs/exceptd-skills 0.19.33 → 0.19.35
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/bin/exceptd.js +895 -2828
- package/data/_indexes/_meta.json +2 -2
- package/lib/auto-discovery.js +56 -286
- package/lib/canonical-eq.js +7 -40
- package/lib/citation-resolve.js +22 -70
- package/lib/collectors/ai-api.js +170 -76
- package/lib/collectors/cicd-pipeline-compromise.js +113 -136
- package/lib/collectors/citation-hygiene.js +72 -210
- package/lib/collectors/containers.js +41 -130
- package/lib/collectors/cred-stores.js +31 -115
- package/lib/collectors/crypto-codebase.js +55 -138
- package/lib/collectors/crypto.js +24 -54
- package/lib/collectors/hardening.js +20 -78
- package/lib/collectors/kernel.js +16 -46
- package/lib/collectors/library-author.js +198 -211
- package/lib/collectors/mcp.js +24 -70
- package/lib/collectors/runtime.js +24 -86
- package/lib/collectors/sbom.js +130 -118
- package/lib/collectors/scan-excludes.js +33 -139
- package/lib/collectors/secrets.js +62 -178
- package/lib/cross-ref-api.js +39 -123
- package/lib/currency-severity.js +8 -27
- package/lib/cve-batch.js +13 -21
- package/lib/cve-cli.js +13 -20
- package/lib/cve-curation.js +72 -239
- package/lib/cve-regression-watcher.js +29 -155
- package/lib/cvss.js +13 -54
- package/lib/doctor-bucketing.js +3 -19
- package/lib/exit-codes.js +10 -42
- package/lib/flag-suggest.js +7 -25
- package/lib/framework-gap.js +39 -113
- package/lib/gap-detectors.js +37 -159
- package/lib/id-validation.js +9 -30
- package/lib/job-queue.js +13 -36
- package/lib/lint-skills.js +88 -236
- package/lib/playbook-runner.js +759 -2107
- package/lib/prefetch.js +101 -376
- package/lib/refresh-external.js +199 -633
- package/lib/refresh-network.js +78 -311
- package/lib/rfc-cli.js +23 -68
- package/lib/scoring.js +85 -146
- package/lib/sign.js +43 -229
- package/lib/source-advisories.js +43 -194
- package/lib/source-ghsa.js +37 -120
- package/lib/source-osv.js +94 -266
- package/lib/ttp-mapper.js +28 -27
- package/lib/upstream-check-cli.js +36 -29
- package/lib/upstream-check.js +19 -44
- package/lib/validate-catalog-meta.js +17 -61
- package/lib/validate-cve-catalog.js +52 -121
- package/lib/validate-indexes.js +25 -76
- package/lib/validate-package.js +16 -62
- package/lib/validate-playbooks.js +78 -286
- package/lib/validate-vendor.js +16 -49
- package/lib/verify.js +56 -286
- package/lib/version-pins.js +5 -34
- package/lib/worker-pool.js +11 -30
- package/lib/xml-tokenizer.js +47 -152
- package/manifest.json +53 -53
- package/orchestrator/dispatcher.js +17 -68
- package/orchestrator/event-bus.js +11 -74
- package/orchestrator/index.js +138 -413
- package/orchestrator/pipeline.js +28 -85
- package/orchestrator/scanner.js +34 -138
- package/orchestrator/scheduler.js +20 -84
- package/package.json +1 -1
- package/sbom.cdx.json +242 -242
- package/scripts/audit-catalog-gaps.js +9 -62
- package/scripts/audit-cross-skill.js +5 -31
- package/scripts/audit-perf.js +29 -28
- package/scripts/backfill-theater-test.js +7 -64
- package/scripts/bootstrap.js +12 -44
- package/scripts/build-indexes.js +40 -154
- package/scripts/builders/activity-feed.js +4 -14
- package/scripts/builders/catalog-summaries.js +3 -10
- package/scripts/builders/currency.js +7 -20
- package/scripts/builders/cwe-chains.js +7 -30
- package/scripts/builders/did-ladders.js +6 -13
- package/scripts/builders/frequency.js +5 -19
- package/scripts/builders/jurisdiction-clocks.js +6 -25
- package/scripts/builders/recipes.js +6 -14
- package/scripts/builders/section-offsets.js +13 -51
- package/scripts/builders/stale-content.js +7 -28
- package/scripts/builders/summary-cards.js +8 -29
- package/scripts/builders/theater-fingerprints.js +21 -31
- package/scripts/builders/token-budget.js +4 -31
- package/scripts/check-agents-md-collectors.js +26 -57
- package/scripts/check-catalog-gap-budget.js +15 -32
- package/scripts/check-changelog-extract.js +18 -48
- package/scripts/check-codebase-patterns-currency.js +6 -22
- package/scripts/check-codebase-patterns.js +63 -143
- package/scripts/check-epss-consistency.js +9 -64
- package/scripts/check-framework-gap-coverage.js +13 -31
- package/scripts/check-manifest-snapshot.js +62 -81
- package/scripts/check-sbom-currency.js +44 -142
- package/scripts/check-test-count.js +15 -52
- package/scripts/check-test-coverage.js +83 -198
- package/scripts/check-test-subjects.js +21 -62
- package/scripts/check-ttp-references.js +14 -38
- package/scripts/check-ttp-upstream.js +8 -40
- package/scripts/check-version-bump.js +9 -61
- package/scripts/check-version-tags.js +20 -121
- package/scripts/predeploy.js +38 -184
- package/scripts/refresh-manifest-snapshot.js +16 -38
- package/scripts/refresh-mitre-atlas.js +7 -8
- package/scripts/refresh-mitre-attack.js +1 -8
- package/scripts/refresh-mitre-d3fend.js +3 -9
- package/scripts/refresh-mitre-ics-attack.js +7 -8
- package/scripts/refresh-reverse-refs.js +27 -94
- package/scripts/refresh-rfc-index.js +7 -10
- package/scripts/refresh-sbom.js +31 -161
- package/scripts/refresh-upstream-catalogs.js +63 -148
- package/scripts/release.js +69 -234
- package/scripts/run-e2e-scenarios.js +26 -73
- package/scripts/sync-manifest-metadata.js +10 -34
- package/scripts/sync-package-description.js +8 -17
- package/scripts/validate-vendor-online.js +13 -44
- package/scripts/verify-shipped-tarball.js +35 -141
|
@@ -1,38 +1,10 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
"use strict";
|
|
3
3
|
/**
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
* never overwrites operator-curated entries (rows that lack the
|
|
9
|
-
* `_auto_imported: true` flag are preserved verbatim).
|
|
10
|
-
*
|
|
11
|
-
* rfc ietf-rfc-index data/rfc-references.json
|
|
12
|
-
* attack mitre-attack-stix data/attack-techniques.json
|
|
13
|
-
* atlas mitre-atlas-stix data/atlas-ttps.json
|
|
14
|
-
* d3fend mitre-d3fend-owl data/d3fend-catalog.json
|
|
15
|
-
*
|
|
16
|
-
* CLI usage:
|
|
17
|
-
*
|
|
18
|
-
* node scripts/refresh-upstream-catalogs.js # all four
|
|
19
|
-
* node scripts/refresh-upstream-catalogs.js --source rfc # one
|
|
20
|
-
* node scripts/refresh-upstream-catalogs.js --source rfc,atlas # two
|
|
21
|
-
* node scripts/refresh-upstream-catalogs.js --dry-run # report only
|
|
22
|
-
* CAP=200 node scripts/refresh-upstream-catalogs.js --source attack
|
|
23
|
-
*
|
|
24
|
-
* Module usage (per-type wrappers under scripts/refresh-{rfc,attack,atlas,
|
|
25
|
-
* d3fend}.js import the corresponding refreshX function from this file):
|
|
26
|
-
*
|
|
27
|
-
* const { refreshRfc } = require("./refresh-upstream-catalogs.js");
|
|
28
|
-
* await refreshRfc({ dry: false });
|
|
29
|
-
*
|
|
30
|
-
* npm aliases (package.json scripts):
|
|
31
|
-
* refresh-upstream-catalogs runs all four
|
|
32
|
-
* refresh-rfc-index --source rfc
|
|
33
|
-
* refresh-mitre-attack --source attack
|
|
34
|
-
* refresh-mitre-atlas --source atlas
|
|
35
|
-
* refresh-mitre-d3fend --source d3fend
|
|
4
|
+
* Refreshes the upstream catalogs (IETF RFC index, MITRE ATT&CK, ICS-ATT&CK,
|
|
5
|
+
* ATLAS, D3FEND) from their official sources. Refreshers are idempotent and
|
|
6
|
+
* additive: a populated field is never overwritten, and only `_auto_imported`
|
|
7
|
+
* rows take a status bump.
|
|
36
8
|
*/
|
|
37
9
|
|
|
38
10
|
const fs = require("fs");
|
|
@@ -42,30 +14,11 @@ const path = require("path");
|
|
|
42
14
|
const ROOT = path.join(__dirname, "..");
|
|
43
15
|
const TODAY = new Date().toISOString().slice(0, 10);
|
|
44
16
|
|
|
45
|
-
// v0.13.20 class-3.11 fix: refreshers read their required-context list
|
|
46
|
-
// from the audit SPEC. Eliminates the parallel hardcoded field arrays
|
|
47
|
-
// that v0.13.17→19 carried (and forgot to keep in sync — the v0.13.19
|
|
48
|
-
// audit found 106 ATT&CK rows missing `description` + `tactic` because
|
|
49
|
-
// the v0.13.18 backfill list omitted those fields). One source of truth
|
|
50
|
-
// = the audit-catalog-gaps SPEC.
|
|
51
|
-
const AUDIT_SPEC = require("./audit-catalog-gaps.js").SPEC;
|
|
52
|
-
function specRequiredFields(catalogKey) {
|
|
53
|
-
const spec = AUDIT_SPEC[catalogKey];
|
|
54
|
-
if (!spec || !Array.isArray(spec.required_context)) return [];
|
|
55
|
-
return spec.required_context.map((r) => r.field);
|
|
56
|
-
}
|
|
57
|
-
|
|
58
17
|
const MAX_REDIRECTS = 5;
|
|
59
18
|
|
|
60
|
-
//
|
|
61
|
-
//
|
|
62
|
-
//
|
|
63
|
-
// unbounded) and a relative Location resolves against the current URL.
|
|
64
|
-
// 2. 4xx/5xx (and a missing statusCode edge) reject instead of resolving an
|
|
65
|
-
// error body as a "successful" empty result — every consumer fails
|
|
66
|
-
// closed on an HTTP error rather than stamping _meta on a non-fetch.
|
|
67
|
-
// 3. A 3xx with no Location header rejects with a clear message rather than
|
|
68
|
-
// throwing an opaque ERR_INVALID_URL on `new URL(undefined, url)`.
|
|
19
|
+
// Rejects on anything but a 2xx, so an error body can never reach a consumer as
|
|
20
|
+
// a "successful" empty result and get _meta stamped on a non-fetch. A redirect
|
|
21
|
+
// resolves Location against the current URL and rejects past MAX_REDIRECTS.
|
|
69
22
|
function fetchUrl(url, depth = 0) {
|
|
70
23
|
return new Promise((resolve, reject) => {
|
|
71
24
|
https.get(url, { headers: { "User-Agent": "exceptd-refresh-upstream-catalogs" } }, (r) => {
|
|
@@ -99,11 +52,8 @@ function loadCatalog(rel) {
|
|
|
99
52
|
return JSON.parse(fs.readFileSync(path.join(ROOT, "data", rel), "utf8"));
|
|
100
53
|
}
|
|
101
54
|
|
|
102
|
-
//
|
|
103
|
-
//
|
|
104
|
-
// atomic on POSIX and on same-volume Windows renames (the .tmp sibling is
|
|
105
|
-
// adjacent to the target, same volume), so a reader / the next run only ever
|
|
106
|
-
// sees the complete old or complete new file. Mirrors build-indexes#writeJson.
|
|
55
|
+
// Temp sibling + atomic rename, so a crash mid-write cannot leave a truncated
|
|
56
|
+
// catalog behind. Mirrors build-indexes#writeJson.
|
|
107
57
|
function writeCatalog(rel, obj) {
|
|
108
58
|
const abs = path.join(ROOT, "data", rel);
|
|
109
59
|
const tmp = `${abs}.tmp-${process.pid}`;
|
|
@@ -117,8 +67,6 @@ function getTag(blk, tag) {
|
|
|
117
67
|
return m ? m[1].trim() : null;
|
|
118
68
|
}
|
|
119
69
|
|
|
120
|
-
// ---------------- RFC ----------------
|
|
121
|
-
|
|
122
70
|
const RFC_SRC = "https://www.rfc-editor.org/rfc-index.xml";
|
|
123
71
|
const RFC_STATUS_MAP = {
|
|
124
72
|
"INTERNET STANDARD": "Internet Standard",
|
|
@@ -136,8 +84,7 @@ const RFC_MONTHS = {
|
|
|
136
84
|
September: "09", October: "10", November: "11", December: "12"
|
|
137
85
|
};
|
|
138
86
|
|
|
139
|
-
//
|
|
140
|
-
// <obsoleted-by><doc-id>RFC123</doc-id><doc-id>RFC456</doc-id></obsoleted-by>.
|
|
87
|
+
// Every <doc-id> inside a parent tag: <obsoleted-by><doc-id>RFC123</doc-id>…
|
|
141
88
|
function getDocIdList(blk, parentTag) {
|
|
142
89
|
const re = new RegExp(`<${parentTag}>([\\s\\S]*?)<\\/${parentTag}>`);
|
|
143
90
|
const m = blk.match(re);
|
|
@@ -166,7 +113,6 @@ function getAbstract(blk) {
|
|
|
166
113
|
return paras.length ? paras.join(" ") : null;
|
|
167
114
|
}
|
|
168
115
|
|
|
169
|
-
// <keywords><kw>k1</kw><kw>k2</kw></keywords>
|
|
170
116
|
function getKeywords(blk) {
|
|
171
117
|
const m = blk.match(/<keywords>([\s\S]*?)<\/keywords>/);
|
|
172
118
|
if (!m) return [];
|
|
@@ -211,10 +157,6 @@ function parseRfcEntry(blk) {
|
|
|
211
157
|
const published = year && month ? `${year}-${month}` : (year || "unknown");
|
|
212
158
|
const hasErrata = /<errata-url>/.test(blk);
|
|
213
159
|
const obsoleted = /<obsoleted-by>/.test(blk);
|
|
214
|
-
// New context-search fields (v0.13.18+): the AI needs more than the
|
|
215
|
-
// title to locate an RFC by topic. abstract + keywords + area +
|
|
216
|
-
// wg_acronym + stream + authors + obsoletes/updates relationships are
|
|
217
|
-
// all present in the IETF index — we were only extracting title before.
|
|
218
160
|
const abstract = getAbstract(blk);
|
|
219
161
|
const keywords = getKeywords(blk);
|
|
220
162
|
const area = getTag(blk, "area");
|
|
@@ -235,7 +177,7 @@ function parseRfcEntry(blk) {
|
|
|
235
177
|
};
|
|
236
178
|
}
|
|
237
179
|
|
|
238
|
-
async function refreshRfc({ dry = false, _deps = {} } = {}) {
|
|
180
|
+
async function refreshRfc({ dry = false, cap = Infinity, _deps = {} } = {}) {
|
|
239
181
|
const _fetchUrl = _deps.fetchUrl || fetchUrl;
|
|
240
182
|
const _loadCatalog = _deps.loadCatalog || loadCatalog;
|
|
241
183
|
const _writeCatalog = _deps.writeCatalog || writeCatalog;
|
|
@@ -253,12 +195,8 @@ async function refreshRfc({ dry = false, _deps = {} } = {}) {
|
|
|
253
195
|
if (e.obsoleted || e.status === "HISTORIC" || e.status === "UNKNOWN") continue;
|
|
254
196
|
entries.push(e);
|
|
255
197
|
}
|
|
256
|
-
//
|
|
257
|
-
//
|
|
258
|
-
// error/empty/soft-error body (a 200 with a CDN error page, a captive portal,
|
|
259
|
-
// or a truncated body the HTTP-status guard can't see) — refuse to stamp
|
|
260
|
-
// _meta or write rfc-references.json, matching the JSON-parse failures the
|
|
261
|
-
// STIX sources surface for free (an empty index is never a legitimate result).
|
|
198
|
+
// A successful fetch of a ~9000-RFC index can never parse to zero. Zero means a
|
|
199
|
+
// soft-error body — a CDN error page under a 200 — the status guard cannot see.
|
|
262
200
|
if (backfillable.length === 0) {
|
|
263
201
|
throw new Error("RFC index parsed 0 entries (fetch likely returned an error/empty body) — refusing to stamp _meta or write rfc-references.json");
|
|
264
202
|
}
|
|
@@ -266,19 +204,16 @@ async function refreshRfc({ dry = false, _deps = {} } = {}) {
|
|
|
266
204
|
const cat = _loadCatalog("rfc-references.json");
|
|
267
205
|
const existing = new Set(Object.keys(cat).filter((k) => k !== "_meta"));
|
|
268
206
|
let added = 0, statusBumped = 0, backfilledCount = 0;
|
|
269
|
-
//
|
|
270
|
-
//
|
|
271
|
-
// obsoleted RFC in for documentation; we still want abstract/authors.
|
|
207
|
+
// Backfill every existing row from the broader entry set, obsoleted historics
|
|
208
|
+
// included: a curated obsoleted RFC still wants its abstract and authors.
|
|
272
209
|
for (const e of backfillable) {
|
|
273
210
|
const id = `RFC-${e.num}`;
|
|
274
211
|
if (!existing.has(id)) continue;
|
|
275
212
|
const cur = cat[id];
|
|
276
213
|
if (!cur) continue;
|
|
277
214
|
let touched = false;
|
|
278
|
-
//
|
|
279
|
-
//
|
|
280
|
-
// status field on an existing curated row). Fall back to the raw upstream
|
|
281
|
-
// status, and only bump when the mapped value actually differs.
|
|
215
|
+
// An upstream status outside RFC_STATUS_MAP falls back to the raw status
|
|
216
|
+
// rather than writing `undefined` and dropping the field on an existing row.
|
|
282
217
|
const mapped = RFC_STATUS_MAP[e.status] || e.status;
|
|
283
218
|
if (cur._auto_imported && mapped && cur.status !== mapped) {
|
|
284
219
|
cur.status = mapped;
|
|
@@ -302,18 +237,17 @@ async function refreshRfc({ dry = false, _deps = {} } = {}) {
|
|
|
302
237
|
if (!cur.html_url) { cur.html_url = `https://www.rfc-editor.org/rfc/rfc${e.num}.html`; touched = true; }
|
|
303
238
|
if (touched) { cur.last_verified = TODAY; backfilledCount++; }
|
|
304
239
|
}
|
|
305
|
-
//
|
|
306
|
-
//
|
|
307
|
-
//
|
|
308
|
-
// had no offline answer and forced a datatracker lookup. They are added here
|
|
309
|
-
// marked `_obsoleted` (with obsoleted_by populated) so the resolver can say
|
|
310
|
-
// "Historic, superseded by RFC X" offline. UNKNOWN-status index rows
|
|
311
|
-
// (placeholders / not-issued numbers) are still skipped.
|
|
240
|
+
// New rows come from the full index, not just the current series: obsoleted and
|
|
241
|
+
// historic RFCs land marked `_obsoleted` with obsoleted_by populated, so the
|
|
242
|
+
// resolver answers "Historic, superseded by RFC X" offline. UNKNOWN is skipped.
|
|
312
243
|
for (const e of backfillable) {
|
|
313
244
|
const id = `RFC-${e.num}`;
|
|
314
245
|
// Existing rows handled in the first-pass backfill above.
|
|
315
246
|
if (existing.has(id)) continue;
|
|
316
247
|
if (e.status === "UNKNOWN") continue;
|
|
248
|
+
// The cap bounds new adds only; the backfill pass above is unbounded, so a
|
|
249
|
+
// capped run still completes context on every row already curated.
|
|
250
|
+
if (added >= cap) continue;
|
|
317
251
|
const obsoleted = !!e.obsoleted || e.status === "HISTORIC";
|
|
318
252
|
cat[id] = {
|
|
319
253
|
number: e.num,
|
|
@@ -347,10 +281,8 @@ async function refreshRfc({ dry = false, _deps = {} } = {}) {
|
|
|
347
281
|
existing.add(id);
|
|
348
282
|
added++;
|
|
349
283
|
}
|
|
350
|
-
//
|
|
351
|
-
//
|
|
352
|
-
// spurious _meta-only diff (and so the freshness gates stay honest — a
|
|
353
|
-
// wall-clock restamp on an unchanged catalog masks real staleness).
|
|
284
|
+
// Restamp _meta and write only on a real change: restamping an unchanged
|
|
285
|
+
// catalog masks real staleness from the freshness gates.
|
|
354
286
|
const changed = added > 0 || backfilledCount > 0 || statusBumped > 0;
|
|
355
287
|
if (dry) {
|
|
356
288
|
console.log(`[refresh-upstream:rfc] DRY-RUN: +${added} new, ${backfilledCount} backfilled, ${statusBumped} status bumps.`);
|
|
@@ -369,8 +301,6 @@ async function refreshRfc({ dry = false, _deps = {} } = {}) {
|
|
|
369
301
|
return { added, statusBumped, backfilled: backfilledCount };
|
|
370
302
|
}
|
|
371
303
|
|
|
372
|
-
// ---------------- ATT&CK ----------------
|
|
373
|
-
|
|
374
304
|
const ATTACK_SRC = "https://raw.githubusercontent.com/mitre/cti/master/enterprise-attack/enterprise-attack.json";
|
|
375
305
|
const ATTACK_TACTIC_NAME = {
|
|
376
306
|
"reconnaissance": "Reconnaissance", "resource-development": "Resource Development",
|
|
@@ -383,10 +313,8 @@ const ATTACK_TACTIC_NAME = {
|
|
|
383
313
|
"exfiltration": "Exfiltration", "impact": "Impact"
|
|
384
314
|
};
|
|
385
315
|
|
|
386
|
-
//
|
|
387
|
-
//
|
|
388
|
-
// MITRE description; description (short) is the first-sentence
|
|
389
|
-
// extractive summary used by token-budgeted consumers.
|
|
316
|
+
// description_full keeps MITRE's text; description is the first-sentence
|
|
317
|
+
// summary that token-budgeted consumers read.
|
|
390
318
|
function attackEntryFromStix(t, extRef) {
|
|
391
319
|
const id = extRef.external_id;
|
|
392
320
|
const tactics = (t.kill_chain_phases || [])
|
|
@@ -429,13 +357,9 @@ function backfillAttack(cur, fresh) {
|
|
|
429
357
|
if (!cur[key] && val) { cur[key] = val; touched = true; }
|
|
430
358
|
}
|
|
431
359
|
};
|
|
432
|
-
// v0.13.19: include description (short) + tactic in the backfill set.
|
|
433
|
-
// Existing rows from the original 110-entry catalog often have only
|
|
434
|
-
// {name, version} — they need tactic + short-description too, not just
|
|
435
|
-
// the v0.13.18 description_full / platforms / detection additions.
|
|
436
360
|
fillIfEmpty("description", fresh.description);
|
|
437
|
-
//
|
|
438
|
-
//
|
|
361
|
+
// Arrays only: an existing row may carry a string tactic, which must not be
|
|
362
|
+
// overwritten with the array form.
|
|
439
363
|
if ((!cur.tactic || (Array.isArray(cur.tactic) && cur.tactic.length === 0)) && Array.isArray(fresh.tactic) && fresh.tactic.length) {
|
|
440
364
|
cur.tactic = fresh.tactic;
|
|
441
365
|
touched = true;
|
|
@@ -462,12 +386,9 @@ async function refreshAttack({ dry = false, cap = Infinity, _deps = {} } = {}) {
|
|
|
462
386
|
console.log("[refresh-upstream:attack] fetching MITRE ATT&CK STIX...");
|
|
463
387
|
const body = await _fetchUrl(ATTACK_SRC);
|
|
464
388
|
const stix = JSON.parse(body);
|
|
465
|
-
//
|
|
466
|
-
//
|
|
467
|
-
//
|
|
468
|
-
// the context fields (name / description / platforms) from the
|
|
469
|
-
// pre-revocation STIX record. Same logic as the RFC obsoleted-but-
|
|
470
|
-
// backfillable two-pass design.
|
|
389
|
+
// New adds take live techniques only; the backfill pass includes revoked and
|
|
390
|
+
// deprecated ones, so a curated row on a revoked MITRE ID still gets its
|
|
391
|
+
// context fields from the pre-revocation record.
|
|
471
392
|
const liveTechs = (stix.objects || []).filter(
|
|
472
393
|
(o) => o.type === "attack-pattern" && !o.revoked && !o.x_mitre_deprecated
|
|
473
394
|
);
|
|
@@ -488,8 +409,6 @@ async function refreshAttack({ dry = false, cap = Infinity, _deps = {} } = {}) {
|
|
|
488
409
|
});
|
|
489
410
|
let added = 0;
|
|
490
411
|
let backfilled = 0;
|
|
491
|
-
// First pass: backfill existing rows against the FULL technique set
|
|
492
|
-
// (including revoked) so operator-curated rows still get context.
|
|
493
412
|
for (const t of backfillTechs) {
|
|
494
413
|
const extRef = (t.external_references || []).find((r) => r.source_name === "mitre-attack");
|
|
495
414
|
if (!extRef || !extRef.external_id) continue;
|
|
@@ -502,7 +421,6 @@ async function refreshAttack({ dry = false, cap = Infinity, _deps = {} } = {}) {
|
|
|
502
421
|
backfilled++;
|
|
503
422
|
}
|
|
504
423
|
}
|
|
505
|
-
// Second pass: add new entries from live techniques only.
|
|
506
424
|
for (const t of techs) {
|
|
507
425
|
const extRef = (t.external_references || []).find((r) => r.source_name === "mitre-attack");
|
|
508
426
|
if (!extRef || !extRef.external_id) continue;
|
|
@@ -525,8 +443,6 @@ async function refreshAttack({ dry = false, cap = Infinity, _deps = {} } = {}) {
|
|
|
525
443
|
return { added, backfilled };
|
|
526
444
|
}
|
|
527
445
|
|
|
528
|
-
// ---------------- ICS-ATT&CK ----------------
|
|
529
|
-
|
|
530
446
|
const ICS_ATTACK_SRC = "https://raw.githubusercontent.com/mitre/cti/master/ics-attack/ics-attack.json";
|
|
531
447
|
const ICS_TACTIC_NAME = {
|
|
532
448
|
"initial-access": "Initial Access (ICS)",
|
|
@@ -557,6 +473,7 @@ async function refreshIcsAttack({ dry = false, cap = Infinity, _deps = {} } = {}
|
|
|
557
473
|
const local = _loadCatalog("attack-techniques.json");
|
|
558
474
|
const existing = new Set(Object.keys(local).filter((k) => k !== "_meta"));
|
|
559
475
|
let added = 0, backfilled = 0;
|
|
476
|
+
const skippedNoIcsTactic = [];
|
|
560
477
|
for (const t of techs) {
|
|
561
478
|
const extRef = (t.external_references || []).find((r) => r.source_name === "mitre-ics-attack" || r.source_name === "mitre-attack");
|
|
562
479
|
if (!extRef || !extRef.external_id) continue;
|
|
@@ -564,6 +481,17 @@ async function refreshIcsAttack({ dry = false, cap = Infinity, _deps = {} } = {}
|
|
|
564
481
|
const tactics = (t.kill_chain_phases || [])
|
|
565
482
|
.filter((p) => (p.kill_chain_name || "").includes("ics"))
|
|
566
483
|
.map((p) => ICS_TACTIC_NAME[p.phase_name] || `${p.phase_name} (ICS)`);
|
|
484
|
+
// The external-reference match above accepts a cross-listed enterprise
|
|
485
|
+
// reference, while the tactic map keeps ICS kill-chain phases only. An
|
|
486
|
+
// object matched by the first and not the second would land with
|
|
487
|
+
// `tactic: []`, which audit-catalog-gaps counts as a missing-context gap
|
|
488
|
+
// the moment it is written — the import spending gap budget on itself.
|
|
489
|
+
// Enterprise-only techniques belong to refreshAttack, so skip them here,
|
|
490
|
+
// and report the count so the omission is observable rather than silent.
|
|
491
|
+
if (tactics.length === 0) {
|
|
492
|
+
skippedNoIcsTactic.push(id);
|
|
493
|
+
continue;
|
|
494
|
+
}
|
|
567
495
|
const fullDesc = String(t.description || "").replace(/\s+/g, " ").trim();
|
|
568
496
|
let shortDesc = fullDesc.split(/\.\s/)[0];
|
|
569
497
|
if (shortDesc.length > 500) shortDesc = shortDesc.slice(0, 497) + "...";
|
|
@@ -592,7 +520,10 @@ async function refreshIcsAttack({ dry = false, cap = Infinity, _deps = {} } = {}
|
|
|
592
520
|
existing.add(id);
|
|
593
521
|
added++;
|
|
594
522
|
}
|
|
595
|
-
if (
|
|
523
|
+
if (skippedNoIcsTactic.length) {
|
|
524
|
+
console.log(`[refresh-upstream:ics-attack] skipped ${skippedNoIcsTactic.length} technique(s) with no ICS kill-chain phase (enterprise-only, handled by the ATT&CK refresher): ${skippedNoIcsTactic.slice(0, 10).join(", ")}${skippedNoIcsTactic.length > 10 ? ", ..." : ""}`);
|
|
525
|
+
}
|
|
526
|
+
if (dry) { console.log(`[refresh-upstream:ics-attack] DRY-RUN: +${added} new, ${backfilled} backfills`); return { added, backfilled, skipped_no_ics_tactic: skippedNoIcsTactic.length }; }
|
|
596
527
|
const changed = added > 0 || backfilled > 0;
|
|
597
528
|
if (changed) {
|
|
598
529
|
if (local._meta) { local._meta.last_updated = TODAY; local._meta.last_threat_review = TODAY; }
|
|
@@ -601,11 +532,9 @@ async function refreshIcsAttack({ dry = false, cap = Infinity, _deps = {} } = {}
|
|
|
601
532
|
} else {
|
|
602
533
|
console.log("[ok] attack-techniques.json: no upstream ICS changes — file unchanged");
|
|
603
534
|
}
|
|
604
|
-
return { added, backfilled };
|
|
535
|
+
return { added, backfilled, skipped_no_ics_tactic: skippedNoIcsTactic.length };
|
|
605
536
|
}
|
|
606
537
|
|
|
607
|
-
// ---------------- ATLAS ----------------
|
|
608
|
-
|
|
609
538
|
const ATLAS_SRC = "https://raw.githubusercontent.com/mitre-atlas/atlas-navigator-data/main/dist/stix-atlas.json";
|
|
610
539
|
|
|
611
540
|
function atlasTactic(phases) {
|
|
@@ -659,14 +588,10 @@ function backfillAtlas(cur, fresh) {
|
|
|
659
588
|
if (!cur[key] && val) { cur[key] = val; touched = true; }
|
|
660
589
|
}
|
|
661
590
|
};
|
|
662
|
-
//
|
|
663
|
-
// backfill set. Existing curated ATLAS rows often carry only {name} and need
|
|
664
|
-
// the short description + tactic too, not just description_full/platforms/etc.
|
|
591
|
+
// Mirrors backfillAttack's set, short description and tactic included.
|
|
665
592
|
fillIfEmpty("description", fresh.description);
|
|
666
|
-
//
|
|
667
|
-
//
|
|
668
|
-
// writes when cur is empty, so an existing (string OR array) tactic is never
|
|
669
|
-
// overwritten — the common single-tactic case is no longer left unhydrated.
|
|
593
|
+
// atlasEntryFromStix emits a string for a single-tactic technique and an array
|
|
594
|
+
// for multi; fillIfEmpty writes only into an empty field, so both survive.
|
|
670
595
|
fillIfEmpty("tactic", fresh.tactic);
|
|
671
596
|
fillIfEmpty("description_full", fresh.description_full);
|
|
672
597
|
fillIfEmpty("platforms", fresh.platforms);
|
|
@@ -679,7 +604,7 @@ function backfillAtlas(cur, fresh) {
|
|
|
679
604
|
return touched;
|
|
680
605
|
}
|
|
681
606
|
|
|
682
|
-
async function refreshAtlas({ dry = false, _deps = {} } = {}) {
|
|
607
|
+
async function refreshAtlas({ dry = false, cap = Infinity, _deps = {} } = {}) {
|
|
683
608
|
const _fetchUrl = _deps.fetchUrl || fetchUrl;
|
|
684
609
|
const _loadCatalog = _deps.loadCatalog || loadCatalog;
|
|
685
610
|
const _writeCatalog = _deps.writeCatalog || writeCatalog;
|
|
@@ -721,14 +646,16 @@ async function refreshAtlas({ dry = false, _deps = {} } = {}) {
|
|
|
721
646
|
if (backfillAtlas(cur, fresh)) { cur.last_verified = TODAY; backfilled++; }
|
|
722
647
|
continue;
|
|
723
648
|
}
|
|
649
|
+
// Bounds new adds only, matching every other refresher; backfill above runs
|
|
650
|
+
// on the full upstream set regardless of the cap.
|
|
651
|
+
if (added >= cap) continue;
|
|
724
652
|
local[id] = atlasEntryFromStix(t, ext);
|
|
725
653
|
existing.add(id);
|
|
726
654
|
added++;
|
|
727
655
|
}
|
|
728
656
|
if (dry) { console.log(`[refresh-upstream:atlas] DRY-RUN: +${added} new, ${backfilled} backfills${atlasVersion ? `, v${atlasVersion}` : ""}`); return { added, backfilled, atlasVersion }; }
|
|
729
|
-
// A
|
|
730
|
-
//
|
|
731
|
-
// together), independent of any added/backfilled rows.
|
|
657
|
+
// A matrix version differing from the recorded one is itself a change, so
|
|
658
|
+
// atlas_version and last_updated bump together with no row edits at all.
|
|
732
659
|
const versionChanged = !!(atlasVersion && local._meta && local._meta.atlas_version !== atlasVersion);
|
|
733
660
|
const changed = added > 0 || backfilled > 0 || versionChanged;
|
|
734
661
|
if (changed) {
|
|
@@ -745,8 +672,6 @@ async function refreshAtlas({ dry = false, _deps = {} } = {}) {
|
|
|
745
672
|
return { added, backfilled, atlasVersion };
|
|
746
673
|
}
|
|
747
674
|
|
|
748
|
-
// ---------------- D3FEND ----------------
|
|
749
|
-
|
|
750
675
|
const D3FEND_SRC = "https://d3fend.mitre.org/ontologies/d3fend.json";
|
|
751
676
|
|
|
752
677
|
function d3fendTactic(parent) {
|
|
@@ -772,12 +697,9 @@ function d3fendIdList(t, field) {
|
|
|
772
697
|
return arr.map((x) => (x && x["@id"]) ? String(x["@id"]).replace(/^d3f:/, "") : null).filter(Boolean);
|
|
773
698
|
}
|
|
774
699
|
|
|
775
|
-
//
|
|
776
|
-
//
|
|
777
|
-
//
|
|
778
|
-
// No legitimate d3fend technique id ends in a period. The SAME normalization
|
|
779
|
-
// must key the catalog (existing.has / local[id]) as well as the entry payload,
|
|
780
|
-
// or a refresh re-adds the period-keyed row every run (duplicate / churn).
|
|
700
|
+
// A few upstream ids ("D3A-C4.") carry a terminal dot that no id-token regex
|
|
701
|
+
// round-trips, leaving the entry unmatchable by the orphan and cross-ref
|
|
702
|
+
// scanners. It must key the catalog too, or every run re-adds the dotted row.
|
|
781
703
|
function normD3fendId(rawId) {
|
|
782
704
|
return typeof rawId === "string" ? rawId.replace(/\.$/, "") : rawId;
|
|
783
705
|
}
|
|
@@ -804,9 +726,6 @@ function d3fendEntryFromOwl(t) {
|
|
|
804
726
|
description: desc || `D3FEND defensive technique ${id}. Reference: https://d3fend.mitre.org/technique/${id}/`,
|
|
805
727
|
description_full: fullDesc || null,
|
|
806
728
|
synonyms,
|
|
807
|
-
// Relationship fields — what offensive techniques this counters,
|
|
808
|
-
// what defensive technique it enables / falls under, the parent
|
|
809
|
-
// narrower/broader classes for hierarchical lookup.
|
|
810
729
|
defends_against: d3fendIdList(t, "d3f:defends-against"),
|
|
811
730
|
counters: d3fendIdList(t, "d3f:counters"),
|
|
812
731
|
enables: d3fendIdList(t, "d3f:enables"),
|
|
@@ -885,8 +804,6 @@ async function refreshD3fend({ dry = false, cap = Infinity, _deps = {} } = {}) {
|
|
|
885
804
|
return { added, backfilled };
|
|
886
805
|
}
|
|
887
806
|
|
|
888
|
-
// ---------------- CLI dispatcher ----------------
|
|
889
|
-
|
|
890
807
|
const SOURCES = {
|
|
891
808
|
rfc: { name: "ietf-rfc-index", run: refreshRfc },
|
|
892
809
|
attack: { name: "mitre-attack-stix", run: refreshAttack },
|
|
@@ -939,11 +856,9 @@ module.exports = {
|
|
|
939
856
|
refreshD3fend,
|
|
940
857
|
SOURCES,
|
|
941
858
|
runCli,
|
|
942
|
-
// Exported for
|
|
943
|
-
// writeCatalog's atomicity are load-bearing fail-closed properties.
|
|
859
|
+
// Exported for tests: fail-closed fetch behaviour and the atomic write.
|
|
944
860
|
fetchUrl,
|
|
945
861
|
writeCatalog,
|
|
946
|
-
// Exported for
|
|
947
|
-
// description + array-tactic backfill on curated rows.
|
|
862
|
+
// Exported for tests: backfillAtlas must keep mirroring backfillAttack.
|
|
948
863
|
backfillAtlas
|
|
949
864
|
};
|