@blamejs/exceptd-skills 0.13.18 → 0.13.20

package/scripts/check-test-coverage.js

@@ -329,6 +329,15 @@ function extractPlaybookIds(content) {
   return { indicators: ind, artifacts: arts };
 }
 
+// Canonical-form recursive equality replaces JSON.stringify comparison.
+// Pre-v0.13.20 the comparator was JSON.stringify(before.iocs) !==
+// JSON.stringify(after.iocs) — non-canonical: key order, trailing
+// whitespace, and numeric format differences all flagged as "changed"
+// when the operator made no semantic change. Symptoms were patched
+// twice with skip rules (_auto_imported, _iocs_stub) instead of fixing
+// the comparator. v0.13.20 fixes the root cause.
+const { canonicalEqual } = require("../lib/canonical-eq");
+
 function extractCveIocChanges(beforeStr, afterStr) {
   const before = safeParse(beforeStr) || {};
   const after = safeParse(afterStr) || {};
@@ -336,19 +345,16 @@ function extractCveIocChanges(beforeStr, afterStr) {
   const ids = new Set([...Object.keys(before), ...Object.keys(after)]);
   for (const id of ids) {
     if (!/^CVE-\d{4}-\d+/.test(id)) continue;
-    // v0.13.18: skip bulk-imported entries. Auto-imported rows carry stub
-    // IoCs by design ("Refer to vendor advisory for IOC list — bulk-
-    // imported KEV entry, IOCs not extracted at intake time."); their
-    // per-entry IoCs are not the operator-curated surface the diff-
-    // coverage gate is designed to police. When an operator later
-    // curates the row, the next run will see the diff against the
-    // curated form and route through the normal coverage path.
+    // v0.13.18 retained skip rule: bulk-imported rows whose IoCs are
+    // stub-by-design on both sides — pure intake-class events, not
+    // operator curation. Removing this would surface every fresh KEV
+    // bulk-import as a per-CVE iocs-modified finding.
     const beforeAuto = !!(before[id] && before[id]._auto_imported);
     const afterAuto  = !!(after[id]  && after[id]._auto_imported);
     if (beforeAuto && afterAuto) continue;
-    const b = JSON.stringify((before[id] && before[id].iocs) || null);
-    const a = JSON.stringify((after[id] && after[id].iocs) || null);
-    if (b !== a) changed.add(id);
+    const bIocs = (before[id] && before[id].iocs) || null;
+    const aIocs = (after[id]  && after[id].iocs)  || null;
+    if (!canonicalEqual(bIocs, aIocs)) changed.add(id);
   }
   return changed;
 }

package/scripts/refresh-mitre-ics-attack.js

@@ -0,0 +1,15 @@
+#!/usr/bin/env node
+"use strict";
+/**
+ * scripts/refresh-mitre-ics-attack.js
+ *
+ * Thin per-type wrapper for the MITRE ICS-attack STIX refresher. Logic
+ * lives in scripts/refresh-upstream-catalogs.js#refreshIcsAttack.
+ *
+ *   node scripts/refresh-mitre-ics-attack.js [--dry-run]
+ *
+ * Wired as `npm run refresh-mitre-ics-attack`.
+ */
+const { refreshIcsAttack } = require("./refresh-upstream-catalogs.js");
+const dry = process.argv.includes("--dry-run");
+refreshIcsAttack({ dry }).catch((e) => { console.error("[err]", e); process.exit(1); });

package/scripts/refresh-upstream-catalogs.js

@@ -42,6 +42,19 @@ const path = require("path");
 const ROOT = path.join(__dirname, "..");
 const TODAY = new Date().toISOString().slice(0, 10);
 
+// v0.13.20 class-3.11 fix: refreshers read their required-context list
+// from the audit SPEC. Eliminates the parallel hardcoded field arrays
+// that v0.13.17→19 carried (and forgot to keep in sync — the v0.13.19
+// audit found 106 ATT&CK rows missing `description` + `tactic` because
+// the v0.13.18 backfill list omitted those fields). One source of truth
+// = the audit-catalog-gaps SPEC.
+const AUDIT_SPEC = require("./audit-catalog-gaps.js").SPEC;
+function specRequiredFields(catalogKey) {
+  const spec = AUDIT_SPEC[catalogKey];
+  if (!spec || !Array.isArray(spec.required_context)) return [];
+  return spec.required_context.map((r) => r.field);
+}
+
 function fetchUrl(url) {
   return new Promise((resolve, reject) => {
     https.get(url, { headers: { "User-Agent": "exceptd-refresh-upstream-catalogs" } }, (r) => {
@@ -190,53 +203,56 @@ async function refreshRfc({ dry = false } = {}) {
   const body = await fetchUrl(RFC_SRC);
   console.log(`[refresh-upstream:rfc] index size: ${(body.length / 1e6).toFixed(2)} MB`);
   const re = /<rfc-entry>([\s\S]*?)<\/rfc-entry>/g;
-  const entries = [];
+  const entries = [];           // current — eligible for new-add
+  const backfillable = [];      // any-status — eligible for backfill on existing rows
   let m;
   while ((m = re.exec(body)) !== null) {
     const e = parseRfcEntry(m[1]);
     if (!e) continue;
+    backfillable.push(e);
     if (e.obsoleted || e.status === "HISTORIC" || e.status === "UNKNOWN") continue;
     entries.push(e);
   }
-  console.log(`[refresh-upstream:rfc] current entries: ${entries.length}`);
+  console.log(`[refresh-upstream:rfc] current entries: ${entries.length} (+ ${backfillable.length - entries.length} obsoleted/historic available for backfill on existing rows)`);
   const cat = loadCatalog("rfc-references.json");
   const existing = new Set(Object.keys(cat).filter((k) => k !== "_meta"));
-  let added = 0, statusBumped = 0;
-  for (const e of entries) {
+  let added = 0, statusBumped = 0, backfilledCount = 0;
+  // First pass: backfill ALL existing rows from the broader entry set
+  // (including obsoleted historics). Operator may have curated an
+  // obsoleted RFC in for documentation; we still want abstract/authors.
+  for (const e of backfillable) {
     const id = `RFC-${e.num}`;
-    if (existing.has(id)) {
-      const cur = cat[id];
-      let touched = false;
-      // Status bumps only on auto-imported rows — operator-curated rows
-      // may have a deliberately-different status (e.g. tracking the
-      // older standard while a successor is in Proposed Standard limbo).
-      if (cur && cur._auto_imported && cur.status !== RFC_STATUS_MAP[e.status]) {
-        cur.status = RFC_STATUS_MAP[e.status];
-        touched = true;
-        statusBumped++;
-      }
-      // Backfill context-search fields on ANY row (auto-imported or
-      // operator-curated) when the local field is absent. Backfill is
-      // additive — it never overwrites a value an operator chose, only
-      // fills holes left by the pre-v0.13.18 catalog shape that stored
-      // only title + status.
-      if (cur && !cur.abstract && e.abstract) { cur.abstract = e.abstract; touched = true; }
-      if (cur && (!cur.keywords || cur.keywords.length === 0) && e.keywords.length) { cur.keywords = e.keywords; touched = true; }
-      if (cur && !cur.area && e.area) { cur.area = e.area; touched = true; }
-      if (cur && !cur.working_group && e.wg) { cur.working_group = e.wg; touched = true; }
-      if (cur && !cur.stream && e.stream) { cur.stream = e.stream; touched = true; }
-      if (cur && (!cur.authors || cur.authors.length === 0) && e.authors.length) { cur.authors = e.authors; touched = true; }
-      if (cur && !cur.doi && e.doi) { cur.doi = e.doi; touched = true; }
-      if (cur && !cur.page_count && e.pageCount) { cur.page_count = e.pageCount; touched = true; }
-      if (cur && (!cur.obsoletes || cur.obsoletes.length === 0) && e.obsoletes.length) { cur.obsoletes = e.obsoletes; touched = true; }
-      if (cur && (!cur.updates || cur.updates.length === 0) && e.updates.length) { cur.updates = e.updates; touched = true; }
-      if (cur && (!cur.updated_by || cur.updated_by.length === 0) && e.updatedBy.length) { cur.updated_by = e.updatedBy; touched = true; }
-      if (cur && (!cur.is_also || cur.is_also.length === 0) && e.isAlso.length) { cur.is_also = e.isAlso; touched = true; }
-      if (cur && !cur.txt_url) { cur.txt_url = `https://www.rfc-editor.org/rfc/rfc${e.num}.txt`; touched = true; }
-      if (cur && !cur.html_url) { cur.html_url = `https://www.rfc-editor.org/rfc/rfc${e.num}.html`; touched = true; }
-      if (touched && cur) cur.last_verified = TODAY;
-      continue;
+    if (!existing.has(id)) continue;
+    const cur = cat[id];
+    if (!cur) continue;
+    let touched = false;
+    if (cur._auto_imported && cur.status !== RFC_STATUS_MAP[e.status]) {
+      cur.status = RFC_STATUS_MAP[e.status];
+      touched = true;
+      statusBumped++;
     }
+    if (!cur.abstract && e.abstract) { cur.abstract = e.abstract; touched = true; }
+    if ((!cur.keywords || cur.keywords.length === 0) && e.keywords.length) { cur.keywords = e.keywords; touched = true; }
+    if (!cur.area && e.area) { cur.area = e.area; touched = true; }
+    if (!cur.working_group && e.wg) { cur.working_group = e.wg; touched = true; }
+    if (!cur.stream && e.stream) { cur.stream = e.stream; touched = true; }
+    if ((!cur.authors || cur.authors.length === 0) && e.authors.length) { cur.authors = e.authors; touched = true; }
+    if (!cur.doi && e.doi) { cur.doi = e.doi; touched = true; }
+    if (!cur.page_count && e.pageCount) { cur.page_count = e.pageCount; touched = true; }
+    if ((!cur.obsoletes || cur.obsoletes.length === 0) && e.obsoletes.length) { cur.obsoletes = e.obsoletes; touched = true; }
+    if ((!cur.updates || cur.updates.length === 0) && e.updates.length) { cur.updates = e.updates; touched = true; }
+    if ((!cur.updated_by || cur.updated_by.length === 0) && e.updatedBy.length) { cur.updated_by = e.updatedBy; touched = true; }
+    if ((!cur.obsoleted_by || cur.obsoleted_by.length === 0) && e.obsoletedBy.length) { cur.obsoleted_by = e.obsoletedBy; touched = true; }
+    if ((!cur.is_also || cur.is_also.length === 0) && e.isAlso.length) { cur.is_also = e.isAlso; touched = true; }
+    if (!cur.txt_url) { cur.txt_url = `https://www.rfc-editor.org/rfc/rfc${e.num}.txt`; touched = true; }
+    if (!cur.html_url) { cur.html_url = `https://www.rfc-editor.org/rfc/rfc${e.num}.html`; touched = true; }
+    if (touched) { cur.last_verified = TODAY; backfilledCount++; }
+  }
+  // Second pass: add new "current" entries that weren't in the catalog.
+  for (const e of entries) {
+    const id = `RFC-${e.num}`;
+    // Existing rows handled in the first-pass backfill above.
+    if (existing.has(id)) continue;
     cat[id] = {
       number: e.num,
       title: e.title,
@@ -272,12 +288,12 @@ async function refreshRfc({ dry = false } = {}) {
     cat._meta.last_threat_review = TODAY;
   }
   if (dry) {
-    console.log(`[refresh-upstream:rfc] DRY-RUN: +${added} new, ${statusBumped} status bumps.`);
-    return { added, statusBumped };
+    console.log(`[refresh-upstream:rfc] DRY-RUN: +${added} new, ${backfilledCount} backfilled, ${statusBumped} status bumps.`);
+    return { added, statusBumped, backfilled: backfilledCount };
   }
   writeCatalog("rfc-references.json", cat);
-  console.log(`[ok] rfc-references.json: +${added} entries, ${statusBumped} status bumps (now ${existing.size} total)`);
-  return { added, statusBumped };
+  console.log(`[ok] rfc-references.json: +${added} entries, ${backfilledCount} backfilled, ${statusBumped} status bumps (now ${existing.size} total)`);
+  return { added, statusBumped, backfilled: backfilledCount };
 }
 
 // ---------------- ATT&CK ----------------
@@ -340,6 +356,17 @@ function backfillAttack(cur, fresh) {
       if (!cur[key] && val) { cur[key] = val; touched = true; }
     }
   };
+  // v0.13.19: include description (short) + tactic in the backfill set.
+  // Existing rows from the original 110-entry catalog often have only
+  // {name, version} — they need tactic + short-description too, not just
+  // the v0.13.18 description_full / platforms / detection additions.
+  fillIfEmpty("description", fresh.description);
+  // tactic: arrays only (existing rows may have a string tactic; do
+  // not overwrite a stringified tactic with an array form).
+  if ((!cur.tactic || (Array.isArray(cur.tactic) && cur.tactic.length === 0)) && Array.isArray(fresh.tactic) && fresh.tactic.length) {
+    cur.tactic = fresh.tactic;
+    touched = true;
+  }
   fillIfEmpty("description_full", fresh.description_full);
   fillIfEmpty("platforms", fresh.platforms);
   fillIfEmpty("data_sources", fresh.data_sources);
@@ -359,10 +386,20 @@ async function refreshAttack({ dry = false, cap = Infinity } = {}) {
   console.log("[refresh-upstream:attack] fetching MITRE ATT&CK STIX...");
   const body = await fetchUrl(ATTACK_SRC);
   const stix = JSON.parse(body);
-  const techs = (stix.objects || []).filter(
+  // For NEW adds: live techniques only (skip revoked / deprecated).
+  // For BACKFILL on existing rows: include revoked too — an operator-
+  // curated row that references a now-revoked MITRE ID may still want
+  // the context fields (name / description / platforms) from the
+  // pre-revocation STIX record. Same logic as the RFC obsoleted-but-
+  // backfillable two-pass design.
+  const liveTechs = (stix.objects || []).filter(
     (o) => o.type === "attack-pattern" && !o.revoked && !o.x_mitre_deprecated
   );
-  console.log(`[refresh-upstream:attack] STIX live techniques: ${techs.length}`);
+  const backfillTechs = (stix.objects || []).filter(
+    (o) => o.type === "attack-pattern"
+  );
+  console.log(`[refresh-upstream:attack] STIX live techniques: ${liveTechs.length} (+ ${backfillTechs.length - liveTechs.length} revoked/deprecated available for backfill on existing rows)`);
+  const techs = liveTechs;
   const local = loadCatalog("attack-techniques.json");
   const existing = new Set(Object.keys(local).filter((k) => k !== "_meta"));
   techs.sort((a, b) => {
@@ -375,28 +412,106 @@ async function refreshAttack({ dry = false, cap = Infinity } = {}) {
   });
   let added = 0;
   let backfilled = 0;
+  // First pass: backfill existing rows against the FULL technique set
+  // (including revoked) so operator-curated rows still get context.
+  for (const t of backfillTechs) {
+    const extRef = (t.external_references || []).find((r) => r.source_name === "mitre-attack");
+    if (!extRef || !extRef.external_id) continue;
+    const id = extRef.external_id;
+    if (!existing.has(id)) continue;
+    const fresh = attackEntryFromStix(t, extRef);
+    const cur = local[id];
+    if (backfillAttack(cur, fresh)) {
+      cur.last_verified = TODAY;
+      backfilled++;
+    }
+  }
+  // Second pass: add new entries from live techniques only.
   for (const t of techs) {
     const extRef = (t.external_references || []).find((r) => r.source_name === "mitre-attack");
     if (!extRef || !extRef.external_id) continue;
     const id = extRef.external_id;
+    if (existing.has(id)) continue;
+    if (added >= cap) continue;
+    local[id] = attackEntryFromStix(t, extRef);
+    existing.add(id);
+    added++;
+  }
+  if (dry) { console.log(`[refresh-upstream:attack] DRY-RUN: +${added} new, ${backfilled} context backfills`); return { added, backfilled }; }
+  if (local._meta) { local._meta.last_updated = TODAY; local._meta.last_threat_review = TODAY; }
+  writeCatalog("attack-techniques.json", local);
+  console.log(`[ok] attack-techniques.json: +${added} entries, ${backfilled} context backfills (now ${existing.size} total)`);
+  return { added, backfilled };
+}
+
+// ---------------- ICS-ATT&CK ----------------
+
+const ICS_ATTACK_SRC = "https://raw.githubusercontent.com/mitre/cti/master/ics-attack/ics-attack.json";
+const ICS_TACTIC_NAME = {
+  "initial-access": "Initial Access (ICS)",
+  "execution": "Execution (ICS)",
+  "persistence": "Persistence (ICS)",
+  "privilege-escalation": "Privilege Escalation (ICS)",
+  "evasion": "Evasion (ICS)",
+  "discovery": "Discovery (ICS)",
+  "lateral-movement": "Lateral Movement (ICS)",
+  "collection": "Collection (ICS)",
+  "command-and-control": "Command and Control (ICS)",
+  "inhibit-response-function": "Inhibit Response Function",
+  "impair-process-control": "Impair Process Control",
+  "impact": "Impact (ICS)"
+};
+
+async function refreshIcsAttack({ dry = false, cap = Infinity } = {}) {
+  console.log("[refresh-upstream:ics-attack] fetching MITRE ICS-attack STIX...");
+  const body = await fetchUrl(ICS_ATTACK_SRC);
+  const stix = JSON.parse(body);
+  const techs = (stix.objects || []).filter(
+    (o) => o.type === "attack-pattern" && !o.revoked && !o.x_mitre_deprecated
+  );
+  console.log(`[refresh-upstream:ics-attack] STIX live ICS techniques: ${techs.length}`);
+  const local = loadCatalog("attack-techniques.json");
+  const existing = new Set(Object.keys(local).filter((k) => k !== "_meta"));
+  let added = 0, backfilled = 0;
+  for (const t of techs) {
+    const extRef = (t.external_references || []).find((r) => r.source_name === "mitre-ics-attack" || r.source_name === "mitre-attack");
+    if (!extRef || !extRef.external_id) continue;
+    const id = extRef.external_id;
+    const tactics = (t.kill_chain_phases || [])
+      .filter((p) => (p.kill_chain_name || "").includes("ics"))
+      .map((p) => ICS_TACTIC_NAME[p.phase_name] || `${p.phase_name} (ICS)`);
+    const fullDesc = String(t.description || "").replace(/\s+/g, " ").trim();
+    let shortDesc = fullDesc.split(/\.\s/)[0];
+    if (shortDesc.length > 500) shortDesc = shortDesc.slice(0, 497) + "...";
+    if (shortDesc && !shortDesc.endsWith(".")) shortDesc += ".";
+    const fresh = {
+      id, name: t.name, version: "ics-attack-v15",
+      tactic: tactics,
+      description: shortDesc,
+      description_full: fullDesc,
+      platforms: Array.isArray(t.x_mitre_platforms) ? t.x_mitre_platforms : [],
+      detection: (t.x_mitre_detection || "").replace(/\s+/g, " ").trim() || null,
+      reference_url: extRef.url || `https://attack.mitre.org/techniques/${id}/`,
+      stix_id: t.id || null,
+      last_verified: TODAY,
+      _auto_imported: true,
+      _intake_method: "mitre-ics-attack-stix",
+      _matrix: "ics-attack"
+    };
     if (existing.has(id)) {
-      const fresh = attackEntryFromStix(t, extRef);
       const cur = local[id];
-      if (backfillAttack(cur, fresh)) {
-        cur.last_verified = TODAY;
-        backfilled++;
-      }
+      if (backfillAttack(cur, fresh)) { cur.last_verified = TODAY; backfilled++; }
       continue;
     }
     if (added >= cap) continue;
-    local[id] = attackEntryFromStix(t, extRef);
+    local[id] = fresh;
     existing.add(id);
     added++;
   }
-  if (dry) { console.log(`[refresh-upstream:attack] DRY-RUN: +${added} new, ${backfilled} context backfills`); return { added, backfilled }; }
+  if (dry) { console.log(`[refresh-upstream:ics-attack] DRY-RUN: +${added} new, ${backfilled} backfills`); return { added, backfilled }; }
   if (local._meta) { local._meta.last_updated = TODAY; local._meta.last_threat_review = TODAY; }
   writeCatalog("attack-techniques.json", local);
-  console.log(`[ok] attack-techniques.json: +${added} entries, ${backfilled} context backfills (now ${existing.size} total)`);
+  console.log(`[ok] attack-techniques.json: +${added} ICS entries, ${backfilled} backfills (now ${existing.size} total)`);
   return { added, backfilled };
 }
 
@@ -645,10 +760,11 @@ async function refreshD3fend({ dry = false, cap = Infinity } = {}) {
 // ---------------- CLI dispatcher ----------------
 
 const SOURCES = {
-  rfc:    { name: "ietf-rfc-index",    run: refreshRfc },
-  attack: { name: "mitre-attack-stix", run: refreshAttack },
-  atlas:  { name: "mitre-atlas-stix",  run: refreshAtlas },
-  d3fend: { name: "mitre-d3fend-owl",  run: refreshD3fend }
+  rfc:        { name: "ietf-rfc-index",       run: refreshRfc },
+  attack:     { name: "mitre-attack-stix",    run: refreshAttack },
+  "ics-attack": { name: "mitre-ics-attack-stix", run: refreshIcsAttack },
+  atlas:      { name: "mitre-atlas-stix",     run: refreshAtlas },
+  d3fend:     { name: "mitre-d3fend-owl",     run: refreshD3fend }
 };
 
 function parseArgs(argv) {
@@ -690,6 +806,7 @@ if (require.main === module) {
 module.exports = {
   refreshRfc,
   refreshAttack,
+  refreshIcsAttack,
   refreshAtlas,
   refreshD3fend,
   SOURCES,