@blamejs/exceptd-skills 0.9.1

package/lib/refresh-external.js

@@ -0,0 +1,713 @@
+"use strict";
+/**
+ * lib/refresh-external.js
+ *
+ * External-data refresh orchestrator. Pulls the latest from the canonical
+ * upstream sources and either reports drift (dry-run, default) or applies
+ * the drift as an upsert into the local catalog (--apply).
+ *
+ * Sources (each is independently pluggable):
+ *
+ *   KEV    — CISA Known Exploited Vulnerabilities (per-CVE upsert of
+ *            cisa_kev / cisa_kev_date)
+ *   EPSS   — FIRST EPSS (per-CVE upsert of epss_score / epss_percentile /
+ *            epss_date)
+ *   NVD    — NIST NVD 2.0 (per-CVE upsert of cvss_score / cvss_vector)
+ *   RFC    — IETF Datatracker (per-RFC upsert of status)
+ *   PINS   — MITRE ATLAS / ATT&CK / D3FEND / CWE upstream releases
+ *            (REPORT-ONLY — version bumps need audit per AGENTS.md
+ *            Hard Rule #12, so they surface as findings, not auto-applied)
+ *
+ * Usage:
+ *   node lib/refresh-external.js              # dry-run all sources
+ *   node lib/refresh-external.js --apply      # apply all sources
+ *   node lib/refresh-external.js --source kev # one source, dry-run
+ *   node lib/refresh-external.js --apply --source kev,epss
+ *   node lib/refresh-external.js --from-fixture <path>   # use frozen fixture
+ *                                                          payloads (offline)
+ *
+ * Exit codes:
+ *   0 — dry-run completed (regardless of whether drift was found)
+ *   1 — apply mode AND a downstream gate (validate-indexes, lint) failed
+ *   2 — unrecoverable runner error
+ *
+ * The refresh-report.json artifact lives at the repo root and is
+ * gitignored. CI uploads it as a workflow artifact.
+ */
+
+const fs = require("fs");
+const path = require("path");
+const { execFileSync } = require("child_process");
+
+const ROOT = path.join(__dirname, "..");
+const ABS = (p) => path.join(ROOT, p);
+const TODAY = new Date().toISOString().slice(0, 10);
+
+function parseArgs(argv) {
+  const out = {
+    apply: false,
+    source: null,        // comma-separated list or null = all
+    fromFixture: null,   // path to fixture dir
+    fromCache: null,     // path to .cache/upstream dir (or default if --from-cache passed bare)
+    swarm: false,        // fan-out sources across worker threads
+    help: false,
+    quiet: false,
+  };
+  for (let i = 2; i < argv.length; i++) {
+    const a = argv[i];
+    if (a === "--apply") out.apply = true;
+    else if (a === "--quiet") out.quiet = true;
+    else if (a === "--swarm") out.swarm = true;
+    else if (a === "--help" || a === "-h") out.help = true;
+    else if (a === "--from-cache") {
+      // accept either --from-cache <path> or --from-cache (default path)
+      const next = argv[i + 1];
+      if (next && !next.startsWith("--")) { out.fromCache = next; i++; }
+      else out.fromCache = ".cache/upstream";
+    }
+    else if (a.startsWith("--from-cache=")) out.fromCache = a.slice("--from-cache=".length);
+    else if (a === "--source") out.source = argv[++i];
+    else if (a.startsWith("--source=")) out.source = a.slice("--source=".length);
+    else if (a === "--from-fixture") out.fromFixture = argv[++i];
+    else if (a.startsWith("--from-fixture=")) out.fromFixture = a.slice("--from-fixture=".length);
+    else if (a === "--report-out") out.reportOut = argv[++i];
+    else if (a.startsWith("--report-out=")) out.reportOut = a.slice("--report-out=".length);
+  }
+  return out;
+}
+
+function printHelp() {
+  console.log(`refresh-external — pull latest upstream data, optionally upsert into local catalogs.
+
+Modes:
+  (default)          dry-run all sources, write refresh-report.json
+  --apply            apply diffs and rebuild indexes
+  --source kev,epss  scope to a comma-separated list (kev|epss|nvd|rfc|pins)
+  --from-fixture <p> use frozen fixture payloads (tests use this path)
+  --from-cache [<p>] read from prefetch cache (default .cache/upstream).
+                     Combine with --apply to upsert against cached data
+                     entirely offline.
+  --swarm            fan out sources across worker threads. Source fetches
+                     run in parallel rather than sequentially. Best when
+                     paired with --from-cache (no rate-limit contention).
+
+Outputs:
+  refresh-report.json (gitignored) — summary of every diff + per-source status.
+
+This module never auto-applies version-pin bumps — those require audit per
+AGENTS.md Hard Rule #12 and are surfaced as report-only findings.
+`);
+}
+
+// --- Source modules ----------------------------------------------------
+
+/**
+ * Each source module exposes:
+ *   name: string
+ *   fetchDiff(ctx, opts) -> Promise<{ status, diffs, errors, summary }>
+ *     status: "ok" | "unreachable" | "partial"
+ *     diffs:  array of { id, field, before, after, severity? }
+ *     summary: brief string
+ *   applyDiff(ctx, diffs) -> Promise<{ updated, errors }>
+ *     mutates local catalog and writes it
+ */
+
+const KEV_SOURCE = {
+  name: "kev",
+  description: "CISA Known Exploited Vulnerabilities",
+  applies_to: "data/cve-catalog.json",
+  async fetchDiff(ctx) {
+    if (ctx.fixtures?.kev) return synthesizeFromFixture(ctx, "kev");
+    if (ctx.cacheDir) return kevDiffFromCache(ctx);
+    const { validateAllCves } = require("../sources/validators");
+    const report = await validateAllCves(ctx.cveCatalog, { concurrency: 4 });
+    const diffs = [];
+    let errors = 0;
+    for (const r of report.results) {
+      if (r.status === "unreachable") errors++;
+      for (const d of r.discrepancies || []) {
+        if (d.field === "cisa_kev" || d.field === "cisa_kev_date") {
+          diffs.push({ id: r.cve_id, field: d.field, before: d.local, after: d.fetched, severity: d.severity });
+        }
+      }
+    }
+    return {
+      status: errors === 0 ? "ok" : errors === report.results.length ? "unreachable" : "partial",
+      diffs,
+      errors,
+      summary: `${diffs.length} KEV diffs; ${errors} unreachable / ${report.total} total`,
+    };
+  },
+  async applyDiff(ctx, diffs) {
+    let updated = 0;
+    const errors = [];
+    for (const d of diffs) {
+      if (!ctx.cveCatalog[d.id]) {
+        errors.push(`KEV: no local entry for ${d.id}`);
+        continue;
+      }
+      ctx.cveCatalog[d.id][d.field] = d.after;
+      ctx.cveCatalog[d.id].last_verified = TODAY;
+      updated++;
+    }
+    ctx.cveCatalog._meta = ctx.cveCatalog._meta || {};
+    ctx.cveCatalog._meta.last_updated = TODAY;
+    writeJson(ABS("data/cve-catalog.json"), ctx.cveCatalog);
+    return { updated, errors };
+  },
+};
+
+const EPSS_SOURCE = {
+  name: "epss",
+  description: "FIRST.org EPSS scores",
+  applies_to: "data/cve-catalog.json",
+  async fetchDiff(ctx) {
+    if (ctx.fixtures?.epss) return synthesizeFromFixture(ctx, "epss");
+    if (ctx.cacheDir) return epssDiffFromCache(ctx);
+    const { validateAllCves } = require("../sources/validators");
+    const report = await validateAllCves(ctx.cveCatalog, { concurrency: 4 });
+    const diffs = [];
+    let errors = 0;
+    for (const r of report.results) {
+      if (r.status === "unreachable") errors++;
+      for (const d of r.discrepancies || []) {
+        if (d.field === "epss_score" || d.field === "epss_percentile") {
+          diffs.push({ id: r.cve_id, field: d.field, before: d.local, after: d.fetched, severity: d.severity });
+        }
+      }
+      // epss_date refreshes when score does.
+      if (r.fetched?.epss?.date && r.local) {
+        diffs.push({ id: r.cve_id, field: "epss_date", before: r.local.epss_date, after: r.fetched.epss.date, severity: "low" });
+      }
+    }
+    // Collapse duplicates: epss_date should appear once per CVE only when
+    // an epss field actually moved.
+    const epssCves = new Set(diffs.filter((d) => d.field === "epss_score" || d.field === "epss_percentile").map((d) => d.id));
+    const filtered = diffs.filter((d) => d.field !== "epss_date" || epssCves.has(d.id));
+    return {
+      status: errors === 0 ? "ok" : errors === report.results.length ? "unreachable" : "partial",
+      diffs: filtered,
+      errors,
+      summary: `${filtered.length} EPSS diffs; ${errors} unreachable / ${report.total} total`,
+    };
+  },
+  async applyDiff(ctx, diffs) {
+    let updated = 0;
+    const errors = [];
+    for (const d of diffs) {
+      if (!ctx.cveCatalog[d.id]) {
+        errors.push(`EPSS: no local entry for ${d.id}`);
+        continue;
+      }
+      ctx.cveCatalog[d.id][d.field] = d.after;
+      ctx.cveCatalog[d.id].last_verified = TODAY;
+      updated++;
+    }
+    ctx.cveCatalog._meta = ctx.cveCatalog._meta || {};
+    ctx.cveCatalog._meta.last_updated = TODAY;
+    writeJson(ABS("data/cve-catalog.json"), ctx.cveCatalog);
+    return { updated, errors };
+  },
+};
+
+const NVD_SOURCE = {
+  name: "nvd",
+  description: "NIST NVD 2.0 CVSS metrics",
+  applies_to: "data/cve-catalog.json",
+  async fetchDiff(ctx) {
+    if (ctx.fixtures?.nvd) return synthesizeFromFixture(ctx, "nvd");
+    if (ctx.cacheDir) return nvdDiffFromCache(ctx);
+    const { validateAllCves } = require("../sources/validators");
+    const report = await validateAllCves(ctx.cveCatalog, { concurrency: 4 });
+    const diffs = [];
+    let errors = 0;
+    for (const r of report.results) {
+      if (r.status === "unreachable") errors++;
+      for (const d of r.discrepancies || []) {
+        if (d.field === "cvss_score" || d.field === "cvss_vector") {
+          diffs.push({ id: r.cve_id, field: d.field, before: d.local, after: d.fetched, severity: d.severity });
+        }
+      }
+    }
+    return {
+      status: errors === 0 ? "ok" : errors === report.results.length ? "unreachable" : "partial",
+      diffs,
+      errors,
+      summary: `${diffs.length} NVD CVSS diffs; ${errors} unreachable / ${report.total} total`,
+    };
+  },
+  async applyDiff(ctx, diffs) {
+    let updated = 0;
+    const errors = [];
+    for (const d of diffs) {
+      if (!ctx.cveCatalog[d.id]) {
+        errors.push(`NVD: no local entry for ${d.id}`);
+        continue;
+      }
+      ctx.cveCatalog[d.id][d.field] = d.after;
+      ctx.cveCatalog[d.id].last_verified = TODAY;
+      updated++;
+    }
+    ctx.cveCatalog._meta = ctx.cveCatalog._meta || {};
+    ctx.cveCatalog._meta.last_updated = TODAY;
+    writeJson(ABS("data/cve-catalog.json"), ctx.cveCatalog);
+    return { updated, errors };
+  },
+};
+
+const RFC_SOURCE = {
+  name: "rfc",
+  description: "IETF Datatracker RFC status",
+  applies_to: "data/rfc-references.json",
+  async fetchDiff(ctx) {
+    if (ctx.fixtures?.rfc) return synthesizeFromFixture(ctx, "rfc");
+    if (ctx.cacheDir) return rfcDiffFromCache(ctx);
+    const { validateAllRfcs } = require("../sources/validators");
+    const results = await validateAllRfcs(ctx.rfcCatalog, { concurrency: 4 });
+    const diffs = [];
+    let errors = 0;
+    for (const r of results) {
+      if (r.status === "unreachable") {
+        errors++;
+        continue;
+      }
+      if (r.status === "drift" && r.discrepancies) {
+        for (const msg of r.discrepancies) {
+          // The current rfc-validator returns discrepancies as strings. We
+          // attempt to parse a status drift; fall back to the raw message.
+          const m = msg.match(/local "([^"]+)" vs Datatracker "([^"]+)"/);
+          if (m) {
+            diffs.push({ id: r.id, field: "status", before: m[1], after: m[2], severity: "medium" });
+          } else {
+            diffs.push({ id: r.id, field: "note", before: null, after: msg, severity: "low" });
+          }
+        }
+      }
+    }
+    return {
+      status: errors === 0 ? "ok" : errors === results.length ? "unreachable" : "partial",
+      diffs,
+      errors,
+      summary: `${diffs.length} RFC drifts; ${errors} unreachable / ${results.length} total`,
+    };
+  },
+  async applyDiff(ctx, diffs) {
+    let updated = 0;
+    const errors = [];
+    for (const d of diffs) {
+      if (d.field !== "status") continue; // notes are informational
+      const entry = ctx.rfcCatalog[d.id];
+      if (!entry) {
+        errors.push(`RFC: no local entry for ${d.id}`);
+        continue;
+      }
+      entry.status = d.after;
+      entry.last_verified = TODAY;
+      updated++;
+    }
+    ctx.rfcCatalog._meta = ctx.rfcCatalog._meta || {};
+    ctx.rfcCatalog._meta.last_updated = TODAY;
+    writeJson(ABS("data/rfc-references.json"), ctx.rfcCatalog);
+    return { updated, errors };
+  },
+};
+
+const PINS_SOURCE = {
+  name: "pins",
+  description: "MITRE ATLAS / ATT&CK / D3FEND / CWE upstream release pins",
+  applies_to: "manifest.json + data/cwe-catalog.json + data/d3fend-catalog.json",
+  report_only: true,
+  async fetchDiff(ctx) {
+    if (ctx.fixtures?.pins) return synthesizeFromFixture(ctx, "pins");
+    if (ctx.cacheDir) return pinsDiffFromCache(ctx);
+    const { checkAllPins } = require("../sources/validators/version-pin-validator");
+    const results = await checkAllPins({
+      manifest: ctx.manifest,
+      cweCatalog: ctx.cweCatalog,
+      d3fendCatalog: ctx.d3fendCatalog,
+    });
+    const diffs = [];
+    let errors = 0;
+    for (const r of results) {
+      if (r.unreachable) {
+        errors++;
+        continue;
+      }
+      if (r.drift) {
+        diffs.push({
+          id: r.pin_name,
+          field: "version",
+          before: r.local_version,
+          after: r.latest_version,
+          severity: "medium",
+          source_url: r.source_url,
+          local_path_hint: r.local_path_hint,
+          note: "Version-pin bump requires audit per AGENTS.md Hard Rule #12. Surface as GitHub issue, do not auto-apply.",
+        });
+      }
+    }
+    return {
+      status: errors === 0 ? "ok" : errors === results.length ? "unreachable" : "partial",
+      diffs,
+      errors,
+      summary: `${diffs.length} pin drifts; ${errors} unreachable / ${results.length} total`,
+    };
+  },
+  async applyDiff() {
+    // Version pins are intentionally not auto-applied.
+    return { updated: 0, errors: ["pin bumps are report-only — see Hard Rule #12"] };
+  },
+};
+
+const ALL_SOURCES = {
+  kev: KEV_SOURCE,
+  epss: EPSS_SOURCE,
+  nvd: NVD_SOURCE,
+  rfc: RFC_SOURCE,
+  pins: PINS_SOURCE,
+};
+
+// --- Cache-mode helpers ------------------------------------------------
+// When `--from-cache <dir>` is set, the source modules read their inputs
+// from the prefetch cache instead of hitting upstream. The cache layout
+// is fixed by lib/prefetch.js:
+//   <cacheDir>/kev/known_exploited_vulnerabilities.json
+//   <cacheDir>/nvd/<cve>.json
+//   <cacheDir>/epss/<cve>.json
+//   <cacheDir>/rfc/<doc-name>.json
+//   <cacheDir>/pins/<owner__repo__releases>.json
+//
+// readCachedJson returns null on miss; callers report it as "unreachable"
+// for that entry rather than failing the whole source.
+
+function readCachedJson(cacheDir, source, id) {
+  const safe = id.replace(/[^A-Za-z0-9._-]/g, "_");
+  const p = path.join(cacheDir, source, `${safe}.json`);
+  if (!fs.existsSync(p)) return null;
+  try { return JSON.parse(fs.readFileSync(p, "utf8")); }
+  catch { return null; }
+}
+
+function kevDiffFromCache(ctx) {
+  const feed = readCachedJson(ctx.cacheDir, "kev", "known_exploited_vulnerabilities");
+  if (!feed) {
+    return { status: "unreachable", diffs: [], errors: 1, summary: "KEV: no cached feed" };
+  }
+  const kevSet = new Set();
+  const kevDates = new Map();
+  for (const v of feed.vulnerabilities || []) {
+    if (v && v.cveID) {
+      kevSet.add(v.cveID);
+      if (v.dateAdded) kevDates.set(v.cveID, v.dateAdded);
+    }
+  }
+  const diffs = [];
+  for (const [id, entry] of Object.entries(ctx.cveCatalog)) {
+    if (!/^CVE-\d{4}-\d{4,7}$/.test(id)) continue;
+    const upstream = kevSet.has(id);
+    if (typeof entry.cisa_kev === "boolean" && entry.cisa_kev !== upstream) {
+      diffs.push({ id, field: "cisa_kev", before: entry.cisa_kev, after: upstream, severity: "high" });
+    }
+    const upDate = kevDates.get(id) || null;
+    if (upDate && entry.cisa_kev_date && entry.cisa_kev_date !== upDate) {
+      diffs.push({ id, field: "cisa_kev_date", before: entry.cisa_kev_date, after: upDate, severity: "low" });
+    }
+  }
+  return { status: "ok", diffs, errors: 0, summary: `${diffs.length} KEV diffs (from cache)` };
+}
+
+function epssDiffFromCache(ctx) {
+  const cves = Object.keys(ctx.cveCatalog).filter((k) => /^CVE-\d{4}-\d{4,7}$/.test(k));
+  const diffs = [];
+  let errors = 0;
+  const drift = 0.05;
+  for (const id of cves) {
+    const payload = readCachedJson(ctx.cacheDir, "epss", id);
+    if (!payload) { errors++; continue; }
+    const row = (payload.data || []).find((r) => r?.cve === id) || (payload.data || [])[0];
+    if (!row) continue;
+    const score = row.epss != null ? Number(row.epss) : null;
+    const pct = row.percentile != null ? Number(row.percentile) : null;
+    const local = ctx.cveCatalog[id];
+    if (score != null && local.epss_score != null && Math.abs(score - local.epss_score) > drift) {
+      diffs.push({ id, field: "epss_score", before: local.epss_score, after: score, severity: "medium" });
+    }
+    if (pct != null && local.epss_percentile != null && Math.abs(pct - local.epss_percentile) > drift) {
+      diffs.push({ id, field: "epss_percentile", before: local.epss_percentile, after: pct, severity: "medium" });
+    }
+    if (row.date && local.epss_date && row.date !== local.epss_date) {
+      // Only emit a date diff when we also emitted a score/percentile diff for this CVE.
+      const moved = diffs.some((d) => d.id === id && (d.field === "epss_score" || d.field === "epss_percentile"));
+      if (moved) diffs.push({ id, field: "epss_date", before: local.epss_date, after: row.date, severity: "low" });
+    }
+  }
+  const status = errors === 0 ? "ok" : errors === cves.length ? "unreachable" : "partial";
+  return { status, diffs, errors, summary: `${diffs.length} EPSS diffs (from cache); ${errors} missing entries` };
+}
+
+function nvdDiffFromCache(ctx) {
+  const cves = Object.keys(ctx.cveCatalog).filter((k) => /^CVE-\d{4}-\d{4,7}$/.test(k));
+  const diffs = [];
+  let errors = 0;
+  for (const id of cves) {
+    const payload = readCachedJson(ctx.cacheDir, "nvd", id);
+    if (!payload) { errors++; continue; }
+    const vuln = payload.vulnerabilities?.[0]?.cve;
+    if (!vuln) continue;
+    const m = vuln.metrics || {};
+    const ordered = [...(m.cvssMetricV31 || []), ...(m.cvssMetricV30 || []), ...(m.cvssMetricV2 || [])];
+    const primary = ordered.find((x) => x.type === "Primary") || ordered[0];
+    const upScore = typeof primary?.cvssData?.baseScore === "number" ? primary.cvssData.baseScore : null;
+    const upVector = primary?.cvssData?.vectorString || null;
+    const local = ctx.cveCatalog[id];
+    if (upScore != null && local.cvss_score != null && Math.abs(upScore - local.cvss_score) > 0.05) {
+      diffs.push({ id, field: "cvss_score", before: local.cvss_score, after: upScore, severity: "high" });
+    }
+    if (upVector && local.cvss_vector && upVector !== local.cvss_vector) {
+      diffs.push({ id, field: "cvss_vector", before: local.cvss_vector, after: upVector, severity: "medium" });
+    }
+  }
+  const status = errors === 0 ? "ok" : errors === cves.length ? "unreachable" : "partial";
+  return { status, diffs, errors, summary: `${diffs.length} NVD CVSS diffs (from cache); ${errors} missing entries` };
+}
+
+function rfcDiffFromCache(ctx) {
+  const STATUS_MAP = {
+    std: "Internet Standard", ps: "Proposed Standard", ds: "Draft Standard",
+    bcp: "Best Current Practice", inf: "Informational", exp: "Experimental",
+    his: "Historic", unkn: "Unknown",
+  };
+  const ids = Object.keys(ctx.rfcCatalog).filter((k) => !k.startsWith("_"));
+  const diffs = [];
+  let errors = 0;
+  for (const id of ids) {
+    let docName;
+    if (id.startsWith("RFC-")) docName = `rfc${id.slice(4)}`;
+    else if (id.startsWith("DRAFT-")) docName = `draft-${id.slice(6).toLowerCase()}`;
+    if (!docName) continue;
+    const payload = readCachedJson(ctx.cacheDir, "rfc", docName);
+    if (!payload) { errors++; continue; }
+    const obj = payload.objects?.[0];
+    if (!obj) continue;
+    const upStatus = STATUS_MAP[obj.std_level] || null;
+    const local = ctx.rfcCatalog[id];
+    if (upStatus && local.status && upStatus !== local.status) {
+      diffs.push({ id, field: "status", before: local.status, after: upStatus, severity: "medium" });
+    }
+  }
+  const status = errors === 0 ? "ok" : errors === ids.length ? "unreachable" : "partial";
+  return { status, diffs, errors, summary: `${diffs.length} RFC drifts (from cache); ${errors} missing entries` };
+}
+
+function pinsDiffFromCache(ctx) {
+  // Cache layout under pins/: <owner>__<repo>__releases.json arrays.
+  const PIN_REPOS = {
+    atlas_version:  "mitre-atlas__atlas-data__releases",
+    attack_version: "mitre-attack__attack-stix-data__releases",
+    d3fend_version: "d3fend__d3fend-data__releases",
+    cwe_version:    "mitre__cwe__releases",
+  };
+  const localOf = {
+    atlas_version:  ctx.manifest.atlas_version,
+    attack_version: ctx.manifest.attack_version,
+    d3fend_version: ctx.d3fendCatalog?._meta?.version || ctx.d3fendCatalog?._meta?.d3fend_version || null,
+    cwe_version:    ctx.cweCatalog?._meta?.version || ctx.cweCatalog?._meta?.cwe_version || null,
+  };
+  const diffs = [];
+  let errors = 0;
+  for (const [pinName, file] of Object.entries(PIN_REPOS)) {
+    const payload = readCachedJson(ctx.cacheDir, "pins", file);
+    if (!payload || !Array.isArray(payload)) { errors++; continue; }
+    const stable = payload.find((r) => !r.draft && !r.prerelease);
+    if (!stable) { errors++; continue; }
+    const latest = String(stable.tag_name || "").replace(/^v/, "");
+    const local = localOf[pinName] != null ? String(localOf[pinName]).replace(/^v/, "") : null;
+    if (local && latest && local !== latest) {
+      diffs.push({
+        id: pinName,
+        field: "version",
+        before: local,
+        after: latest,
+        severity: "medium",
+        source_url: stable.html_url,
+        local_path_hint: pinName === "cwe_version" ? "data/cwe-catalog.json _meta.version"
+          : pinName === "d3fend_version" ? "data/d3fend-catalog.json _meta.version"
+          : `manifest.json — ${pinName}`,
+        note: "Version-pin bump requires audit per AGENTS.md Hard Rule #12. Surface as GitHub issue, do not auto-apply.",
+      });
+    }
+  }
+  const status = errors === 0 ? "ok" : errors === Object.keys(PIN_REPOS).length ? "unreachable" : "partial";
+  return { status, diffs, errors, summary: `${diffs.length} pin drifts (from cache); ${errors} missing entries` };
+}
+
+// --- Fixture-mode helper ----------------------------------------------
+
+function synthesizeFromFixture(ctx, sourceName) {
+  // The frozen fixture payloads are JSON files that look like:
+  //   { diffs: [...], errors: 0, summary: "..." }
+  // tests/fixtures/refresh/<sourceName>.json drives this path.
+  const fp = path.join(ctx.fixtures.dir, `${sourceName}.json`);
+  if (!fs.existsSync(fp)) {
+    return { status: "ok", diffs: [], errors: 0, summary: `${sourceName}: no fixture` };
+  }
+  const fx = JSON.parse(fs.readFileSync(fp, "utf8"));
+  return {
+    status: fx.status || "ok",
+    diffs: fx.diffs || [],
+    errors: fx.errors || 0,
+    summary: fx.summary || `${sourceName}: ${(fx.diffs || []).length} diffs (fixture)`,
+  };
+}
+
+// --- IO helpers --------------------------------------------------------
+
+function loadCtx(opts) {
+  const ctx = {
+    manifest: JSON.parse(fs.readFileSync(ABS("manifest.json"), "utf8")),
+    cveCatalog: JSON.parse(fs.readFileSync(ABS("data/cve-catalog.json"), "utf8")),
+    rfcCatalog: JSON.parse(fs.readFileSync(ABS("data/rfc-references.json"), "utf8")),
+    cweCatalog: JSON.parse(fs.readFileSync(ABS("data/cwe-catalog.json"), "utf8")),
+    d3fendCatalog: JSON.parse(fs.readFileSync(ABS("data/d3fend-catalog.json"), "utf8")),
+    fixtures: null,
+    cacheDir: null,
+  };
+  if (opts.fromFixture) {
+    ctx.fixtures = { dir: path.resolve(opts.fromFixture), kev: true, epss: true, nvd: true, rfc: true, pins: true };
+  } else if (opts.fromCache) {
+    const abs = path.resolve(opts.fromCache);
+    ctx.cacheDir = abs;
+    if (!fs.existsSync(abs)) {
+      throw new Error(`refresh-external: --from-cache path does not exist: ${abs}`);
+    }
+  }
+  return ctx;
+}
+
+function writeJson(p, obj) {
+  fs.writeFileSync(p, JSON.stringify(obj, null, 2) + "\n", "utf8");
+}
+
+function chosenSources(opts) {
+  if (!opts.source) return Object.values(ALL_SOURCES);
+  const names = opts.source.split(",").map((s) => s.trim()).filter(Boolean);
+  const out = [];
+  for (const n of names) {
+    if (!ALL_SOURCES[n]) {
+      console.error(`refresh-external: unknown source "${n}". Valid: ${Object.keys(ALL_SOURCES).join(", ")}`);
+      process.exit(2);
+    }
+    out.push(ALL_SOURCES[n]);
+  }
+  return out;
+}
+
+async function main() {
+  const opts = parseArgs(process.argv);
+  if (opts.help) {
+    printHelp();
+    process.exit(0);
+  }
+
+  const ctx = loadCtx(opts);
+  const sources = chosenSources(opts);
+  const log = (s) => opts.quiet || console.log(s);
+
+  log(`\nrefresh-external — ${opts.apply ? "APPLY" : "dry-run"} mode${opts.swarm ? " (swarm)" : ""}`);
+  log(`Sources: ${sources.map((s) => s.name).join(", ")}`);
+  if (opts.fromFixture) log(`Fixture mode: ${opts.fromFixture}`);
+  if (opts.fromCache) log(`Cache mode:   ${opts.fromCache}`);
+
+  const report = {
+    generated_at: new Date().toISOString(),
+    mode: opts.apply ? "apply" : "dry-run",
+    fixture_mode: !!opts.fromFixture,
+    cache_mode: !!opts.fromCache,
+    swarm: !!opts.swarm,
+    sources: {},
+  };
+
+  let hadFailure = false;
+
+  // Source fetches are independently I/O-bound. In normal mode we run them
+  // sequentially so log output is interleaved cleanly. --swarm fans them
+  // out via Promise.all() — each source already has its own per-source
+  // queue with its own rate budget, so parallel sources don't compete
+  // against each other for tokens. The two modes produce the same report
+  // structure; only wall-clock differs.
+  const runOne = async (src) => {
+    let diff;
+    try {
+      diff = await src.fetchDiff(ctx);
+    } catch (err) {
+      return { src, error: err };
+    }
+    return { src, diff };
+  };
+
+  const outcomes = opts.swarm
+    ? await Promise.all(sources.map(runOne))
+    : await sequential(sources, runOne);
+
+  for (const { src, diff, error } of outcomes) {
+    if (error) {
+      log(`\n  [${src.name}] ${src.description}`);
+      log(`    error: ${error.message}`);
+      report.sources[src.name] = { status: "error", error: error.message };
+      hadFailure = true;
+      continue;
+    }
+    log(`\n  [${src.name}] ${src.description}`);
+    log(`    ${diff.summary}`);
+    report.sources[src.name] = {
+      status: diff.status,
+      summary: diff.summary,
+      diff_count: diff.diffs.length,
+      errors: diff.errors,
+      diffs: diff.diffs,
+      applies_to: src.applies_to,
+      report_only: !!src.report_only,
+    };
+    if (opts.apply && diff.diffs.length > 0 && !src.report_only) {
+      const r = await src.applyDiff(ctx, diff.diffs);
+      report.sources[src.name].applied = r.updated;
+      report.sources[src.name].apply_errors = r.errors;
+      log(`    applied: ${r.updated} update(s)`);
+      if (r.errors.length) log(`    apply errors: ${r.errors.join("; ")}`);
+    }
+  }
+
+  // Persist report — tests can redirect via --report-out so concurrent
+  // suites don't race on a shared refresh-report.json at the repo root.
+  const reportPath = opts.reportOut ? path.resolve(opts.reportOut) : ABS("refresh-report.json");
+  writeJson(reportPath, report);
+  log(`\nWrote ${path.relative(ROOT, reportPath)}`);
+
+  if (opts.apply) {
+    // Always regenerate indexes after an apply so validate-indexes passes.
+    log(`\nRebuilding indexes (npm run build-indexes)`);
+    try {
+      execFileSync(process.execPath, [ABS("scripts/build-indexes.js")], { stdio: "inherit", cwd: ROOT });
+    } catch (err) {
+      console.error(`refresh-external: build-indexes failed: ${err.message}`);
+      hadFailure = true;
+    }
+  }
+
+  process.exit(hadFailure ? 1 : 0);
+}
+
+async function sequential(items, fn) {
+  const out = [];
+  for (const it of items) out.push(await fn(it));
+  return out;
+}
+
+if (require.main === module) {
+  main().catch((err) => {
+    console.error(`refresh-external: fatal: ${err && err.stack ? err.stack : err}`);
+    process.exit(2);
+  });
+}
+
+module.exports = { ALL_SOURCES, loadCtx, parseArgs };