@blamejs/exceptd-skills 0.18.21 → 0.18.23

package/lib/prefetch.js

@@ -46,6 +46,23 @@ const DEFAULT_CACHE = path.join(ROOT, ".cache", "upstream");
 const REQUEST_TIMEOUT_MS = 10_000;
 const USER_AGENT = "exceptd-security/prefetch (+https://exceptd.com)";
 
+// CVE ids the CISA KEV feed lists that are NOT yet in the local catalog.
+// Pure — no fs/network. `kevFeed` is the parsed known_exploited_vulnerabilities
+// JSON (or null/undefined if unavailable); `cveCatalog` is data/cve-catalog.json
+// (or a synthetic subset in tests). Consumed by nvd/epss expand() below so a
+// CVE CISA newly added to KEV gets its NVD/EPSS sidecar prefetched in the SAME
+// run auto-discovery will later read via lib/auto-discovery.js:readCachedJson
+// — without this, buildKevDraftEntry's nvd/epss payloads stayed null forever
+// for anything auto-discovery itself found, because nothing had ever asked
+// prefetch to warm a sidecar for an id absent from the local catalog.
+function newKevIds(kevFeed, cveCatalog) {
+  const have = new Set(Object.keys(cveCatalog || {}));
+  const vulns = Array.isArray(kevFeed && kevFeed.vulnerabilities) ? kevFeed.vulnerabilities : [];
+  return vulns
+    .map((v) => v.cveID)
+    .filter((id) => /^CVE-\d{4}-\d{4,7}$/.test(id) && !have.has(id));
+}
+
 const SOURCES = {
   kev: {
     description: "CISA Known Exploited Vulnerabilities (single feed)",
@@ -58,17 +75,27 @@ const SOURCES = {
     rate: { tokens: 5, windowMs: 30_000 },     // anon budget; NVD_API_KEY lifts to 50
     rate_with_key: { tokens: 50, windowMs: 30_000 },
     concurrency: 4,
-    expand: (ctx) => Object.keys(ctx.cveCatalog)
-      .filter((k) => /^CVE-\d{4}-\d{4,7}$/.test(k))
-      .map((id) => ({ id, url: `https://services.nvd.nist.gov/rest/json/cves/2.0?cveId=${encodeURIComponent(id)}` })),
+    // Union the local catalog with any newly-KEV-listed id (ctx.kevFeed, when
+    // the run loop resolved it — see prefetch()'s KEV pre-fetch step below).
+    // ctx.kevFeed absent/null is the documented fallback (KEV out of scope
+    // this run, or --no-network): newKevIds returns [], so this degrades to
+    // the pre-Task-11 catalog-only expansion, not a crash.
+    expand: (ctx) => {
+      const ids = new Set(Object.keys(ctx.cveCatalog).filter((k) => /^CVE-\d{4}-\d{4,7}$/.test(k)));
+      for (const id of newKevIds(ctx.kevFeed, ctx.cveCatalog)) ids.add(id);
+      return [...ids].map((id) => ({ id, url: `https://services.nvd.nist.gov/rest/json/cves/2.0?cveId=${encodeURIComponent(id)}` }));
+    },
   },
   epss: {
     description: "FIRST.org EPSS per-CVE responses",
     rate: { tokens: 30, windowMs: 60_000 },
     concurrency: 4,
-    expand: (ctx) => Object.keys(ctx.cveCatalog)
-      .filter((k) => /^CVE-\d{4}-\d{4,7}$/.test(k))
-      .map((id) => ({ id, url: `https://api.first.org/data/v1/epss?cve=${encodeURIComponent(id)}` })),
+    // Same new-KEV union as nvd above.
+    expand: (ctx) => {
+      const ids = new Set(Object.keys(ctx.cveCatalog).filter((k) => /^CVE-\d{4}-\d{4,7}$/.test(k)));
+      for (const id of newKevIds(ctx.kevFeed, ctx.cveCatalog)) ids.add(id);
+      return [...ids].map((id) => ({ id, url: `https://api.first.org/data/v1/epss?cve=${encodeURIComponent(id)}` }));
+    },
   },
   rfc: {
     description: "IETF Datatracker per-RFC/doc records",
@@ -675,8 +702,156 @@ async function prefetch(options = {}) {
   const idx = loadIndex(opts.cacheDir);
   if (!fs.existsSync(opts.cacheDir)) fs.mkdirSync(opts.cacheDir, { recursive: true });
 
+  const log = (s) => opts.quiet || console.log(s);
+  const result = { fetched: 0, skipped_fresh: 0, errors: 0, errors_transient: 0, errors_hard: 0, by_source: {} };
+  for (const s of chosen) result.by_source[s] = { fetched: 0, skipped_fresh: 0, errors: 0, errors_transient: 0, errors_hard: 0 };
+
+  // Fetch (or fresh-skip) a single plan item: on a live fetch, writes the
+  // payload + updates _index.json under lock and mirrors it into the
+  // in-memory `idx` snapshot; on a fresh-skip, just counts it. Updates
+  // `result` bookkeeping identically either way. Factored out of the main
+  // per-item loop below so the KEV pre-fetch step (Task 11, next) can reuse
+  // the exact same fetch/write/index/error-classification contract instead
+  // of duplicating it.
+  //
+  // Returns the entry's parsed JSON body when `needData` is true — a
+  // freshly-fetched item returns it for free (already in memory from the
+  // fetch); a fresh-skipped item costs one extra cache read via
+  // `readCached`, so `needData` defaults to false and the main plan loop
+  // (hundreds to ~9.7k items on the Monday RFC sweep) never pays it. Only
+  // the one-off KEV pre-fetch below opts in. Returns null on error, or when
+  // `needData` is false.
+  async function fetchEntry(item, { needData = false } = {}) {
+    if (item.fresh) {
+      result.skipped_fresh++;
+      result.by_source[item.source].skipped_fresh++;
+      if (!needData) return null;
+      const cached = readCached(opts.cacheDir, item.source, item.id, { maxAgeMs: opts.maxAgeMs });
+      return cached ? cached.data : null;
+    }
+    const headers = authHeadersForSource(item.source);
+    // NVD takes its key in a custom header.
+    const reqHeaders = item.source === "nvd" && headers.apiKey ? { apiKey: headers.apiKey } : (item.source === "pins" ? headers : {});
+    try {
+      const res = await queue.add({
+        source: item.source,
+        priority: priorityFor(item.source),
+        run: () => timedFetch(item.url, reqHeaders),
+        meta: { id: item.id },
+      });
+      const targetPath = entryPath(opts.cacheDir, item.source, item.id);
+      const dir = path.dirname(targetPath);
+      if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
+      const body = JSON.stringify(res.json, null, 2) + "\n";
+      // Stage the payload to a same-volume tmp file BEFORE attempting
+      // to acquire the index lock. If withIndexLock fails (timeout
+      // after MAX_RETRIES), the partially-completed download must be
+      // discarded — not left on disk as an orphan payload with no
+      // index entry. Air-gap operators feed off `readCached`, which
+      // consults the index; an unindexed payload silently becomes junk
+      // taking cache space. Pattern: stage → lock → rename+index →
+      // release. The rename is atomic same-volume; if it fails inside
+      // the lock we clean up the tmp file. If we never reach the rename
+      // (lock acquisition throws), the tmp file is unlinked in the
+      // catch block below.
+      const tmpPath = `${targetPath}.tmp.${process.pid}.${Math.random().toString(36).slice(2, 10)}`;
+      fs.writeFileSync(tmpPath, body);
+      const meta = {
+        fetched_at: new Date().toISOString(),
+        etag: res.etag,
+        last_modified: res.lastModified,
+        url: item.url,
+        sha256: crypto.createHash("sha256").update(JSON.stringify(res.json)).digest("hex"),
+      };
+      try {
+        // v0.12.12 C2: persist this entry's metadata to _index.json under
+        // lock immediately, merging with whatever the on-disk index has
+        // (another concurrent prefetch may have written sibling entries).
+        // Inside the lock we also rename the staged tmp → final path so
+        // a concurrent reader sees the new payload + new index entry as
+        // an atomic pair.
+        await withIndexLock(opts.cacheDir, (current) => {
+          try {
+            fs.renameSync(tmpPath, targetPath);
+          } catch (renameErr) {
+            // Surface as a failure to mutator: throwing here aborts the
+            // lock's write step. We re-throw to the outer catch which
+            // will increment errors.
+            throw renameErr;
+          }
+          current.entries[entryKey(item.source, item.id)] = meta;
+          return current;
+        });
+        // Mirror the entry into the in-memory idx snapshot so any
+        // later in-run freshness check sees this entry as fresh. The
+        // authoritative on-disk write already happened under the lock
+        // above; this is the in-memory copy only.
+        idx.entries[entryKey(item.source, item.id)] = meta;
+      } catch (lockErr) {
+        // Lock failure OR rename-inside-lock failure — unlink the staged
+        // tmp so the cache directory does not accumulate orphans.
+        try { fs.unlinkSync(tmpPath); } catch {}
+        throw lockErr;
+      }
+      result.fetched++;
+      result.by_source[item.source].fetched++;
+      log(`  [${item.source}] ${item.id} — ok`);
+      return needData ? res.json : null;
+    } catch (err) {
+      result.errors++;
+      result.by_source[item.source].errors++;
+      // Classify the post-retry error. Transient iff the job queue would
+      // have retried it (the same isRetryable classifier the queue used):
+      // HTTP 408/425/429/5xx + ETIMEDOUT/ECONNRESET et al — the upstream
+      // throttling/timing-out, not a data fault. Anything else (404/410/
+      // parse failure) is hard. Only hard errors gate the exit code; a
+      // best-effort warm tolerates transient throttling and retries it on
+      // the next run. The split is surfaced in the summary so nothing hides.
+      const transient = isRetryable(err);
+      if (transient) {
+        result.errors_transient++;
+        result.by_source[item.source].errors_transient++;
+      } else {
+        result.errors_hard++;
+        result.by_source[item.source].errors_hard++;
+      }
+      // Errors go to stderr unconditionally — they are diagnostics, not the
+      // per-entry success chatter --quiet suppresses. A CI run with --quiet
+      // still surfaces which source/id failed and whether it was transient.
+      console.error(`  [${item.source}] ${item.id} — ${transient ? "transient" : "hard"} error: ${err.message}`);
+      return null;
+    }
+  }
+
+  // Task 11: resolve the KEV feed BEFORE nvd/epss build their expansion
+  // list, so a CVE CISA added to KEV today gets an NVD/EPSS sidecar
+  // prefetched in THIS run. Reading a previously-persisted cache file would
+  // not do this reliably — the scheduled workflow's cache directory is not
+  // persisted across runs (each job warms an empty `.cache/upstream` from
+  // scratch), so by the time a later run's plan is built, "today's" KEV
+  // addition would already be a day old. Fetching (or fresh-skipping) KEV
+  // synchronously here, before the main plan is built, guarantees ctx.kevFeed
+  // reflects this run's own KEV data.
+  //
+  // Only fires when "kev" is actually in scope and network fetches are
+  // enabled. Under --no-network (dry-run/--air-gap) or a --source scope that
+  // excludes kev, ctx.kevFeed stays unset and nvd/epss's expand() falls back
+  // to catalog-only expansion via newKevIds' null-safe default — no crash,
+  // and no behavior change from before this task.
+  let kevPrefetched = false;
+  if (chosen.includes("kev") && !opts.noNetwork) {
+    const [kevEntry] = SOURCES.kev.expand();
+    if (kevEntry) {
+      const fresh = !opts.force && isFresh(idx, "kev", kevEntry.id, opts.maxAgeMs);
+      const data = await fetchEntry({ source: "kev", id: kevEntry.id, url: kevEntry.url, fresh }, { needData: true });
+      if (data) ctx.kevFeed = data;
+      kevPrefetched = true;
+    }
+  }
+
   const plan = [];
   for (const sourceName of chosen) {
+    if (sourceName === "kev" && kevPrefetched) continue; // already resolved above
     const cfg = SOURCES[sourceName];
     const entries = cfg.expand(ctx);
     for (const e of entries) {
@@ -685,14 +860,11 @@ async function prefetch(options = {}) {
     }
   }
 
-  const log = (s) => opts.quiet || console.log(s);
-  log(`\nprefetch — ${opts.noNetwork ? "DRY-RUN" : "fetching"} ${plan.length} item(s) across ${chosen.length} source(s)`);
+  const totalItems = plan.length + (kevPrefetched ? 1 : 0);
+  log(`\nprefetch — ${opts.noNetwork ? "DRY-RUN" : "fetching"} ${totalItems} item(s) across ${chosen.length} source(s)`);
   log(`Cache dir: ${path.relative(ROOT, opts.cacheDir)}`);
   log(`Max age:   ${(opts.maxAgeMs / 3_600_000).toFixed(1)}h${opts.force ? "  (forced)" : ""}`);
 
-  const result = { fetched: 0, skipped_fresh: 0, errors: 0, errors_transient: 0, errors_hard: 0, by_source: {} };
-  for (const s of chosen) result.by_source[s] = { fetched: 0, skipped_fresh: 0, errors: 0, errors_transient: 0, errors_hard: 0 };
-
   if (opts.noNetwork) {
     for (const item of plan) {
       const tag = item.fresh ? "FRESH (skip)" : "STALE (would fetch)";
@@ -709,105 +881,7 @@ async function prefetch(options = {}) {
     return result;
   }
 
-  const jobPromises = plan.map((item) => {
-    if (item.fresh) {
-      result.skipped_fresh++;
-      result.by_source[item.source].skipped_fresh++;
-      return Promise.resolve();
-    }
-    const headers = authHeadersForSource(item.source);
-    // NVD takes its key in a custom header.
-    const reqHeaders = item.source === "nvd" && headers.apiKey ? { apiKey: headers.apiKey } : (item.source === "pins" ? headers : {});
-    return queue
-      .add({
-        source: item.source,
-        priority: priorityFor(item.source),
-        run: () => timedFetch(item.url, reqHeaders),
-        meta: { id: item.id },
-      })
-      .then(async (res) => {
-        const targetPath = entryPath(opts.cacheDir, item.source, item.id);
-        const dir = path.dirname(targetPath);
-        if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
-        const body = JSON.stringify(res.json, null, 2) + "\n";
-        // Stage the payload to a same-volume tmp file BEFORE attempting
-        // to acquire the index lock. If withIndexLock fails (timeout
-        // after MAX_RETRIES), the partially-completed download must be
-        // discarded — not left on disk as an orphan payload with no
-        // index entry. Air-gap operators feed off `readCached`, which
-        // consults the index; an unindexed payload silently becomes junk
-        // taking cache space. Pattern: stage → lock → rename+index →
-        // release. The rename is atomic same-volume; if it fails inside
-        // the lock we clean up the tmp file. If we never reach the rename
-        // (lock acquisition throws), the tmp file is unlinked in the
-        // catch block below.
-        const tmpPath = `${targetPath}.tmp.${process.pid}.${Math.random().toString(36).slice(2, 10)}`;
-        fs.writeFileSync(tmpPath, body);
-        const meta = {
-          fetched_at: new Date().toISOString(),
-          etag: res.etag,
-          last_modified: res.lastModified,
-          url: item.url,
-          sha256: crypto.createHash("sha256").update(JSON.stringify(res.json)).digest("hex"),
-        };
-        try {
-          // v0.12.12 C2: persist this entry's metadata to _index.json under
-          // lock immediately, merging with whatever the on-disk index has
-          // (another concurrent prefetch may have written sibling entries).
-          // Inside the lock we also rename the staged tmp → final path so
-          // a concurrent reader sees the new payload + new index entry as
-          // an atomic pair.
-          await withIndexLock(opts.cacheDir, (current) => {
-            try {
-              fs.renameSync(tmpPath, targetPath);
-            } catch (renameErr) {
-              // Surface as a failure to mutator: throwing here aborts the
-              // lock's write step. We re-throw to the outer catch which
-              // will increment errors.
-              throw renameErr;
-            }
-            current.entries[entryKey(item.source, item.id)] = meta;
-            return current;
-          });
-          // Mirror the entry into the in-memory idx snapshot so any
-          // later in-run freshness check sees this entry as fresh. The
-          // authoritative on-disk write already happened under the lock
-          // above; this is the in-memory copy only.
-          idx.entries[entryKey(item.source, item.id)] = meta;
-        } catch (lockErr) {
-          // Lock failure OR rename-inside-lock failure — unlink the staged
-          // tmp so the cache directory does not accumulate orphans.
-          try { fs.unlinkSync(tmpPath); } catch {}
-          throw lockErr;
-        }
-        result.fetched++;
-        result.by_source[item.source].fetched++;
-        log(`  [${item.source}] ${item.id} — ok`);
-      })
-      .catch((err) => {
-        result.errors++;
-        result.by_source[item.source].errors++;
-        // Classify the post-retry error. Transient iff the job queue would
-        // have retried it (the same isRetryable classifier the queue used):
-        // HTTP 408/425/429/5xx + ETIMEDOUT/ECONNRESET et al — the upstream
-        // throttling/timing-out, not a data fault. Anything else (404/410/
-        // parse failure) is hard. Only hard errors gate the exit code; a
-        // best-effort warm tolerates transient throttling and retries it on
-        // the next run. The split is surfaced in the summary so nothing hides.
-        const transient = isRetryable(err);
-        if (transient) {
-          result.errors_transient++;
-          result.by_source[item.source].errors_transient++;
-        } else {
-          result.errors_hard++;
-          result.by_source[item.source].errors_hard++;
-        }
-        // Errors go to stderr unconditionally — they are diagnostics, not the
-        // per-entry success chatter --quiet suppresses. A CI run with --quiet
-        // still surfaces which source/id failed and whether it was transient.
-        console.error(`  [${item.source}] ${item.id} — ${transient ? "transient" : "hard"} error: ${err.message}`);
-      });
-  });
+  const jobPromises = plan.map((item) => fetchEntry(item));
 
   await Promise.all(jobPromises);
   await queue.drain();
@@ -976,6 +1050,11 @@ module.exports = {
   formatSummary,
   SOURCES,
   DEFAULT_CACHE,
+  // Task 11: pure helper (KEV feed minus local catalog) consumed by
+  // nvd/epss expand() above. Exported for direct unit testing and so other
+  // callers (e.g. a future report-only "what's new in KEV" surface) don't
+  // have to re-derive the same set.
+  newKevIds,
   // Ed25519 _index.json signing + verification. Exported so
   // lib/refresh-external.js (which consumes --from-cache) can verify the
   // sidecar before trusting any cached entry, and so test harnesses can

package/lib/scoring.js

@@ -440,6 +440,29 @@ function deriveRwepFromFactors(factors) {
   return Math.max(0, Math.min(100, sum));
 }
 
+// Post-weight (Shape-B) factor object required by cve-catalog.schema.json.
+// Canonical home for the math auto-discovery.js/cve-enrich.js both consume,
+// so Σ Object.values(...) === scoreCustom(inputs) (pre-clamp) by construction.
+function postWeightFactors(inputs) {
+  const i = inputs || {};
+  const aeMultiplier = activeExploitationMultiplier(i.active_exploitation);
+  const reboot = (i.reboot_required === true) || (i.patch_required_reboot === true);
+  let blastRaw = 0;
+  if (typeof i.blast_radius === 'number' && Number.isFinite(i.blast_radius)) blastRaw = i.blast_radius;
+  else if (typeof i.blast_radius === 'string' && i.blast_radius.trim() !== '' && Number.isFinite(Number(i.blast_radius))) blastRaw = Number(i.blast_radius);
+  const blast = Math.max(0, Math.min(RWEP_WEIGHTS.blast_radius, blastRaw));
+  return {
+    cisa_kev: i.cisa_kev ? RWEP_WEIGHTS.cisa_kev : 0,
+    poc_available: i.poc_available ? RWEP_WEIGHTS.poc_available : 0,
+    ai_factor: (i.ai_assisted_weapon || i.ai_assisted_weaponization || i.ai_discovered) ? RWEP_WEIGHTS.ai_factor : 0,
+    active_exploitation: RWEP_WEIGHTS.active_exploitation * aeMultiplier,
+    blast_radius: blast,
+    patch_available: i.patch_available ? RWEP_WEIGHTS.patch_available : 0,
+    live_patch_available: i.live_patch_available ? RWEP_WEIGHTS.live_patch_available : 0,
+    reboot_required: reboot ? RWEP_WEIGHTS.reboot_required : 0,
+  };
+}
+
 function timeline(rwepScore) {
   if (rwepScore >= 90) return { hours: 4, label: 'Immediate — live patch or isolate within 4 hours' };
   if (rwepScore >= 75) return { hours: 24, label: 'Urgent — patch or compensating controls within 24 hours' };
@@ -804,6 +827,7 @@ function packageConfidence(inputs) {
 module.exports = {
   score,
   scoreCustom,
+  postWeightFactors,
   timeline,
   compare,
   packageConfidence,