@blamejs/exceptd-skills 0.19.25 → 0.19.26

package/lib/collectors/citation-hygiene.js

@@ -131,6 +131,15 @@ function recordRejectedOrDisputed(note, selfId) {
   // one, so a different cve appearing as that replacement must NOT suppress the
   // flag (e.g. "this record was rejected as a duplicate of CVE-Y").
   const REPLACEMENT_OF = /\b(?:duplicate|dup)\b[\s\w-]*\bof\b|\b(?:supersed\w+|replaced|merged)\b[\s\w-]*\b(?:by|into)\b|\bin\s+favou?r\s+of\b/i;
+  // "Reject" is also an ordinary verb for what code does to input, and a vector
+  // description is full of it: a parser rejects an invalid DTD, a validator
+  // rejects a traversal sequence. Nothing about the CVE record is being claimed
+  // there. The record sense always names what was rejected — a record, an
+  // entry, an identifier — or who rejected it (MITRE, NVD, the CNA), so require
+  // one of those nearby before treating a reject-word as record-level. The
+  // alternative is rewording accurate mechanism prose to dodge a regex, which
+  // costs the catalog more than the false positive does.
+  const RECORD_CONTEXT = /\b(record|entry|identifier|assignment|advisory|CVE|MITRE|NVD|CNA)\b/i;
   const otherCve = (s) => (s.match(/CVE-\d{4}-\d{4,}/gi) || []).some((c) => c.toUpperCase() !== self);
   let m;
   while ((m = re.exec(note)) !== null) {
@@ -150,6 +159,9 @@ function recordRejectedOrDisputed(note, selfId) {
       const lastTokens = before.trim().split(/[\s-]+/).slice(-3).join(" ");
       if (QUALIFIER.test(lastTokens)) continue;
     }
+    // (d) A 'reject' with nothing record-shaped around it is the ordinary verb
+    //     — code rejecting input — not a statement about this CVE's record.
+    if (word.startsWith("reject") && !RECORD_CONTEXT.test(before + " " + after)) continue;
     return true;
   }
   return false;

package/lib/cve-enrich.js

@@ -29,8 +29,17 @@ function parseCvssVector(vectorString) {
 function extractVendorAdvisories(references, kevVendor, cveId) {
   void cveId;
   const out = [];
+  // Deduplicate by URL. Upstream reference lists repeat the same advisory —
+  // NVD carries a link once per tag combination, and a draft that has already
+  // been through one enrichment pass can carry its own copy alongside the
+  // original. Repeating it produces an advisory list that says the same thing
+  // several times in every rendered output, and duplicated relationships in the
+  // generated indexes.
+  const seen = new Set();
   for (const r of references || []) {
     if (Array.isArray(r.tags) && r.tags.includes('Vendor Advisory')) {
+      if (seen.has(r.url)) continue;
+      seen.add(r.url);
       out.push({ vendor: kevVendor || 'Vendor', advisory_id: r.url.split('/').pop() || r.url,
         url: r.url, severity: 'unknown', published_date: null });
     }
@@ -40,12 +49,32 @@ function extractVendorAdvisories(references, kevVendor, cveId) {
 
 function _num(x) { const n = Number(x); return Number.isFinite(n) ? n : null; }
 
+function _ordinal(n) {
+  const suffixes = ['th', 'st', 'nd', 'rd'];
+  const v = n % 100;
+  return `${n}${suffixes[(v - 20) % 10] || suffixes[v] || suffixes[0]}`;
+}
+
+// The operator-readable restatement of an entry's EPSS fields. It is DERIVED
+// text, not authored, so it has exactly one definition — here — which both the
+// writer (lib/refresh-external.js, when a refresh moves the numbers) and the
+// gate (scripts/check-epss-consistency.js, which compares an entry's stored
+// note against the fields) call. Two copies of this renderer would drift, and
+// the drift would surface as a gate failure on correctly-refreshed data.
+function renderEpssNote(entry) {
+  const pct = Math.round(entry.epss_percentile * 100);
+  return `FIRST EPSS ${entry.epss_score} (${_ordinal(pct)} percentile) as of ${entry.epss_date}.`;
+}
+
 function deriveMechanicalFields(facts, today) {
   const f = facts || {};
   const kev = f.kev || {};
   const cvss = f.cvss || null;
   const parsed = parseCvssVector(cvss && cvss.vector);
-  const cweRefs = (f.cwe_nvd || []).filter((c) => /^CWE-\d+$/.test(c));
+  // Deduplicate: an upstream weakness list can name the same CWE twice (one per
+  // assigning source), and a repeated id becomes a repeated relationship
+  // wherever the catalog is joined against the CWE catalog.
+  const cweRefs = [...new Set((f.cwe_nvd || []).filter((c) => /^CWE-\d+$/.test(c)))];
   const vendorAdv = extractVendorAdvisories(f.references, kev.vendor, f.id);
   const verification = [...new Set([
     `https://nvd.nist.gov/vuln/detail/${f.id}`,
@@ -98,7 +127,22 @@ function assembleEntry(facts, judgment, today) {
   // supplied list (e.g. when NVD only returns NVD-CWE-noinfo). Kept out of the
   // blanket JUDGMENT_KEYS loop so facts take precedence over judgment here.
   if ((!entry.cwe_refs || entry.cwe_refs.length === 0) && Array.isArray(j.cwe_refs))
-    entry.cwe_refs = j.cwe_refs;
+    entry.cwe_refs = [...new Set(j.cwe_refs)];
+
+  // The same guard on the judgment-supplied reference arrays. These are written
+  // by hand, so a repeat is a transcription slip rather than an upstream quirk,
+  // but it lands in exactly the same joined output.
+  for (const k of ['attack_refs', 'atlas_refs', 'verification_sources', 'vendor_update_paths'])
+    if (Array.isArray(entry[k])) entry[k] = [...new Set(entry[k])];
+  if (Array.isArray(entry.vendor_advisories)) {
+    const seenUrl = new Set();
+    entry.vendor_advisories = entry.vendor_advisories.filter((a) => {
+      const u = a && a.url;
+      if (!u || seenUrl.has(u)) return !u;
+      seenUrl.add(u);
+      return true;
+    });
+  }
 
   // Defaults + intake markers.
   if (entry.ai_discovery_source === undefined)
@@ -169,4 +213,4 @@ function findOrphans(entry, catalogs) {
   return out;
 }
 
-module.exports = { parseCvssVector, extractVendorAdvisories, deriveMechanicalFields, assembleEntry, findOrphans };
+module.exports = { parseCvssVector, extractVendorAdvisories, deriveMechanicalFields, assembleEntry, findOrphans, renderEpssNote };

package/lib/refresh-external.js

@@ -39,6 +39,7 @@ const fs = require("fs");
 const path = require("path");
 const { execFileSync } = require("child_process");
 const { selectNvdCvss, cvssVersionOf } = require("./cvss");
+const { renderEpssNote } = require("./cve-enrich");
 
 const ROOT = path.join(__dirname, "..");
 const ABS = (p) => path.join(ROOT, p);
@@ -403,6 +404,79 @@ function kevDiffWithDiscoveryFromCache(ctx) {
   };
 }
 
+/**
+ * EPSS publishes score, percentile and date as ONE row, and the percentile is
+ * the score's rank within that day's publication. Writing one field without the
+ * others leaves a pair drawn from two different days: both numbers in range,
+ * both shaped like EPSS values, and nothing downstream able to tell them apart
+ * from a coherent pair. That is the corruption scripts/check-epss-consistency.js
+ * detects, and a detector cannot repair it — as that gate's own header says, the
+ * provenance guarantee has to come from the write side.
+ *
+ * The bug this replaces applied the drift threshold to score and percentile
+ * INDEPENDENTLY: a score that moved past the threshold was written while its
+ * percentile, having moved less, was left at yesterday's value. So the rule is
+ * now that the threshold decides WHETHER an entry is refreshed, never WHICH of
+ * its fields is. Once either number has moved, every field comes from that same
+ * row — including the one whose own delta is under the threshold, and including
+ * a half of the pair the entry was missing entirely.
+ *
+ * `local` is the catalog entry; `fetched` is {score, percentile, date} read from
+ * one EPSS row.
+ */
+const EPSS_DRIFT = 0.05;
+
+function epssTripleDiffs(id, local, fetched, drift) {
+  const { score, percentile, date } = fetched;
+  // Only a COMPLETE row can produce a coherent triple, so a partial one is
+  // refused before anything else is considered. Guarding this per-branch was
+  // not enough: a row carrying just one number can still cross the drift
+  // threshold on that number, and writing it beside the other half from the
+  // previous publication is precisely the mixed-publication state this function
+  // exists to prevent. At the top, the guard covers a complete local entry and
+  // a half-populated one alike.
+  //
+  // The date is the third member, not decoration. A row with both numbers and
+  // no date would write the new numbers while the entry kept its old
+  // epss_date — and the regenerated note would then state those numbers "as of"
+  // a publication they did not come from, which is the same defect wearing
+  // different clothes.
+  // Finite, not merely present: the callers build these with Number(), which
+  // turns a malformed cached value into NaN rather than null. A NaN would pass
+  // a nullish check, ride the repair path, and land in the catalog serialised
+  // as null — reintroducing the half-populated entry the repair exists to fix.
+  if (!Number.isFinite(score) || !Number.isFinite(percentile) || !date) return [];
+
+  const scoreMoved = local.epss_score != null && Math.abs(score - local.epss_score) > drift;
+  const pctMoved = local.epss_percentile != null && Math.abs(percentile - local.epss_percentile) > drift;
+
+  // An entry holding one half of the pair is already inconsistent, and the
+  // drift threshold cannot reach it: the absent side has nothing to compare
+  // against, so its "moved" test is false by construction, and if the present
+  // side happens to be stable the entry stays half-populated forever while the
+  // consistency gate keeps failing on it. A complete row repairs it regardless
+  // of how far anything moved. An entry carrying NO EPSS at all is left alone —
+  // that is an absent field, not an incoherent pair.
+  const localIncomplete = (local.epss_score == null) !== (local.epss_percentile == null);
+  if (!scoreMoved && !pctMoved && !localIncomplete) return [];
+
+  // `coherent` records PROVENANCE: these diffs were derived from a single
+  // complete, validated row, so whatever subset of fields actually differs, the
+  // entry is left describing one publication. The consumer cannot infer that
+  // from the fields alone — a complete row whose percentile rounds to the same
+  // value legitimately emits the score by itself, which is indistinguishable
+  // from a hand-built partial diff set unless the emitter says so.
+  const out = [];
+  const emit = (field, before, after, severity) =>
+    out.push({ id, field, before, after, severity, coherent: true });
+
+  if (score !== local.epss_score) emit("epss_score", local.epss_score ?? null, score, "medium");
+  if (percentile !== local.epss_percentile) emit("epss_percentile", local.epss_percentile ?? null, percentile, "medium");
+  // `date` is non-empty by the guard at the top of this function.
+  if (date !== local.epss_date) emit("epss_date", local.epss_date ?? null, date, "low");
+  return out;
+}
+
 const EPSS_SOURCE = {
   name: "epss",
   description: "FIRST.org EPSS scores",
@@ -416,25 +490,18 @@ const EPSS_SOURCE = {
     let errors = 0;
     for (const r of report.results) {
       if (r.status === "unreachable") errors++;
-      for (const d of r.discrepancies || []) {
-        if (d.field === "epss_score" || d.field === "epss_percentile") {
-          diffs.push({ id: r.cve_id, field: d.field, before: d.local, after: d.fetched, severity: d.severity });
-        }
-      }
-      // epss_date refreshes when score does.
-      if (r.fetched?.epss?.date && r.local) {
-        diffs.push({ id: r.cve_id, field: "epss_date", before: r.local.epss_date, after: r.fetched.epss.date, severity: "low" });
-      }
+      // Diff off the FETCHED ROW rather than off r.discrepancies. The
+      // discrepancy list names only the fields that differ enough to report,
+      // which is precisely the input that used to let score and percentile be
+      // written from different publications — see epssTripleDiffs.
+      if (!r.fetched?.epss || !r.local) continue;
+      diffs.push(...epssTripleDiffs(r.cve_id, r.local, r.fetched.epss, EPSS_DRIFT));
     }
-    // Collapse duplicates: epss_date should appear once per CVE only when
-    // an epss field actually moved.
-    const epssCves = new Set(diffs.filter((d) => d.field === "epss_score" || d.field === "epss_percentile").map((d) => d.id));
-    const filtered = diffs.filter((d) => d.field !== "epss_date" || epssCves.has(d.id));
     return {
       status: errors === 0 ? "ok" : errors === report.results.length ? "unreachable" : "partial",
-      diffs: filtered,
+      diffs,
       errors,
-      summary: `${filtered.length} EPSS diffs; ${errors} unreachable / ${report.total} total`,
+      summary: `${diffs.length} EPSS diffs; ${errors} unreachable / ${report.total} total`,
     };
   },
   async applyDiff(ctx, diffs) {
@@ -442,6 +509,7 @@ const EPSS_SOURCE = {
     const errors = [];
     const catalogPath = ctx.cvePath || ABS("data/cve-catalog.json");
     await withCatalogLock(catalogPath, (catalog) => {
+      const touchedFields = new Map();
       for (const d of diffs) {
         if (!catalog[d.id]) {
           errors.push(`EPSS: no local entry for ${d.id}`);
@@ -449,8 +517,36 @@ const EPSS_SOURCE = {
         }
         catalog[d.id][d.field] = d.after;
         catalog[d.id].last_verified = TODAY;
+        // An id is coherent only if EVERY diff applied to it declares that
+        // provenance; one unmarked diff in the set is enough to withhold the
+        // note rewrite.
+        touchedFields.set(d.id, (touchedFields.get(d.id) !== false) && d.coherent === true);
         updated++;
       }
+      // `epss_note` restates the three fields in prose. It is derived, so a
+      // refresh that moves the numbers and leaves the sentence behind makes the
+      // entry state two different scores as of two different dates — which the
+      // consistency gate reports as a failure on data that was otherwise
+      // refreshed correctly. Rebuild it from the same renderer the gate checks
+      // against, and only for entries that already carry one: adding the note
+      // to an entry that never had it is a content change, not a refresh.
+      for (const [id, coherent] of touchedFields) {
+        const e = catalog[id];
+        if (typeof e.epss_note !== "string") continue;
+        if (typeof e.epss_score !== "number" || typeof e.epss_percentile !== "number") continue;
+        // Regenerate only for diffs whose emitter vouched that they came from
+        // one complete publication. applyDiff is exported and is also driven by
+        // fixtures carrying a score and a date without a percentile, so a diff
+        // set can move half the pair on its own; rewriting the note from the
+        // surviving stale percentile would make a mixed-publication entry read
+        // as current and suppress the only signal the consistency gate has for
+        // it. Deciding this from WHICH fields changed does not work — a
+        // complete row whose percentile rounds to the same value emits the
+        // score alone, and skipping it there would strand the note stale
+        // forever, because the next refresh sees no diff left to repair it.
+        if (!coherent) continue;
+        e.epss_note = renderEpssNote(e);
+      }
       catalog._meta = catalog._meta || {};
       catalog._meta.last_updated = TODAY;
       ctx.cveCatalog = catalog;
@@ -1019,7 +1115,7 @@ function epssDiffFromCache(ctx) {
   const cves = Object.keys(ctx.cveCatalog).filter((k) => /^CVE-\d{4}-\d{4,7}$/.test(k));
   const diffs = [];
   let errors = 0;
-  const drift = 0.05;
+  const drift = EPSS_DRIFT;
   for (const id of cves) {
     const payload = readCachedJson(ctx.cacheDir, "epss", id, { forceStale: ctx.forceStale });
     if (!payload) { errors++; continue; }
@@ -1031,20 +1127,12 @@ function epssDiffFromCache(ctx) {
     let row = (payload.data || []).find((r) => r?.cve === id);
     if (!row && (payload.data || []).length === 1 && (payload.data || [])[0]?.cve == null) row = (payload.data || [])[0];
     if (!row) continue;
-    const score = row.epss != null ? Number(row.epss) : null;
-    const pct = row.percentile != null ? Number(row.percentile) : null;
-    const local = ctx.cveCatalog[id];
-    if (score != null && local.epss_score != null && Math.abs(score - local.epss_score) > drift) {
-      diffs.push({ id, field: "epss_score", before: local.epss_score, after: score, severity: "medium" });
-    }
-    if (pct != null && local.epss_percentile != null && Math.abs(pct - local.epss_percentile) > drift) {
-      diffs.push({ id, field: "epss_percentile", before: local.epss_percentile, after: pct, severity: "medium" });
-    }
-    if (row.date && local.epss_date && row.date !== local.epss_date) {
-      // Only emit a date diff when we also emitted a score/percentile diff for this CVE.
-      const moved = diffs.some((d) => d.id === id && (d.field === "epss_score" || d.field === "epss_percentile"));
-      if (moved) diffs.push({ id, field: "epss_date", before: local.epss_date, after: row.date, severity: "low" });
-    }
+    const fetched = {
+      score: row.epss != null ? Number(row.epss) : null,
+      percentile: row.percentile != null ? Number(row.percentile) : null,
+      date: row.date || null,
+    };
+    diffs.push(...epssTripleDiffs(id, ctx.cveCatalog[id], fetched, drift));
   }
   const status = errors === 0 ? "ok" : errors === cves.length ? "unreachable" : "partial";
   return { status, diffs, errors, summary: `${diffs.length} EPSS diffs (from cache); ${errors} missing entries` };
@@ -1966,4 +2054,4 @@ if (require.main === module) {
   });
 }
 
-module.exports = { ALL_SOURCES, loadCtx, parseArgs, seedSingleAdvisory, withCatalogLock, writeJsonAtomic, nvdDiffFromCache, kevDiffFromCache };
+module.exports = { ALL_SOURCES, loadCtx, parseArgs, seedSingleAdvisory, withCatalogLock, writeJsonAtomic, nvdDiffFromCache, kevDiffFromCache, epssDiffFromCache, epssTripleDiffs, EPSS_DRIFT };