@blamejs/exceptd-skills 0.19.33 → 0.19.34

package/lib/source-advisories.js

@@ -1,50 +1,12 @@
 'use strict';
 
 /**
- * lib/source-advisories.js — primary-source advisory-feed polling.
+ * Polls primary-source advisory feeds that sit EARLIER in the disclosure
+ * pipeline than the kev / epss / nvd / ghsa / osv sources, which see only its
+ * last step and lag by 3-14 days. Report-only: applyDiff never writes.
  *
- * Why this exists. The post-mortem on CVE-2026-46333 (ssh-keysign-pwn,
- * disclosed 2026-05-14, missed by the toolkit at T+0 through T+3) found
- * that the existing source set (kev, epss, nvd, rfc, pins, ghsa, osv)
- * sits at the END of the disclosure pipeline. Qualys → kernel.org commit
- * → distro advisory → NVD enrichment is sequential; the existing pollers
- * only see the last step, which lags by 3-14 days.
- *
- * The 4 feeds below sit much earlier in the pipeline:
- *
- *   Qualys TRU RSS    — Qualys-disclosed CVEs at T+0 (the originator of
- *                       the ssh-keysign-pwn class of disclosure)
- *   Red Hat RHSA RSS  — RHEL security advisories at T+1, often before NVD
- *   Ubuntu USN RSS    — Ubuntu security notices at T+1, often before NVD
- *   ZDI advisories    — Zero Day Initiative + Pwn2Own disclosures at T+0
- *
- * Behaviour. Each call returns a structured REPORT — not a catalog mutation.
- * Operators consume the report via `exceptd refresh --check-advisories` and
- * decide which advisories warrant a `refresh --advisory <CVE-ID>` auto-import
- * to seed a draft entry. The report is informational; nothing is auto-written
- * to the catalog. This is the conservative-by-default contract: primary-source
- * surfacing must not silently mutate the catalog without operator triage.
- *
- * Output shape:
- *   {
- *     status: 'ok' | 'partial' | 'unreachable',
- *     diffs: [
- *       { id: 'CVE-2026-46333', source: 'qualys', advisory_url: '...',
- *         disclosed_at: '2026-05-14', title: '...', in_catalog: false }
- *     ],
- *     summary: '4/4 feeds reachable; 3 new CVE references found'
- *   }
- *
- * Each diff is read-only — there is no `applyDiff` that writes the catalog.
- * That's by design: a fresh advisory from a primary source has insufficient
- * fields to satisfy Hard Rule #1 (CVSS / KEV / PoC / AI-discovery /
- * active-exploitation / patch-availability). The operator routes the
- * promising ones through `refresh --advisory <CVE-ID>` which goes through
- * the existing GHSA / OSV / NVD enrichment path (those pollers are mature).
- *
- * Cache mode (--from-cache <dir>): expected cache layout is
- *   <dir>/advisories/<feed>.xml — caller passes `ctx.cacheDir`.
- * Fixture mode: `ctx.fixtures.advisories = { qualys: '<xml>', ... }`.
+ * Cache mode (--from-cache <dir>) reads <dir>/advisories/<feed>.<ext> via
+ * `ctx.cacheDir`; fixture mode reads `ctx.fixtures.advisories[<feed>]`.
  */
 
 const path = require('path');
@@ -53,8 +15,7 @@ const { withRetry } = require('../vendor/blamejs/retry.js');
 
 const TODAY = new Date().toISOString().slice(0, 10);
 
-// Feed registry. Each entry has a kind (rss | json), a URL, and a parser.
-// Parsers return [{ cve_ids: [...], title, link, published }, ...].
+// Feed registry; `kind` selects the parser checkFeed() applies.
 const FEEDS = [
   {
     name: 'qualys',
@@ -80,8 +41,6 @@ const FEEDS = [
     kind: 'rss',
     description: 'Zero Day Initiative — vendor-acknowledged advisories from ZDI + Pwn2Own pipeline',
   },
-  // v0.13.3 additions — extend coverage to 4 more primary-source venues
-  // identified in the v0.13.1 post-mortem follow-up:
   {
     name: 'kernel-org',
     url: 'https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/atom?h=master',
@@ -106,17 +65,6 @@ const FEEDS = [
     kind: 'rss',
     description: 'CISA cybersecurity advisories feed — federal-vendor coordinated disclosures (separate from KEV which captures only exploited-in-the-wild items)',
   },
-  // v0.13.14 additions — closes the "silent kernel patch + delayed-research-
-  // disclosure" intake gap surfaced by DirtyDecrypt (CVE-2026-31635). That
-  // CVE was patched in mainline 2026-04-25, the kernel.org Atom-feed rolling
-  // window rotated past the fix commit before the daily intake noticed, the
-  // V12 rediscovery on 2026-05-09 went to maintainers privately rather than
-  // to oss-security@openwall, and the PoC publication on 2026-05-17 surfaced
-  // on vendor security blogs (Microsoft / Sysdig / Trail of Bits) that the
-  // 8-feed primary-source set did not cover. Vendor security blogs are the
-  // canonical signal channel for "kernel-class CVE patched silently, then
-  // class-of-bug research published weeks later" — adding them closes the
-  // class without polluting the catalog with news-aggregator noise.
   {
     name: 'microsoft-security-blog',
     url: 'https://www.microsoft.com/en-us/security/blog/feed/',
@@ -141,22 +89,6 @@ const FEEDS = [
     kind: 'rss',
     description: 'Embrace the Red (Johann Rehberger) — AI-tool prompt-injection + agentic-AI research. Anchored CVE-2025-53773 (Copilot YOLO mode) and the agentic-IDE host-execution class.',
   },
-  // v0.13.17 additions — closes the "researcher GitHub-release drop +
-  // tech-press writeup" intake gap surfaced by the four Nightmare-Eclipse
-  // catalog entries (MiniPlasma / YellowKey / GreenPlasma / UnDefend).
-  // The 12-feed set polled advisory venues + vendor research blogs but
-  // had no coverage for either (a) named researcher handles whose prior
-  // drops are already in the catalog (BlueHammer / CVE-2026-33825 was in,
-  // four sibling drops were not) nor (b) the canonical tech-press
-  // venues that surface "researcher dropped PoC on GitHub, no advisory
-  // yet" events. The bleepingcomputer-security + thehackernews feeds
-  // close (b); the nightmare-eclipse-gitlab feed closes (a) as the
-  // anchor handle for the v0.13.17 backfill. NEW-CTRL-073 generalises
-  // the handle-tracker control — additional handles get registered as
-  // their drops land in the catalog. See also NEW-CTRL-074 (CVE-
-  // regression-watcher) in lib/cve-regression-watcher.js for the
-  // complementary detection method that catches re-broken historical
-  // CVEs without depending on a new ID being assigned.
   {
     name: 'bleepingcomputer-security',
     url: 'https://www.bleepingcomputer.com/feed/',
@@ -178,15 +110,10 @@ const FEEDS = [
   },
 ];
 
-// Permissive CVE-ID matcher. The official format is CVE-YYYY-NNNN+ but
-// some feeds embed CVEs in arbitrary surrounding markup AND occasionally
-// emit lowercase "cve-yyyy-nnnn" in URLs or filenames. Case-insensitive
-// match, then uppercase + dedupe in extractCveIds().
+// Permissive: some feeds emit lowercase "cve-yyyy-nnnn" in URLs and filenames,
+// so the match is case-insensitive and extractCveIds() uppercases and dedupes.
 const CVE_RE = /CVE-(?:19|20)\d{2}-\d{4,7}/gi;
 
-/**
- * Extract CVE IDs from a string blob. De-duplicates within the blob.
- */
 function extractCveIds(text) {
   if (typeof text !== 'string' || text.length === 0) return [];
   const matches = text.match(CVE_RE);
@@ -194,23 +121,10 @@ function extractCveIds(text) {
   return [...new Set(matches.map((s) => s.toUpperCase()))];
 }
 
-/**
- * RSS / Atom parser. v0.13.20 replaces the original regex-based parser
- * (which silently failed on XML namespaces, nested CDATA, self-closing
- * tags, HTML entities, and multi-line content) with a proper streaming
- * XML tokenizer defined in lib/xml-tokenizer.js. Parser errors surface
- * via the second `errors` argument so consumers can observe parse
- * failures instead of receiving a silent empty array.
- *
- * Returns [{ title, link, published, body }, ...].
- */
 const { parseFeedDetailed: tokenizerParseFeedDetailed } = require('./xml-tokenizer');
 
-// Parser errors are ALWAYS collected and surfaced — the tokenizer's loud-error
-// contract is no longer opt-in. The optional `errors` array is filled when a
-// caller passes one (so a reachable-but-unparsable feed reads 'partial' in the
-// refresh report instead of '0 new CVEs'). A caller that forgets the array
-// still triggers the always-on collection via parseFeedDetailed.
+// Returns [{ title, link, published, body }, ...]. Passing `errors` copies the
+// tokenizer's parse errors out, so an unparsable feed reads 'partial', not '0'.
 function parseRssAtom(xml, errors = null) {
   const { items, errors: collected } = tokenizerParseFeedDetailed(xml);
   if (Array.isArray(errors)) {
@@ -220,11 +134,8 @@ function parseRssAtom(xml, errors = null) {
 }
 
 /**
- * CSAF index parser — Red Hat ships a plain-text index of advisory JSON
- * files under data/csaf/v2/advisories/YYYY/index.txt. Each line is a
- * relative filename. We don't fetch the per-advisory JSON in v0.13.1
- * (would blow the polling budget); we surface the advisory IDs that
- * mention CVE-YYYY-NNNN inline.
+ * Red Hat ships a plain-text index whose lines are advisory JSON filenames. The
+ * per-advisory JSON is never fetched, so only CVE IDs in a filename surface.
  */
 function parseCsafIndex(text) {
   if (typeof text !== 'string') return [];
@@ -236,20 +147,10 @@ function parseCsafIndex(text) {
 }
 
 /**
- * GitHub public-events parser — NEW-CTRL-073 researcher-handle tracker.
- *
- * Input is the JSON body of `https://api.github.com/users/<handle>/events/public`
- * (array of event objects). Output flattens each event into a standard
- * { title, link, published, body, researcher_handle } shape that
- * extractCveIds() can scan and the diff-deduplicator can route. Events of
- * interest: PushEvent (release commits), ReleaseEvent (tagged drops),
- * CreateEvent (tag / branch creation often signals an upcoming drop),
- * PublicEvent (a private repo flipped public — frequent drop pattern).
- *
- * The diff carries `researcher_handle` so callers + tests can assert the
- * handle-tracker fired even when no CVE ID is present in the commit text
- * (the MiniPlasma class — drop names a researcher-coined alias rather
- * than a CVE ID until MITRE / vendor assigns).
+ * Researcher-handle tracker (NEW-CTRL-073) over the JSON body of
+ * `https://api.github.com/users/<handle>/events/public`. Flattens each
+ * PushEvent / ReleaseEvent / CreateEvent / PublicEvent into the standard
+ * { title, link, published, body, researcher_handle } shape.
  */
 function parseGitHubEvents(body, feed) {
   let arr;
@@ -287,25 +188,15 @@ function parseGitHubEvents(body, feed) {
 }
 
 /**
- * GitLab public-activity parser — NEW-CTRL-073 researcher-handle tracker.
- *
- * GitLab publishes a public activity Atom feed at
- * https://gitlab.com/<handle>.atom. Each <entry> describes one activity
- * (pushed a tag, created a project/repository, pushed commits). We parse it
- * with the shared Atom tokenizer and classify the entry title into the same
- * event_type vocabulary the GitHub tracker emits, so the handle-drop diff in
- * checkFeed fires identically across both sources. researcher_handle comes
- * from the feed's explicit field (authoritative; the .atom URL also carries
- * the handle as a fallback). This is the migration target after the handle's
- * GitHub account was removed — the .atom feed needs no API token and the
- * existing parseRssAtom tokenizer already handles its XML.
+ * Researcher-handle tracker over a GitLab public-activity Atom feed
+ * (https://gitlab.com/<handle>.atom). Entry titles are classified into the same
+ * event_type vocabulary parseGitHubEvents emits, so checkFeed's handle-drop diff
+ * fires identically for both sources.
  */
 function parseGitLabActivity(body, feed, errorsOut = null) {
   const errors = [];
   const entries = parseRssAtom(body, errors);
-  // Thread the Atom parse errors back to the caller's channel (checkFeed)
-  // instead of dropping them — a reachable-but-unparsable GitLab activity feed
-  // must read 'partial' in the refresh report, same as the RSS/Atom path.
+  // Thread the Atom parse errors back so this feed reads 'partial' like RSS.
   if (Array.isArray(errorsOut)) {
     for (const e of errors) errorsOut.push(e);
   }
@@ -319,8 +210,7 @@ function parseGitLabActivity(body, feed, errorsOut = null) {
     else if (/created (?:repository|project)|imported project/i.test(t)) event_type = 'PublicEvent';
     else if (/pushed (?:new commits|to)/i.test(t)) event_type = 'PushEvent';
     else if (/created branch/i.test(t)) event_type = 'CreateEvent';
-    // Project path: GitLab activity titles read "<handle> pushed to project
-    // <ns>/<repo>"; the entry link path is the secondary source.
+    // Activity titles read "<handle> pushed to project <ns>/<repo>"; link is fallback.
     let repo_name = '';
     const titleM = t.match(/project\s+(\S+\/\S+)/i);
     if (titleM) repo_name = titleM[1];
@@ -340,9 +230,6 @@ function parseGitLabActivity(body, feed, errorsOut = null) {
   });
 }
 
-/**
- * Fetch a feed body. In fixture / cache modes, read from disk.
- */
 async function fetchFeed(feed, ctx) {
   if (ctx.fixtures && ctx.fixtures.advisories && ctx.fixtures.advisories[feed.name]) {
     return { ok: true, body: ctx.fixtures.advisories[feed.name] };
@@ -354,10 +241,8 @@ async function fetchFeed(feed, ctx) {
     return { ok: true, body: fs.readFileSync(p, 'utf8') };
   }
   if (typeof fetch !== 'function') return { ok: false, error: 'fetch() not available — Node 18+ required' };
-  // Adopt the vendored blamejs retry primitive (BJS-02): a transient feed
-  // failure (HTTP 429/5xx, timeout/abort, network reset) is retried with
-  // backoff + jitter instead of dropping the feed on the first hiccup. A
-  // permanent 4xx is not retried. Each attempt gets a fresh abort timer.
+  // A transient failure (429/5xx, timeout/abort, network reset) is retried with
+  // backoff + jitter; a permanent 4xx is not. Each attempt gets its own timer.
   const attempt = async () => {
     const ac = new AbortController();
     const timer = setTimeout(() => ac.abort(), 8000);
@@ -384,21 +269,17 @@ async function fetchFeed(feed, ctx) {
 }
 
 /**
- * Walk one feed: fetch, parse, extract CVE IDs, compare to local catalog.
- * Returns { diffs, errors, status }.
+ * Walk one feed: fetch, parse, extract CVE IDs, compare to the local catalog.
+ * Returns { diffs, observations, errors, status, parse_errors }.
  */
 async function checkFeed(feed, ctx) {
   const res = await fetchFeed(feed, ctx);
   if (!res.ok) return { diffs: [], errors: 1, status: 'unreachable', _why: res.error };
   let items;
-  // Parse errors are collected on the XML-parsing feed kinds so a reachable-
-  // but-unparsable feed reads 'partial' in the report instead of silently
-  // returning 0 new CVEs (the loud-error contract was opt-in and the live
-  // path never opted in).
+  // Collected on the XML feed kinds so an unparsable feed reads 'partial'.
   const parseErrors = [];
   if (feed.kind === 'csaf-index') {
     items = parseCsafIndex(res.body);
-    // Flatten cves_from_filename onto cve_ids field uniformly.
     items = items.map((it) => ({ ...it, cve_ids: it.cves_from_filename || [] }));
   } else if (feed.kind === 'github-events') {
     items = parseGitHubEvents(res.body, feed);
@@ -411,17 +292,11 @@ async function checkFeed(feed, ctx) {
     items = items.map((it) => ({ ...it, cve_ids: extractCveIds(`${it.title} ${it.body} ${it.link}`) }));
   }
   const diffs = [];
-  // v0.13.17: surface EVERY extracted CVE-ID — including IDs already in
-  // catalog — for NEW-CTRL-074 (cve-regression-watcher) consumption. The
-  // diffs[] array remains filtered to NOT-IN-CATALOG so existing callers
-  // see no behavior change, but a parallel observations[] carries the
-  // full set with the in_catalog flag preserved. The regression watcher
-  // routes historical-CVE references (e.g. CVE-2020-17103) through this
-  // path so its action: "annotate" verdict can fire even when the
-  // historical entry IS a catalog key.
+  // observations carries EVERY extracted CVE ID, in-catalog included, for the
+  // regression watcher (lib/cve-regression-watcher.js), whose "annotate" verdict
+  // fires on a catalog key. diffs[] stays filtered to not-in-catalog.
   const observations = [];
   for (const it of items) {
-    // Standard CVE-ID-bearing diff (every feed kind).
     for (const cveId of it.cve_ids) {
       const inCatalog = !!ctx.cveCatalog[cveId];
       const baseRecord = {
@@ -437,11 +312,8 @@ async function checkFeed(feed, ctx) {
         diffs.push({ ...baseRecord });
       }
     }
-    // NEW-CTRL-073 — researcher-handle diff. Github-events surfaces drop
-    // activity even when no CVE ID is present yet (the MiniPlasma class:
-    // researcher publishes alias name, vendor advisory + CVE assignment
-    // arrive days / weeks later). The diff carries handle + event_type
-    // so downstream triage can prioritise without waiting for an ID.
+    // Handle-drop diff: a researcher publishes an alias name before the advisory
+    // and CVE assignment, so the drop surfaces with handle + event_type.
     if ((feed.kind === 'github-events' || feed.kind === 'gitlab-activity') && it.cve_ids.length === 0 && (it.event_type === 'ReleaseEvent' || it.event_type === 'PublicEvent')) {
       diffs.push({
         id: `HANDLE:${it.researcher_handle || feed.name}:${it.repo_name || 'unknown'}@${(it.published || '').slice(0, 10)}`,
@@ -457,10 +329,8 @@ async function checkFeed(feed, ctx) {
       });
     }
   }
-  // Fold reachable-but-unparsable into a 'partial' status via a NEW channel.
-  // The integer `errors` field stays the unreachable count (0 here — the feed
-  // WAS reached) so the aggregate unreachable===FEEDS.length math and the
-  // refresh-* assertions that key off it keep working untouched.
+  // `errors` stays the UNREACHABLE count (0 here — the feed WAS reached), so the
+  // aggregate math keeps working. A parse failure travels on parse_errors.
   return {
     diffs,
     observations,
@@ -471,22 +341,15 @@ async function checkFeed(feed, ctx) {
   };
 }
 
-/**
- * The exported SOURCE definition, matching the shape ALL_SOURCES expects.
- */
+/** The exported SOURCE definition, in the shape ALL_SOURCES expects. */
 const ADVISORIES_SOURCE = {
   name: 'advisories',
   description: 'Primary-source advisory feeds (Qualys TRU, Red Hat RHSA, Ubuntu USN, Zero Day Initiative / ZDI) — surfaces CVE IDs disclosed at T+0 to T+1 that lag NVD enrichment. Report-only — does not auto-write the catalog.',
   applies_to: 'data/cve-catalog.json',
-  // The no-op applyDiff below already refuses to write, but the flag is what
-  // the *callers* read. refresh-external only counts a source toward the
-  // upsert total when `!report_only`, and the refresh workflow gates its apply
-  // step and its entire open-pr job on that total being non-zero. This feed
-  // reports every CVE seen across USN / RHSA / ZDI / CISA that the catalog
-  // does not track — structurally in the thousands, and never zero, because
-  // the catalog deliberately does not carry every Ubuntu kernel CVE. Leaving
-  // the flag off therefore pinned the upsert count permanently above zero and
-  // made "open a PR only when something changed" unconditional.
+  // Callers read this flag, not the no-op applyDiff: refresh-external counts a
+  // source toward the upsert total only when `!report_only`, and the refresh
+  // workflow gates its apply step and open-pr job on that total. This feed never
+  // reports zero, so clearing the flag opens a PR unconditionally, forever.
   report_only: true,
   async fetchDiff(ctx) {
     const results = await Promise.all(FEEDS.map((feed) => checkFeed(feed, ctx)));
@@ -508,9 +371,7 @@ const ADVISORIES_SOURCE = {
         }
       }
     }
-    // Deduplicate by CVE-ID across feeds — multiple advisories for the
-    // same CVE collapse to one entry with sources[] array of contributing
-    // feed names.
+    // Collapse per CVE across feeds; sources[] lists every contributing feed.
     const byCve = new Map();
     for (const d of allDiffs) {
       if (!byCve.has(d.id)) {
@@ -526,18 +387,10 @@ const ADVISORIES_SOURCE = {
       delete d.advisory_url;
       return d;
     });
-    // v0.13.17: observations are the FULL per-feed extraction list,
-    // including in-catalog CVE-IDs. NEW-CTRL-074 cve-regression-watcher
-    // consumes this so an "annotate" verdict can fire on historical IDs
-    // (CVE-2020-17103) that are already keys in the catalog — diffs[]
-    // alone would have filtered them out and broken the watcher's
-    // annotate path. Dedupe + sources[] follow the diffs[] pattern so
-    // downstream consumers see one record per CVE with all observing
-    // feed names listed.
+    // Deduped like diffs[], but keeping in-catalog IDs for the annotate path.
     const obsByCve = new Map();
     for (const o of allObservations) {
-      // The github-events handle-drop diffs use a HANDLE:* id shape and
-      // are not CVE observations — skip them.
+      // HANDLE:* ids are handle drops, not CVE observations.
       if (!/^CVE-/.test(o.id)) continue;
       if (!obsByCve.has(o.id)) {
         obsByCve.set(o.id, {
@@ -554,9 +407,7 @@ const ADVISORIES_SOURCE = {
       }
     }
     const observations = Array.from(obsByCve.values());
-    // Status ladder folds reachable-but-unparsable into 'partial'. The integer
-    // `errors` field stays the unreachable count (downstream refresh math keys
-    // off it); parse errors are surfaced via the separate parse_errors channel.
+    // Reachable-but-unparsable folds into 'partial'; `errors` stays unreachable-only.
     const status =
       unreachable === FEEDS.length ? 'unreachable' :
       (unreachable > 0 || parseErrorFeeds > 0) ? 'partial' :
@@ -573,8 +424,6 @@ const ADVISORIES_SOURCE = {
       summary,
     };
   },
-  // Report-only: no applyDiff. Operators route promising CVE IDs through
-  // `exceptd refresh --advisory <CVE-ID>` (GHSA / OSV / NVD enrichment).
   applyDiff(_ctx, _diffs) {
     return {
       updated: 0,