@blamejs/exceptd-skills 0.12.13 → 0.12.15

package/lib/source-ghsa.js

@@ -26,6 +26,7 @@
 
 const https = require("https");
 const fs = require("fs");
+const { withRetry } = require("../vendor/blamejs/retry.js");
 
 const GHSA_HOST = "api.github.com";
 const GHSA_PATH = "/advisories?per_page=50&type=reviewed&sort=published&direction=desc";
@@ -33,40 +34,93 @@ const REQUEST_TIMEOUT_MS = 10000;
 const USER_AGENT = "exceptd-security/source-ghsa (+https://exceptd.com)";
 
 /**
- * Fetch a page of advisories (default: latest 50).
+ * Field-dropped watch set — fields the buildDiff regression-detector
+ * watches when the upstream still has an entry but a previously-populated
+ * local value has gone null. Mirrors lib/source-osv.js. Finding 9.
+ */
+const FIELD_DROPPED_WATCH = Object.freeze([
+  "cvss_score",
+  "cisa_kev_pending",
+  "active_exploitation",
+  "ai_discovered",
+  "poc_available",
+]);
+
+/**
+ * Return true when the runtime context requests air-gap mode. Sources MUST
+ * refuse network calls when this is set — fall through to fixture or return
+ * a structured `air-gap: no fixture available` error so the operator sees
+ * an explicit refusal, not a silent network attempt. Mirrors source-osv.
+ */
+function isAirGap(opts) {
+  if (opts && opts.airGap) return true;
+  if (process.env.EXCEPTD_AIR_GAP === "1") return true;
+  return false;
+}
+
+/**
+ * Read EXCEPTD_GHSA_FIXTURE and return a structured envelope. Finding 5:
+ * mirror the OSV-source convention so a fixture file containing `null`,
+ * a number, or a string at its root doesn't slip through as an empty
+ * advisories array — the strict catalog validator would later swallow the
+ * resulting drift as "no advisories returned" instead of surfacing it as
+ * a fixture configuration error. Returns:
  *
- * Returns:
- *   { ok: true,  advisories: [...], source: "github-api" | "fixture", rate_limit?: { remaining, reset } }
- *   { ok: false, error, source: "offline" }
+ *   null                                          when env var is unset
+ *   { ok: true, advisories: [...], source }       on success
+ *   { ok: false, error, source: "offline" }       on any failure
  */
-async function fetchAdvisories({ timeoutMs = REQUEST_TIMEOUT_MS, path = GHSA_PATH, token = null } = {}) {
-  if (process.env.EXCEPTD_GHSA_FIXTURE) {
-    try {
-      const arr = JSON.parse(fs.readFileSync(process.env.EXCEPTD_GHSA_FIXTURE, "utf8"));
-      return { ok: true, advisories: Array.isArray(arr) ? arr : [arr], source: "fixture" };
-    } catch (e) {
-      return { ok: false, error: `fixture: ${e.message}`, source: "offline" };
-    }
+function readFixture() {
+  const fp = process.env.EXCEPTD_GHSA_FIXTURE;
+  if (!fp) return null;
+  let raw;
+  try {
+    raw = fs.readFileSync(fp, "utf8");
+  } catch (e) {
+    return { ok: false, error: `fixture: ${e.message}`, source: "offline" };
+  }
+  let parsed;
+  try {
+    parsed = JSON.parse(raw);
+  } catch (e) {
+    return { ok: false, error: `fixture: ${e.message}`, source: "offline" };
   }
+  if (parsed == null || typeof parsed !== "object") {
+    return { ok: false, error: `fixture: invalid root shape (got ${typeof parsed}); expected GHSA advisory object or array`, source: "offline" };
+  }
+  return { ok: true, advisories: Array.isArray(parsed) ? parsed : [parsed], source: "fixture" };
+}
 
-  return new Promise((resolve) => {
-    const headers = {
-      "Accept": "application/vnd.github+json",
-      "User-Agent": USER_AGENT,
-      "X-GitHub-Api-Version": "2022-11-28",
-    };
-    if (token || process.env.GITHUB_TOKEN) {
-      headers.Authorization = `Bearer ${token || process.env.GITHUB_TOKEN}`;
-    }
+/**
+ * One HTTPS GET against api.github.com. Throws on retryable conditions so
+ * withRetry's default classifier (HTTP 408/425/429/5xx + ECONNRESET et al)
+ * picks them up; resolves to a structured envelope on permanent conditions.
+ */
+function ghsaRequestOnce({ path, headers, timeoutMs }) {
+  return new Promise((resolve, reject) => {
     const req = https.get({
       host: GHSA_HOST,
       path,
       headers,
       timeout: timeoutMs,
     }, (res) => {
-      if (res.statusCode !== 200) {
+      const status = res.statusCode;
+      if (status === 429 || status === 503 ||
+          (status >= 500 && status <= 599) ||
+          status === 408 || status === 425) {
         res.resume();
-        return resolve({ ok: false, error: `GHSA returned HTTP ${res.statusCode}`, source: "offline" });
+        const err = new Error(`GHSA returned HTTP ${status}`);
+        err.statusCode = status;
+        const ra = res.headers["retry-after"];
+        if (ra) {
+          const secs = parseInt(ra, 10);
+          if (Number.isFinite(secs)) err.retryAfterMs = secs * 1000;
+        }
+        return reject(err);
+      }
+      if (status !== 200) {
+        res.resume();
+        return resolve({ ok: false, error: `GHSA returned HTTP ${status}`, source: "offline" });
       }
       const chunks = [];
       res.on("data", (c) => chunks.push(c));
@@ -88,11 +142,60 @@ async function fetchAdvisories({ timeoutMs = REQUEST_TIMEOUT_MS, path = GHSA_PAT
         }
       });
     });
-    req.on("timeout", () => req.destroy(new Error("timeout")));
-    req.on("error", (e) => resolve({ ok: false, error: e.message, source: "offline" }));
+    req.on("timeout", () => {
+      const err = new Error("timeout");
+      err.code = "ETIMEDOUT";
+      req.destroy(err);
+    });
+    req.on("error", (e) => {
+      if (e && e.code && /^(ECONNRESET|ECONNREFUSED|ECONNABORTED|ETIMEDOUT|EPIPE|EAGAIN|ENOTFOUND|ENETUNREACH)$/.test(e.code)) {
+        return reject(e);
+      }
+      resolve({ ok: false, error: e.message, source: "offline" });
+    });
   });
 }
 
+/**
+ * Fetch a page of advisories (default: latest 50). Wraps the underlying
+ * HTTPS request in withRetry so transient 429/503/5xx + net errors back off
+ * automatically.
+ *
+ * Returns:
+ *   { ok: true,  advisories: [...], source: "github-api" | "fixture", rate_limit?: { remaining, reset } }
+ *   { ok: false, error, source: "offline" }
+ */
+async function fetchAdvisories({ timeoutMs = REQUEST_TIMEOUT_MS, path = GHSA_PATH, token = null, airGap = false } = {}) {
+  const fixture = readFixture();
+  if (fixture) return fixture;
+  // Finding 7: air-gap hard-refuses network when no fixture is configured.
+  if (isAirGap({ airGap })) {
+    return { ok: false, error: "air-gap: no fixture available (set EXCEPTD_GHSA_FIXTURE)", source: "offline" };
+  }
+  const headers = {
+    "Accept": "application/vnd.github+json",
+    "User-Agent": USER_AGENT,
+    "X-GitHub-Api-Version": "2022-11-28",
+  };
+  if (token || process.env.GITHUB_TOKEN) {
+    headers.Authorization = `Bearer ${token || process.env.GITHUB_TOKEN}`;
+  }
+  try {
+    return await withRetry(() => ghsaRequestOnce({ path, headers, timeoutMs }), {
+      maxAttempts: 3,
+      baseDelayMs: 100,
+      maxDelayMs: 2000,
+      jitterFactor: 0.5,
+    });
+  } catch (e) {
+    const status = typeof e?.statusCode === "number" ? e.statusCode : null;
+    const error = status
+      ? `GHSA returned HTTP ${status}`
+      : `GHSA request failed: ${e.message || e}`;
+    return { ok: false, error, status, source: "offline" };
+  }
+}
+
 /**
  * Fetch a single advisory by ID — accepts CVE-* or GHSA-* identifiers.
  *
@@ -103,12 +206,18 @@ async function fetchAdvisoryById(id, opts = {}) {
   if (!id || typeof id !== "string") {
     return { ok: false, error: "id is required (CVE-* or GHSA-*)", source: "offline" };
   }
+  // Finding 8: trim whitespace at the entry seam.
+  id = id.trim();
+  if (!id) {
+    return { ok: false, error: "id is required (CVE-* or GHSA-*)", source: "offline" };
+  }
   if (process.env.EXCEPTD_GHSA_FIXTURE) {
     const r = await fetchAdvisories(opts);
     if (!r.ok) return r;
+    const want = id.toUpperCase();
     const match = r.advisories.find(a =>
-      (a.ghsa_id && a.ghsa_id.toUpperCase() === id.toUpperCase()) ||
-      (a.cve_id && a.cve_id.toUpperCase() === id.toUpperCase())
+      (a.ghsa_id && String(a.ghsa_id).toUpperCase() === want) ||
+      (a.cve_id && String(a.cve_id).toUpperCase() === want)
     );
     if (!match) return { ok: false, error: `${id} not in fixture`, source: "fixture" };
     return { ok: true, advisories: [match], source: "fixture" };
@@ -128,6 +237,23 @@ async function fetchAdvisoryById(id, opts = {}) {
   return { ok: false, error: `unrecognized id format: ${id}. Expected one of: CVE-YYYY-NNNN, GHSA-* (routed through source-ghsa); MAL-* / SNYK-* / RUSTSEC-* / USN-* / PYSEC-* / GO-* / MGASA-* / UVI- (routed through source-osv).`, source: "offline" };
 }
 
+/**
+ * Validate + slice a published_at timestamp. Findings 2 + 17:
+ *  - typeof guard so non-string inputs (number, object, undefined) become
+ *    null instead of throwing on .slice().
+ *  - ISO-prefix + year sanity bound so garbage timestamps don't pollute
+ *    downstream surfaces.
+ */
+function safeDateSlice(value) {
+  if (typeof value !== "string") return null;
+  const head = value.slice(0, 10);
+  if (!/^\d{4}-\d{2}-\d{2}$/.test(head)) return null;
+  const year = parseInt(head.slice(0, 4), 10);
+  const now = new Date().getUTCFullYear();
+  if (!Number.isFinite(year) || year < 1990 || year > now + 1) return null;
+  return head;
+}
+
 /**
  * Normalize a GHSA advisory object to the exceptd catalog draft shape.
  * Fields the GHSA carries authoritatively: cve_id, ghsa_id, summary,
@@ -146,7 +272,9 @@ function normalizeAdvisory(adv) {
   const ecosystems = new Set();
   const affected = [];
   const ecosystemPackages = [];
-  for (const v of (adv.vulnerabilities || [])) {
+  // Finding 3: vulnerabilities may not be an array — guard before iterating.
+  const vulnList = Array.isArray(adv.vulnerabilities) ? adv.vulnerabilities : [];
+  for (const v of vulnList) {
     if (v?.package?.ecosystem) ecosystems.add(v.package.ecosystem);
     if (v?.package?.name) {
       ecosystemPackages.push(`${v.package.ecosystem || "?"}:${v.package.name}`);
@@ -156,7 +284,14 @@ function normalizeAdvisory(adv) {
     }
   }
 
-  const cvssScore = adv.cvss?.score ?? null;
+  // Finding 4: cvss.score may arrive as a string ("9.8") rather than a
+  // number. Number-coerce + finite-check so the catalog field stays
+  // numeric across upstream shape drift.
+  let cvssScore = null;
+  if (adv.cvss != null && adv.cvss.score !== undefined && adv.cvss.score !== null) {
+    const n = Number(adv.cvss.score);
+    cvssScore = Number.isFinite(n) ? n : null;
+  }
   const cvssVector = adv.cvss?.vector_string || null;
   const severity = (adv.severity || "").toLowerCase();
   // Derive a coarse type from package ecosystem when nothing better available.
@@ -166,6 +301,13 @@ function normalizeAdvisory(adv) {
     : ecosystems.has("rubygems") ? "supply-chain-gem"
     : "supply-chain-other";
 
+  // Finding 2 + 17: type-safe + format-validated published_at slicing.
+  const publishedDate = safeDateSlice(adv.published_at);
+
+  // Finding 20: references may not be an array — guard the spread before
+  // it silently truncates to an empty list.
+  const refList = Array.isArray(adv.references) ? adv.references : [];
+
   return {
     [adv.cve_id]: {
       name: adv.summary || adv.cve_id,
@@ -205,7 +347,7 @@ function normalizeAdvisory(adv) {
       verification_sources: [
         ...(adv.html_url ? [adv.html_url] : []),
         ...(adv.cve_id ? [`https://nvd.nist.gov/vuln/detail/${adv.cve_id}`] : []),
-        ...(adv.references || []).slice(0, 10),
+        ...refList.slice(0, 10),
       ],
       vendor_advisories: [
         {
@@ -213,7 +355,7 @@ function normalizeAdvisory(adv) {
           advisory_id: adv.ghsa_id || null,
           url: adv.html_url || `https://github.com/advisories?query=${encodeURIComponent(adv.cve_id)}`,
           severity: severity || null,
-          published_date: (adv.published_at || "").slice(0, 10) || null,
+          published_date: publishedDate,
         },
       ],
       iocs: null,
@@ -231,19 +373,51 @@ function normalizeAdvisory(adv) {
  * Build a refresh diff for the existing refresh-external orchestrator.
  * Compares the latest 50 advisories' CVE IDs against the local catalog;
  * any CVE ID not in the catalog becomes an "add" diff.
+ *
+ * Finding 9: when the advisory is already in the catalog but a watched
+ * field has dropped from populated -> null, surface a `field_dropped`
+ * diff so curators don't silently lose signal.
+ *
+ * Finding 18: count + surface GHSA-only advisories (no CVE id) that were
+ * skipped, so the summary explains why N upstream advisories produced
+ * fewer than N diffs.
  */
 async function buildDiff(ctx) {
-  const result = await fetchAdvisories({});
+  const result = await fetchAdvisories({ airGap: ctx?.airGap });
   if (!result.ok) {
     return { status: "unreachable", diffs: [], errors: 1, summary: `GHSA fetch failed: ${result.error}` };
   }
-  const existing = new Set(Object.keys(ctx.cveCatalog || {}).filter(k => /^CVE-/.test(k)));
+  const cveCatalog = ctx.cveCatalog || {};
+  const existing = new Set(Object.keys(cveCatalog).filter(k => /^CVE-/.test(k)));
   const diffs = [];
+  let ghsaOnlySkipped = 0;
   for (const adv of result.advisories) {
-    if (!adv.cve_id) continue;
-    if (existing.has(adv.cve_id)) continue;
+    if (!adv.cve_id) { ghsaOnlySkipped++; continue; }
     const normalized = normalizeAdvisory(adv);
     if (!normalized) continue;
+    if (existing.has(adv.cve_id)) {
+      // Finding 9: field-dropped detection on the existing entry.
+      const before = cveCatalog[adv.cve_id] || {};
+      const after = normalized[adv.cve_id];
+      for (const field of FIELD_DROPPED_WATCH) {
+        const had = before[field];
+        const has = after[field];
+        const wasPopulated = had !== null && had !== undefined && had !== "" && had !== false;
+        const isNowEmpty = has === null || has === undefined;
+        if (wasPopulated && isNowEmpty) {
+          diffs.push({
+            id: adv.cve_id,
+            field,
+            before: had,
+            after: null,
+            severity: null,
+            source: "ghsa",
+            variant: "field_dropped",
+          });
+        }
+      }
+      continue;
+    }
     diffs.push({
       id: adv.cve_id,
       field: "_new_entry",
@@ -257,9 +431,17 @@ async function buildDiff(ctx) {
     status: "ok",
     diffs,
     errors: 0,
-    summary: `GHSA returned ${result.advisories.length} reviewed advisories; ${diffs.length} new CVE ID(s) not yet in local catalog.`,
+    ghsa_only_skipped: ghsaOnlySkipped,
+    summary: `GHSA returned ${result.advisories.length} reviewed advisories; ${diffs.length} new CVE ID(s) not yet in local catalog, ${ghsaOnlySkipped} ghsa_only_skipped.`,
     rate_limit: result.rate_limit || null,
   };
 }
 
-module.exports = { fetchAdvisories, fetchAdvisoryById, normalizeAdvisory, buildDiff };
+module.exports = {
+  fetchAdvisories,
+  fetchAdvisoryById,
+  normalizeAdvisory,
+  buildDiff,
+  FIELD_DROPPED_WATCH,
+  safeDateSlice,
+};