muaddib-scanner 2.11.67 → 2.11.68

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "muaddib-scanner",
-  "version": "2.11.67",
+  "version": "2.11.68",
   "description": "Supply-chain threat detection & response for npm & PyPI/Python",
   "main": "src/index.js",
   "bin": {

package/{self-scan-v2.11.67.json → self-scan-v2.11.68.json}

@@ -1,6 +1,6 @@
 {
   "target": "node_modules",
-  "timestamp": "2026-06-07T12:54:23.816Z",
+  "timestamp": "2026-06-07T13:41:08.649Z",
   "threats": [
     {
       "type": "string_mutation_obfuscation",

package/src/monitor/state.js

@@ -1059,6 +1059,116 @@ function loadScanLedger() {
   return entries;
 }
 
+// Bounded distinct-key tracking for the `vanished` cross-reference (CLAUDE.md §2).
+// Sits above the MAX_SCAN_LEDGER file ceiling so it is a pure safety valve: in normal
+// operation the in-window key sets are far smaller than the file, so `exactVanished`
+// stays true. Only an operator setting MUADDIB_SCAN_LEDGER_MAX above this would trip it.
+const MAX_ROLLUP_KEYS = 1_200_000;
+
+/**
+ * Phase 0b: roll up the per-scan ledger into operational-coverage metrics.
+ *
+ * Single streaming pass (never loads the whole file at once — same machinery as
+ * getDetectionStats). It distinguishes:
+ *   - scanned : entries that reached a real verdict (outcome !== 'dropped')
+ *   - dropped : queue-cap evictions (outcome === 'dropped') — never scanned
+ *   - vanished: DISTINCT name@version that were dropped AND never (re)scanned in-window
+ *               = a permanent coverage hole (the "which Miasma versions never ran" case)
+ *
+ * HONEST METRIC NOTE — `alertRate` is (suspect+confirmed) / scanned, i.e. "of what we
+ * scanned, the fraction we flagged". It is NOT a true-positive rate: the ledger carries
+ * no ground truth. The GHSA-denominated operational TPR (the 105/429 audit number) needs
+ * the ledger cross-referenced against the GHSA malware feed — that is the Phase 5
+ * coverage-audit, not this rollup. Do not relabel `alertRate` as TPR (CLAUDE.md: pas
+ * d'embellissement des métriques).
+ *
+ * @param {number|string|null} [sinceTs] window start — ms epoch, ISO string, or null for
+ *        "whole ledger". Entries with ts < sinceTs (or unparseable ts) are skipped.
+ * @param {object} [opts]
+ * @param {string} [opts.file] ledger path override (tests). Defaults to SCAN_LEDGER_FILE.
+ * @returns {{
+ *   generatedAt:string, since:string|null, windowStart:string|null, windowEnd:string|null,
+ *   total:number, scanned:number, dropped:number, vanished:number, exactVanished:boolean,
+ *   alerted:number, alertRate:number|null,
+ *   byOutcome:Object.<string,number>,
+ *   byEcosystem:Object.<string,{total:number,scanned:number,dropped:number,alerted:number}>
+ * }}
+ */
+function computeLedgerRollup(sinceTs, opts = {}) {
+  const file = opts.file || SCAN_LEDGER_FILE;
+
+  let sinceMs = null;
+  if (typeof sinceTs === 'number' && Number.isFinite(sinceTs)) {
+    sinceMs = sinceTs;
+  } else if (typeof sinceTs === 'string') {
+    const p = Date.parse(sinceTs);
+    if (!Number.isNaN(p)) sinceMs = p;
+  }
+
+  const byOutcome = Object.create(null);
+  const byEcosystem = Object.create(null);
+  let total = 0, scanned = 0, dropped = 0, alerted = 0;
+  let earliest = null, latest = null;
+  // Two sets so `vanished` is correct regardless of drop/scan ordering in the file.
+  // droppedKeys is small (drops only happen under queue-cap pressure); scannedKeys is
+  // bounded by the in-window line count (≤ MAX_SCAN_LEDGER), and further by MAX_ROLLUP_KEYS.
+  const scannedKeys = new Set();
+  const droppedKeys = new Set();
+  let exactVanished = true;
+
+  _iterateJsonlSync(file, (e) => {
+    if (!e || !e.name) return;
+    let t = null;
+    if (e.ts) { const p = Date.parse(e.ts); if (!Number.isNaN(p)) t = p; }
+    if (sinceMs !== null && (t === null || t < sinceMs)) return;
+
+    total++;
+    if (t !== null) {
+      if (earliest === null || t < earliest) earliest = t;
+      if (latest === null || t > latest) latest = t;
+    }
+
+    const outcome = (typeof e.outcome === 'string' && e.outcome) ? e.outcome : 'clean';
+    byOutcome[outcome] = (byOutcome[outcome] || 0) + 1;
+
+    const eco = e.ecosystem || 'unknown';
+    let ecoNode = byEcosystem[eco];
+    if (!ecoNode) ecoNode = byEcosystem[eco] = { total: 0, scanned: 0, dropped: 0, alerted: 0 };
+    ecoNode.total++;
+
+    const key = `${e.name}@${e.version || ''}`;
+    const underCap = exactVanished && (scannedKeys.size + droppedKeys.size) < MAX_ROLLUP_KEYS;
+    if (outcome === 'dropped') {
+      dropped++; ecoNode.dropped++;
+      if (underCap) droppedKeys.add(key); else exactVanished = false;
+    } else {
+      scanned++; ecoNode.scanned++;
+      if (outcome === 'suspect' || outcome === 'confirmed') { alerted++; ecoNode.alerted++; }
+      if (underCap) scannedKeys.add(key); else exactVanished = false;
+    }
+  });
+
+  let vanished = 0;
+  for (const k of droppedKeys) { if (!scannedKeys.has(k)) vanished++; }
+
+  return {
+    generatedAt: new Date().toISOString(),
+    since: sinceMs !== null ? new Date(sinceMs).toISOString() : null,
+    windowStart: earliest !== null ? new Date(earliest).toISOString() : null,
+    windowEnd: latest !== null ? new Date(latest).toISOString() : null,
+    total,
+    scanned,
+    dropped,
+    vanished,
+    exactVanished,
+    alerted,
+    // NOT a TPR — see the HONEST METRIC NOTE above. null when nothing was scanned.
+    alertRate: scanned > 0 ? alerted / scanned : null,
+    byOutcome,
+    byEcosystem
+  };
+}
+
 // --- Scan stats (FP rate tracking) ---
 
 function loadScanStats() {
@@ -1568,6 +1678,7 @@ module.exports = {
   appendDetection,
   appendScanLedger,
   loadScanLedger,
+  computeLedgerRollup,
   _compactScanLedgerJsonl,
   getDetectionStats,
   runStateMigrations,

package/src/monitor/webhook.js

@@ -28,7 +28,8 @@ const {
   saveState,
   loadStateRaw,
   getScansSinceLastMemoryPersist,
-  setScansSinceLastMemoryPersist
+  setScansSinceLastMemoryPersist,
+  computeLedgerRollup
 } = require('./state.js');
 const {
   HIGH_CONFIDENCE_MALICE_TYPES,
@@ -897,7 +898,52 @@ function formatDelta(current, previous) {
   return '=0';
 }
 
-function buildDailyReportEmbed(stats, dailyAlerts) {
+// Phase 0b: rolling window for the daily report's ledger section. The report runs
+// once/day, so 24h is the natural "what happened today" view and keeps the rollup's
+// distinct-key sets small (one day of scans, far below MAX_ROLLUP_KEYS). Env-tunable.
+const LEDGER_ROLLUP_WINDOW_MS = (() => {
+  const v = parseInt(process.env.MUADDIB_LEDGER_ROLLUP_WINDOW_MS, 10);
+  return Number.isFinite(v) && v > 0 ? v : 24 * 60 * 60 * 1000;
+})();
+
+/**
+ * Compute the per-scan ledger rollup for the daily-report window. Best-effort: a
+ * rollup failure (corrupt ledger, I/O) must NEVER break the daily report, so this
+ * swallows errors and returns null. Also returns null when the ledger is empty so
+ * the report omits the section instead of showing a noise row of zeros.
+ */
+function safeLedgerRollup() {
+  try {
+    const rollup = computeLedgerRollup(Date.now() - LEDGER_ROLLUP_WINDOW_MS);
+    return (rollup && rollup.total > 0) ? rollup : null;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Format the ledger rollup as a Discord embed field, or null to omit it (no data).
+ * Surfaces operational scan coverage: scanned, alert rate (NOT a TPR — see
+ * computeLedgerRollup's HONEST METRIC NOTE), the dropped/vanished coverage holes,
+ * and a per-ecosystem split. Compact, well under Discord's 1024-char field limit.
+ */
+function formatLedgerField(rollup) {
+  if (!rollup || rollup.total <= 0) return null;
+  const pct = rollup.alertRate != null ? (rollup.alertRate * 100).toFixed(2) : '0.00';
+  const lines = [`Scanned ${rollup.scanned} · Alerted ${rollup.alerted} (${pct}%)`];
+  if (rollup.dropped > 0) {
+    const vanishedNote = rollup.exactVanished ? `${rollup.vanished}` : `≥${rollup.vanished}`;
+    lines.push(`Dropped ${rollup.dropped} (${vanishedNote} vanished)`);
+  }
+  const ecos = Object.keys(rollup.byEcosystem)
+    .sort((a, b) => rollup.byEcosystem[b].total - rollup.byEcosystem[a].total);
+  if (ecos.length > 0) {
+    lines.push(ecos.slice(0, 4).map(e => `${e} ${rollup.byEcosystem[e].total}`).join(' · '));
+  }
+  return { name: 'Ledger (24h)', value: lines.join('\n'), inline: false };
+}
+
+function buildDailyReportEmbed(stats, dailyAlerts, ledgerRollup) {
   // Use in-memory stats (accumulated since last reset, restored from disk on restart)
   // instead of disk-based daily entries which can undercount due to UTC/Paris date mismatch
   const { top3: diskTop3 } = buildReportFromDisk();
@@ -1000,6 +1046,12 @@ function buildDailyReportEmbed(stats, dailyAlerts) {
   } catch { /* non-fatal */ }
   const healthText = `Up ${uptimeH}h${uptimeM}m | Heap ${heapMB}MB${jsonlInfo}`;
 
+  // --- Phase 0b: per-scan ledger rollup (operational coverage) ---
+  // Caller may pass a precomputed rollup (sendDailyReport does, to persist the same
+  // numbers it displays); undefined → compute here; explicit null → omit the section.
+  const ledger = ledgerRollup !== undefined ? ledgerRollup : safeLedgerRollup();
+  const ledgerField = formatLedgerField(ledger);
+
   const now = new Date();
   const readableTime = now.toISOString().replace('T', ' ').replace(/\.\d+Z$/, ' UTC');
 
@@ -1022,6 +1074,7 @@ function buildDailyReportEmbed(stats, dailyAlerts) {
           ? [{ name: 'Deferred Sandbox', value: `Enqueued: ${stats.sandboxDeferred || 0} | Processed: ${stats.deferredProcessed || 0} | Expired: ${stats.deferredExpired || 0}`, inline: false }]
           : []),
         { name: 'Stability', value: `Restarts (24h): ${stats.restartsToday || 0} | Temporal load-shed: ${stats.temporalLoadShed || 0} | Queue hard-drops: ${stats.queueHardDrops || 0}`, inline: false },
+        ...(ledgerField ? [ledgerField] : []),
         { name: 'System', value: healthText, inline: false }
       ],
       footer: {
@@ -1060,7 +1113,10 @@ async function sendDailyReport(stats, dailyAlerts, recentlyScanned, downloadsCac
   // delta. Written before the (now last) webhook so a mid-send kill can't double-count.
   saveLastDailyReportDate(today, captureScanStatsBaseline());
 
-  const payload = buildDailyReportEmbed(stats, dailyAlerts);
+  // Phase 0b: compute the ledger rollup ONCE so the embed shows exactly the numbers
+  // we persist (no double-scan, no drift between Discord and the on-disk metrics).
+  const ledgerRollup = safeLedgerRollup();
+  const payload = buildDailyReportEmbed(stats, dailyAlerts, ledgerRollup);
 
   // Persist locally with full raw metrics (independent of webhook — enables trend analysis)
   persistDailyReport(payload, {
@@ -1081,6 +1137,7 @@ async function sendDailyReport(stats, dailyAlerts, recentlyScanned, downloadsCac
     restartsToday: stats.restartsToday || 0,
     temporalLoadShed: stats.temporalLoadShed || 0,
     queueHardDrops: stats.queueHardDrops || 0,
+    ledger: ledgerRollup || null,
     topSuspects: dailyAlerts.slice().sort((a, b) => (b.score || 0) - (a.score || 0) || b.findingsCount - a.findingsCount).slice(0, 10)
   });
 
@@ -1337,6 +1394,7 @@ module.exports = {
   buildMaintainerChangeWebhookEmbed,
   buildCanaryExfiltrationWebhookEmbed,
   buildDailyReportEmbed,
+  formatLedgerField,
   sendDailyReport,
   buildReportFromDisk,
   buildReportEmbedFromDisk,