muaddib-scanner 2.11.119 → 2.11.120

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "muaddib-scanner",
-  "version": "2.11.119",
+  "version": "2.11.120",
   "description": "Supply-chain threat detection & response for npm & PyPI/Python",
   "main": "src/index.js",
   "bin": {

package/{self-scan-v2.11.119.json → self-scan-v2.11.120.json}

@@ -1,6 +1,6 @@
 {
   "target": "node_modules",
-  "timestamp": "2026-06-16T08:29:32.212Z",
+  "timestamp": "2026-06-18T19:56:37.339Z",
   "threats": [
     {
       "type": "string_mutation_obfuscation",

package/src/monitor/daemon.js

@@ -13,6 +13,7 @@ const { poll, getPollBackoffMs, SOFT_BACKPRESSURE_THRESHOLD } = require('./inges
 const { ensureWorkers, drainWorkers, getTargetConcurrency, setTargetConcurrency, getActiveWorkers, terminateAllWorkers, getInFlightItems, computeInterruptDisposition } = require('./queue.js');
 const { computeTarget, ADJUST_INTERVAL_MS, BASE_CONCURRENCY } = require('./adaptive-concurrency.js');
 const { startHealthcheck } = require('./healthcheck.js');
+const { startLagSampler } = require('./event-loop-monitor.js');
 const { startDeferredWorker, stopDeferredWorker, persistDeferredQueue, restoreDeferredQueue, clearDeferredQueue } = require('./deferred-sandbox.js');
 const { evictFromScanQueueBulk, enqueueScan } = require('./scan-queue.js');
 const { isSpillEnabled, shouldDrain, drainBacklog, getBacklogSize } = require('./spill.js');
@@ -966,6 +967,7 @@ async function startMonitor(options, stats, dailyAlerts, recentlyScanned, downlo
   let pollIntervalHandle = null;   // Decoupled poll timer — set after initial poll
   let queuePersistHandle = null;   // Queue persistence timer
   let concurrencyAdjustHandle = null; // Adaptive concurrency timer
+  let loopLagSamplerStop = null;   // Event-loop stall attribution (instrumentation)
 
   // Restore queue from previous run (if file exists and is < 24h old)
   const restoredCount = restoreQueue(scanQueue);
@@ -1001,6 +1003,10 @@ async function startMonitor(options, stats, dailyAlerts, recentlyScanned, downlo
       clearInterval(concurrencyAdjustHandle);
       concurrencyAdjustHandle = null;
     }
+    if (loopLagSamplerStop) {
+      loopLagSamplerStop();
+      loopLagSamplerStop = null;
+    }
     // Bounded drain (phase C, C7). The old unbounded `await drainWorkers()`
     // could outlive systemd's TimeoutStopSec (scans run up to 300s): SIGKILL
     // then landed MID-drain, persistQueue never ran, and every in-flight scan
@@ -1109,6 +1115,24 @@ async function startMonitor(options, stats, dailyAlerts, recentlyScanned, downlo
     }
   }, ADJUST_INTERVAL_MS);
 
+  // ─── Event-loop stall attribution (instrumentation only — see event-loop-monitor.js) ───
+  // The RSS breaker, governor RSS feed and EMERGENCY purge all live on these
+  // main-thread timers; a long SYNC op (extractAllTo) that wedges the loop
+  // disables them silently → unchecked RSS climb → cgroup OOM (4-6 min of zero
+  // completions/logs before every kill, 2026-06-17/18). NAME the culprit op so the
+  // real fix targets it. Off-switch: MUADDIB_LOOP_MONITOR=0.
+  if (process.env.MUADDIB_LOOP_MONITOR !== '0') {
+    loopLagSamplerStop = startLagSampler({
+      onStall: (r) => {
+        const o = r.op;
+        const where = o
+          ? ` during op=${o.label}${o.meta && o.meta.name ? ` ${o.meta.name}@${o.meta.version || '?'}${o.meta.unpackedSizeMb != null ? ` (${o.meta.unpackedSizeMb}MB)` : ''}` : ''}${o.running ? ', STILL RUNNING' : ''}`
+          : ' (no op breadcrumb — widen instrumentation)';
+        console.warn(`[MONITOR] LOOP-STALL: event loop blocked ${r.blockedSec}s${where} | rss=${r.rssMb}MB`);
+      }
+    });
+  }
+
   // ─── Decoupled polling ───
   // Poll runs on its own interval, independent of processing.
   // This ensures new packages are ingested even while a large batch is being scanned.

package/src/monitor/event-loop-monitor.js

@@ -0,0 +1,186 @@
+'use strict';
+
+/**
+ * Event-loop stall ATTRIBUTION (2026-06-18). Pure observability — NO change to
+ * scan behavior, scoring, or detection.
+ *
+ * Why: every in-process OOM defense — the RSS circuit breaker, the memory
+ * governor's RSS feed (`updateGovernorRss`), the EMERGENCY queue purge — runs on
+ * the main-thread `setInterval` poll loop (`daemon.js`). A long SYNCHRONOUS op on
+ * that loop wedges it: none of those timers fire, so RSS climbs to the cgroup
+ * MemoryMax (9.5G) unchecked → kernel SIGKILL (cgroup OOM). Measured signature
+ * 2026-06-17/18: 4-6 min of ZERO scan completions AND zero daemon log lines
+ * immediately before EVERY OOM kill. Leading suspect for the block: adm-zip
+ * `extractAllTo` (synchronous), which `extractArchive` runs on the main thread for
+ * the large-package quick-scan (4071 size_skip/24h) and pre-worker extraction.
+ *
+ * This module does NOT fix the stall — it NAMES it, so the next refactor targets
+ * the confirmed culprit instead of a guess:
+ *   - `beginOp`/`endOp` leave a breadcrumb naming the op (and package) in flight.
+ *   - `startLagSampler` runs its OWN low-frequency timer; its tardiness measures
+ *     how long the loop was blocked (the sampler is starved the same way the
+ *     breaker is, but decoupled so it can't perturb the breaker). On a stall it
+ *     reports the op whose lifetime OVERLAPS the blocked window — the still-running
+ *     op, or the one that just ended (the blocking op has usually returned by the
+ *     time the loop frees and the sampler can finally fire).
+ * Resolving stalls share their source with the fatal one, so attributing the
+ * resolving precursors pins the culprit.
+ *
+ * Bounded (CLAUDE.md §2): a single current + single last-ended breadcrumb slot;
+ * the stall log is size-capped. Instrumentation NEVER throws.
+ */
+
+const fs = require('fs');
+const path = require('path');
+
+const STALL_LOG_FILE = process.env.MUADDIB_LOOP_STALL_FILE
+  || path.join(__dirname, '..', '..', 'data', 'loop-stalls.jsonl');
+const STALL_LOG_MAX_BYTES = 256 * 1024; // bounded: truncate-then-append past this
+
+// Threshold 5s: far above normal GC/IO jitter (sub-100ms) and a full scan's
+// main-thread slice, far below the multi-minute fatal stalls — catches the
+// resolving precursors without noise. Sampler cadence 1s.
+const DEFAULT_INTERVAL_MS = 1000;
+const DEFAULT_THRESHOLD_MS = 5000;
+
+// Injectable clock (tests drive it via _reset(clockFn)); default real time.
+let _clock = () => Date.now();
+function _now() { return _clock(); }
+
+// ─── Breadcrumb: the op currently / most-recently on the main thread ───
+let _current = null;   // { label, meta, startedAt }
+let _lastEnded = null; // { label, meta, startedAt, endedAt }
+
+/** Mark the start of a (potentially blocking, synchronous) main-thread op. */
+function beginOp(label, meta) {
+  _current = { label: String(label || 'op'), meta: meta || null, startedAt: _now() };
+  return _current;
+}
+
+/** Mark it done. Token optional; a mismatched token is ignored (nesting-safe). */
+function endOp(token) {
+  if (!_current) return;
+  if (token && token !== _current) return;
+  _lastEnded = { ..._current, endedAt: _now() };
+  _current = null;
+}
+
+/**
+ * The op whose lifetime overlaps [windowStartMs, windowEndMs] — i.e. the op on
+ * the thread during the blocked window. Prefers the still-running op, else the
+ * one that ended inside the window. Null when nothing was instrumented (a real
+ * signal: widen the breadcrumbs).
+ */
+function opOverlapping(windowStartMs, windowEndMs) {
+  if (_current && _current.startedAt <= windowEndMs) {
+    return { label: _current.label, meta: _current.meta, running: true, elapsedMs: _now() - _current.startedAt };
+  }
+  if (_lastEnded && _lastEnded.endedAt >= windowStartMs && _lastEnded.startedAt <= windowEndMs) {
+    return { label: _lastEnded.label, meta: _lastEnded.meta, running: false, durationMs: _lastEnded.endedAt - _lastEnded.startedAt };
+  }
+  return null;
+}
+
+// ─── Lag core (pure, unit-testable: timestamps in, no timers) ───
+const _state = { lastTickMs: null, intervalMs: DEFAULT_INTERVAL_MS, thresholdMs: DEFAULT_THRESHOLD_MS, maxLagMs: 0, stalls: 0 };
+
+function configure(opts = {}) {
+  if (Number.isFinite(opts.intervalMs)) _state.intervalMs = opts.intervalMs;
+  if (Number.isFinite(opts.thresholdMs)) _state.thresholdMs = opts.thresholdMs;
+}
+
+/**
+ * Feed one sampler tick. Returns { lagMs, windowStartMs, firstTick }. lagMs = how
+ * much LATER than the configured interval this tick fired = the time the loop was
+ * blocked since the previous tick. First call seeds (lag 0).
+ */
+function observeTick(nowMs) {
+  if (_state.lastTickMs === null) {
+    _state.lastTickMs = nowMs;
+    return { lagMs: 0, windowStartMs: nowMs, firstTick: true };
+  }
+  const windowStartMs = _state.lastTickMs;
+  const lagMs = Math.max(0, nowMs - _state.lastTickMs - _state.intervalMs);
+  _state.lastTickMs = nowMs;
+  if (lagMs > _state.maxLagMs) _state.maxLagMs = lagMs;
+  return { lagMs, windowStartMs, firstTick: false };
+}
+
+function isStall(lagMs, thresholdMs = _state.thresholdMs) { return lagMs >= thresholdMs; }
+function getMaxLagMs() { return _state.maxLagMs; }
+function getStallCount() { return _state.stalls; }
+
+function _appendStall(record) {
+  try {
+    try {
+      const st = fs.statSync(STALL_LOG_FILE);
+      if (st.size > STALL_LOG_MAX_BYTES) fs.truncateSync(STALL_LOG_FILE, 0); // bounded
+    } catch { /* absent — first write */ }
+    fs.appendFileSync(STALL_LOG_FILE, JSON.stringify(record) + '\n');
+  } catch { /* instrumentation must never throw */ }
+}
+
+/** Build the structured stall record for a detected lag (pure; exported for tests). */
+function buildStallRecord(lagMs, windowStartMs, nowMs) {
+  const op = opOverlapping(windowStartMs, nowMs);
+  return {
+    ts: new Date(nowMs).toISOString(),
+    lagMs,
+    blockedSec: Math.round(lagMs / 100) / 10,
+    op: op
+      ? { label: op.label, meta: op.meta, running: op.running, durationMs: op.running ? op.elapsedMs : op.durationMs }
+      : null,
+    rssMb: Math.round(process.memoryUsage().rss / 1024 / 1024)
+  };
+}
+
+/**
+ * Start the lag sampler. Its OWN tardiness measures loop lag. unref'd: a pure
+ * monitor never keeps the process alive on its own. Returns a stop fn.
+ * onStall(record) is an optional extra sink (the daemon logs it); every stall is
+ * also appended to STALL_LOG_FILE for post-hoc analysis.
+ */
+function startLagSampler(opts = {}) {
+  configure(opts);
+  _state.lastTickMs = null;
+  const onStall = typeof opts.onStall === 'function' ? opts.onStall : null;
+  const timer = setInterval(() => {
+    const now = Date.now();
+    const { lagMs, windowStartMs, firstTick } = observeTick(now);
+    if (firstTick || !isStall(lagMs)) return;
+    _state.stalls += 1;
+    const record = buildStallRecord(lagMs, windowStartMs, now);
+    _appendStall(record);
+    if (onStall) { try { onStall(record); } catch { /* ignore */ } }
+  }, _state.intervalMs);
+  if (timer && typeof timer.unref === 'function') timer.unref();
+  return () => clearInterval(timer);
+}
+
+/** Test helper: clear all state; optionally install a deterministic clock. */
+function _reset(clockFn) {
+  _current = null;
+  _lastEnded = null;
+  _state.lastTickMs = null;
+  _state.intervalMs = DEFAULT_INTERVAL_MS;
+  _state.thresholdMs = DEFAULT_THRESHOLD_MS;
+  _state.maxLagMs = 0;
+  _state.stalls = 0;
+  _clock = typeof clockFn === 'function' ? clockFn : (() => Date.now());
+}
+
+module.exports = {
+  beginOp,
+  endOp,
+  opOverlapping,
+  configure,
+  observeTick,
+  isStall,
+  getMaxLagMs,
+  getStallCount,
+  buildStallRecord,
+  startLagSampler,
+  _reset,
+  _state,
+  STALL_LOG_FILE
+};

package/src/monitor/queue.js

@@ -22,6 +22,7 @@ const { buildTrainingRecord } = require('../ml/feature-extractor.js');
 const { appendWorkerMem } = require('./worker-mem.js');
 const { acquireHeavySlot, releaseHeavySlot, isHeavyScan, getHeavyLaneState, heavyWaitMaxMs, HEAVY_REQUEUE_MAX } = require('./heavy-lane.js');
 const { isGovernorEnabled, classifyWeight, acquireMemoryTicket, releaseMemoryTicket, isFrozen: isGovernorFrozen, getGovernorState } = require('./memory-governor.js');
+const { beginOp, endOp } = require('./event-loop-monitor.js');
 const { appendRecord: appendTrainingRecord, relabelRecords } = require('../ml/jsonl-writer.js');
 
 // From ./state.js
@@ -764,7 +765,8 @@ async function scanPackage(name, version, ecosystem, tarballUrl, registryMeta, s
         // Validates actual tarball contents (not just registry metadata).
         let bypassQuickScan = false;
         try {
-          extractedDir = extractArchive(tgzPath, tmpDir);
+          const _crumb = beginOp('extract:quickscan', { name, version, unpackedSizeMb: Math.round(unpackedSize / 1024 / 1024) });
+          try { extractedDir = extractArchive(tgzPath, tmpDir); } finally { endOp(_crumb); }
 
           const [pkgThreats, shellThreats] = await Promise.all([
             scanPackageJson(extractedDir),
@@ -813,7 +815,8 @@ async function scanPackage(name, version, ecosystem, tarballUrl, registryMeta, s
     }
 
     if (!extractedDir) {
-      extractedDir = extractArchive(tgzPath, tmpDir);
+      const _crumb = beginOp('extract:prework', { name, version, unpackedSizeMb: Math.round((meta.unpackedSize || 0) / 1024 / 1024) });
+      try { extractedDir = extractArchive(tgzPath, tmpDir); } finally { endOp(_crumb); }
     }
 
     // ML Phase 2a: Count JS files and detect test presence for enriched features

package/src/monitor/webhook.js

@@ -1117,10 +1117,17 @@ function safeLedgerRollup() {
 function formatLedgerField(rollup) {
   if (!rollup || rollup.total <= 0) return null;
   const pct = rollup.alertRate != null ? (rollup.alertRate * 100).toFixed(2) : '0.00';
-  const lines = [`Scanned ${rollup.scanned} · Alerted ${rollup.alerted} (${pct}%)`];
+  // All counts here are name@version scan EVENTS (NOT distinct package names — that
+  // ratio is the headline's "Noms uniques"). alertRate = suspect+confirmed / scanned
+  // (NOT a TPR — see computeLedgerRollup's HONEST METRIC NOTE).
+  const lines = [`Scans: ${rollup.scanned} events · alertés ${rollup.alerted} (${pct}%)`];
   if (rollup.dropped > 0) {
     const vanishedNote = rollup.exactVanished ? `${rollup.vanished}` : `≥${rollup.vanished}`;
-    lines.push(`Dropped ${rollup.dropped} (${vanishedNote} vanished)`);
+    // `dropped` folds in recoverable spill (backlog awaiting drain) + queue-cap evictions
+    // + burst-extras; `vanished` is the distinct name@version subset never (re)scanned
+    // in-window — still version-granular (a name with 50 dropped versions = 50 here) and
+    // it too still includes not-yet-drained spill, so it is NOT a registry-removal count.
+    lines.push(`Non scannés: ${rollup.dropped} events (${vanishedNote} name@ver jamais (re)scannés)`);
   }
   const ecos = Object.keys(rollup.byEcosystem)
     .sort((a, b) => rollup.byEcosystem[b].total - rollup.byEcosystem[a].total);
@@ -1235,36 +1242,35 @@ function buildDailyReportEmbed(stats, dailyAlerts, ledgerRollup) {
   const pypiPub = stats.pypiChangelogPackages || 0;
   const published = npmPub + pypiPub;
   const catchupSkipped = (stats.npmCatchupSkippedSeqs || 0) + (stats.pypiCatchupSkippedEvents || 0);
-  // Clarify the Ops headline so it isn't read as an overnight drop: it counts
-  // COMPLETED scans in the exact ledger window [last report → now], version/
-  // dedup-collapsed — intentionally lower than the in-memory counter (stats.scanned),
-  // which also tallies retries, burst extras and size-cap rejections
-  // (cf. queue.js uniqueScanAttempts). Surface the raw counter when it diverges.
-  const opsQualifier = headline ? ' (completed, deduped, 24h)' : '';
+  // UNIT DISCIPLINE (2026-06-18): the Coverage field stacked three counting units
+  // with no labels, so the headline read as self-contradictory — e.g.
+  // "30K/90K pkgs · 112K vanished" looks like 112K > 90K packages (impossible), but
+  // `vanished` counts name@VERSION events while the ratio counts distinct NAMES.
+  // Each line below now states its unit; the version-granular drop/vanished detail
+  // lives in the Ops embed's Ledger field, NOT next to a name ratio.
+  //   • "Noms uniques" → distinct package NAMES (version-collapsed) — the honest headline
+  //   • "Scans"        → name@version scan events (same unit as Clean/Suspects/Errors)
+  //   • "compteur"     → in-memory stats.scanned (events + retries + burst + size-cap)
+  const opsQualifier = headline ? ' (events terminés)' : '';
   const rawCounter = (headline && typeof stats.scanned === 'number' && stats.scanned > hScanned)
-    ? ` · counter ${stats.scanned} (incl. retries/burst)`
+    ? ` · compteur ${stats.scanned} (retries/burst inclus)`
     : '';
   const opsSuffix = catchupSkipped > 0
-    ? `\nOps: ${hScanned}${opsQualifier}${rawCounter} | Catch-up skip: ${catchupSkipped}`
-    : `\nOps: ${hScanned}${opsQualifier}${rawCounter}`;
+    ? `\nScans: ${hScanned}${opsQualifier}${rawCounter} | Catch-up skip: ${catchupSkipped}`
+    : `\nScans: ${hScanned}${opsQualifier}${rawCounter}`;
   let coverageText;
   if (ledger && ledger.distinctPackages > 0 && ledger.distinctCoverage != null) {
     const pct = (ledger.distinctCoverage * 100).toFixed(0);
     const approx = ledger.exactVanished === false ? '~' : '';
-    coverageText = `${ledger.distinctScanned}/${ledger.distinctPackages} pkgs (${approx}${pct}%)`;
-    // Honest 24h coverage loss surfaced next to coverage: `vanished` = distinct names
-    // dropped and never re-scanned in the window — the real miss count. The raw
-    // `dropped` aggregate (which also folds in recoverable spill + retries, so it
-    // OVERSTATES loss) is relegated to the Ops embed's Ledger field, not the headline.
-    if (ledger.vanished > 0) {
-      coverageText += ` · ${ledger.exactVanished ? '' : '≥'}${ledger.vanished} vanished`;
-    }
-    if (published > 0) coverageText += `\nRaw events: ${attempted}/${published}`;
+    // Headline = distinct package NAMES scanned vs seen (version-collapsed, immune to
+    // version-spam). The name@version drop/vanished detail is in the Ledger field below.
+    coverageText = `Noms uniques: ${ledger.distinctScanned}/${ledger.distinctPackages} (${approx}${pct}%)`;
+    if (published > 0) coverageText += `\nPubliés: ${attempted}/${published}`;
     coverageText += opsSuffix;
   } else if (published > 0) {
     // Fallback: ledger unavailable (first boot / empty ledger) → legacy event ratio.
     const coverageRatio = (attempted / published * 100).toFixed(0);
-    coverageText = `${attempted}/${published} (${coverageRatio}%)${opsSuffix}`;
+    coverageText = `Publiés: ${attempted}/${published} (${coverageRatio}%)${opsSuffix}`;
   } else {
     coverageText = `${attempted} attempted${opsSuffix}`;
   }
@@ -1363,16 +1369,19 @@ function buildDailyReportEmbed(stats, dailyAlerts, ledgerRollup) {
       // --- Embed 2: Ops / system state (kept OUT of the daily headline) ---
       // Operator feedback: a daily that mixes 24h outcome with multi-day system state
       // reads as failure when it isn't. Each line here carries its own clock:
-      //   • Ledger      → 24h window. Its `dropped` folds in recoverable spill + retries,
-      //                   so it OVERSTATES loss — `vanished` (in the Coverage field) is the
-      //                   honest miss count, which is why dropped sits here, not the headline.
+      //   • Ledger      → 24h window, in name@version scan EVENTS (NOT package names —
+      //                   the name ratio is the headline's "Noms uniques"). `dropped` folds
+      //                   in recoverable spill (backlog awaiting drain) + queue-cap evictions
+      //                   + burst-extras; `vanished` is the distinct name@version subset never
+      //                   (re)scanned in-window — also version-granular, and it too still
+      //                   includes not-yet-drained spill, so neither is a registry-removal count.
       //   • Stability   → cumulative since the 08:00 reset (backlog = point-in-time depth
       //                   of the persistent spill file, the one snapshot in this field).
       //   • Degradations / System → instantaneous snapshot (degradations have no TTL: if
       //                   shown, the condition is active right now, not earlier in the window).
       title: '⚙️ Ops / état système',
       color: 0x95a5a6,
-      description: 'Ledger = fenêtre 24h (dropped inclut le spill récupérable — voir « vanished » pour la perte réelle) · Stability = cumulé depuis 08:00 (backlog = instantané) · Degradations/System = instantané',
+      description: 'Ledger = fenêtre 24h en events name@version (pas des noms de paquets — voir « Noms uniques » dans le headline) · « Non scannés » inclut le spill récupérable en attente de drain · Stability = cumulé depuis 08:00 (backlog = instantané) · Degradations/System = instantané',
       fields: [
         ...((stats.sandboxDeferred || stats.deferredProcessed || stats.deferredExpired)
           ? [{ name: 'Deferred Sandbox', value: `Enqueued: ${stats.sandboxDeferred || 0} | Processed: ${stats.deferredProcessed || 0} | Expired: ${stats.deferredExpired || 0}`, inline: false }]