muaddib-scanner 2.11.62 → 2.11.64

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "muaddib-scanner",
-  "version": "2.11.62",
+  "version": "2.11.64",
   "description": "Supply-chain threat detection & response for npm & PyPI/Python",
   "main": "src/index.js",
   "bin": {

package/{self-scan-v2.11.62.json → self-scan-v2.11.64.json}

@@ -1,6 +1,6 @@
 {
   "target": "node_modules",
-  "timestamp": "2026-06-05T19:51:12.540Z",
+  "timestamp": "2026-06-06T11:45:08.682Z",
   "threats": [
     {
       "type": "string_mutation_obfuscation",

package/src/monitor/daemon.js

@@ -8,7 +8,7 @@ const { setVerboseMode, isSandboxEnabled, isCanaryEnabled, isLlmDetectiveEnabled
 const { loadState, saveState, loadDailyStats, saveDailyStats, purgeTarballCache, getParisHour, atomicWriteFileSync, saveNpmSeq, ALERTS_FILE, runStateMigrations, loadRecentlyScanned, saveRecentlyScanned } = require('./state.js');
 const { isTemporalEnabled, isTemporalAstEnabled, isTemporalPublishEnabled, isTemporalMaintainerEnabled } = require('./temporal.js');
 const { pendingGrouped, flushScopeGroup, sendDailyReport, DAILY_REPORT_HOUR, alertedPackageRules, ALERTED_PACKAGES_MAX: MAX_ALERTED_PACKAGES } = require('./webhook.js');
-const { poll } = require('./ingestion.js');
+const { poll, getPollBackoffMs } = require('./ingestion.js');
 const { ensureWorkers, drainWorkers, getTargetConcurrency, setTargetConcurrency, getActiveWorkers, terminateAllWorkers } = require('./queue.js');
 const { computeTarget, ADJUST_INTERVAL_MS, BASE_CONCURRENCY } = require('./adaptive-concurrency.js');
 const { startHealthcheck } = require('./healthcheck.js');
@@ -31,6 +31,120 @@ const QUEUE_STATE_MAX_AGE_MS = 24 * 60 * 60 * 1000; // 24h expiry
 const MAX_QUEUE_PERSIST_SIZE = 200_000; // Don't persist if queue > 200K items (OOM guard)
 const MAX_RESTORE_QUEUE_SIZE = 100_000; // Cap restored queue at 100K items
 
+// ─── Poll-loop watchdog ───
+// The decoupled poll runs on a setInterval guarded by a `pollInProgress` flag.
+// If a poll cycle's awaited promise never settles (e.g. an HTTP response whose
+// body trickles forever, so the socket-inactivity timeout in httpsGet never
+// fires and it only resolves on 'end'), `pollInProgress` would stay `true` and
+// every subsequent tick would silently early-return — wedging ingestion at 0
+// scanned until a manual `systemctl restart`. The watchdog bounds every cycle
+// so the flag is ALWAYS released; shouldSkipPoll() adds a stale-flag backstop
+// for any future hang path that bypasses runPollCycle().
+const POLL_WATCHDOG_MS = Math.max(60_000, parseInt(process.env.MUADDIB_POLL_WATCHDOG_MS, 10) || 300_000);
+
+/**
+ * Run ONE poll cycle bounded by a watchdog so the caller's pollInProgress flag
+ * can never stay stuck. On timeout it REJECTS (does not resolve) with a
+ * 'poll watchdog' error, so the caller's existing catch logs it and the finally
+ * resets the flag — the next tick retries. The local timer is cleared on every
+ * settle path, so a fast poll leaves no dangling timer.
+ * @param {Function} pollFn - injectable for tests; defaults to the real poll().
+ * @returns {Promise<void>}
+ */
+async function runPollCycle(state, scanQueue, stats, watchdogMs = POLL_WATCHDOG_MS, pollFn = poll) {
+  let timer;
+  const watchdog = new Promise((_, reject) => {
+    timer = setTimeout(
+      () => reject(new Error(`poll watchdog: poll exceeded ${Math.round(watchdogMs / 1000)}s`)),
+      watchdogMs
+    );
+  });
+  try {
+    await Promise.race([pollFn(state, scanQueue, stats), watchdog]);
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+/**
+ * Decide whether the poll scheduler should skip this tick, and whether the
+ * pollInProgress flag is stale enough to force-reset. Pure — unit-testable
+ * without timers. forceReset fires only when a cycle has been "in flight" for
+ * longer than watchdogMs + one interval, i.e. a hang path that bypassed the
+ * per-cycle watchdog (runPollCycle always settles within watchdogMs).
+ * @returns {{skip: boolean, forceReset: boolean}}
+ */
+function shouldSkipPoll(pollInProgress, pollStartedAt, now, watchdogMs, interval) {
+  if (!pollInProgress) return { skip: false, forceReset: false };
+  if (now - pollStartedAt > watchdogMs + interval) return { skip: false, forceReset: true };
+  return { skip: true, forceReset: false };
+}
+
+// ─── Heap diagnostics (restart root-cause) ───
+// mem-trend shows the main-thread heap balloons to 6-7GB in the worker-starved
+// regime while documented structures sum to <1GB — i.e. ~5GB+ unaccounted. These
+// helpers localise it: a cheap always-on heap-spaces line (retention vs churn)
+// plus an OPT-IN, disk-guarded, one-shot v8 heap snapshot for dominator-tree
+// analysis. Snapshot is OFF unless MUADDIB_HEAPSNAPSHOT_MB is set (writing a
+// multi-GB snapshot blocks the event loop ~10-60s — must be deliberate).
+const HEAPSNAPSHOT_MB = parseInt(process.env.MUADDIB_HEAPSNAPSHOT_MB, 10) || 0; // 0 = disabled
+const HEAPSNAPSHOT_MIN_FREE_GB = Math.max(1, parseInt(process.env.MUADDIB_HEAPSNAPSHOT_MIN_FREE_GB, 10) || 12);
+const HEAPSNAPSHOT_DIR = process.env.MUADDIB_HEAPSNAPSHOT_DIR || path.join(__dirname, '..', '..', 'data');
+let heapSnapshotTaken = false;
+
+/**
+ * Pure decision: write a heap snapshot now? Separated from the I/O so it is
+ * unit-testable without producing a multi-GB file.
+ * @returns {{take: boolean, reason: string}}
+ */
+function shouldSnapshot(heapUsedMB, thresholdMB, alreadyTaken, freeGB, minFreeGB) {
+  if (!thresholdMB || thresholdMB <= 0) return { take: false, reason: 'disabled' };
+  if (alreadyTaken) return { take: false, reason: 'already-taken' };
+  if (heapUsedMB < thresholdMB) return { take: false, reason: 'below-threshold' };
+  if (freeGB < minFreeGB) return { take: false, reason: `low-disk(${Math.round(freeGB)}<${minFreeGB}GB)` };
+  return { take: true, reason: 'ok' };
+}
+
+/**
+ * Compact one-line summary of v8.getHeapSpaceStatistics() used sizes (MB).
+ * old_space high ⇒ retained objects (leak); large_object_space high ⇒ big
+ * strings/arrays; new_space high ⇒ allocation churn. Pure — unit-testable.
+ */
+function formatHeapSpaces(stats) {
+  return (stats || [])
+    .map(s => `${s.space_name}=${(s.space_used_size / 1024 / 1024).toFixed(0)}`)
+    .join(' ');
+}
+
+function getFreeDiskGB(dir) {
+  try {
+    const st = fs.statfsSync(dir);
+    return (st.bavail * st.bsize) / (1024 ** 3);
+  } catch {
+    return Infinity; // statfsSync unavailable (older Node) — don't block on disk
+  }
+}
+
+// Opt-in, one-shot, disk-guarded heap snapshot. BLOCKS the event loop while
+// writing (≈ size of the live heap) — only fires when explicitly enabled.
+function maybeHeapSnapshot(heapUsedMB) {
+  if (!HEAPSNAPSHOT_MB || heapSnapshotTaken || heapUsedMB < HEAPSNAPSHOT_MB) return;
+  const decision = shouldSnapshot(heapUsedMB, HEAPSNAPSHOT_MB, heapSnapshotTaken, getFreeDiskGB(HEAPSNAPSHOT_DIR), HEAPSNAPSHOT_MIN_FREE_GB);
+  if (!decision.take) {
+    console.log(`[MONITOR] HEAP-SNAPSHOT skipped: ${decision.reason} (heap=${heapUsedMB}MB)`);
+    return;
+  }
+  heapSnapshotTaken = true; // set BEFORE writing so a failed/slow write can't loop
+  const file = path.join(HEAPSNAPSHOT_DIR, `heap-${new Date().toISOString().replace(/[:.]/g, '-')}.heapsnapshot`);
+  try {
+    console.log(`[MONITOR] HEAP-SNAPSHOT writing (heap=${heapUsedMB}MB) → ${file} — blocks the event loop`);
+    v8.writeHeapSnapshot(file);
+    console.log(`[MONITOR] HEAP-SNAPSHOT written → ${file} (scp it off + open in Chrome DevTools → dominator tree)`);
+  } catch (err) {
+    console.error(`[MONITOR] HEAP-SNAPSHOT failed: ${err.message}`);
+  }
+}
+
 // ─── Memory pressure circuit breaker ───
 // Graduated response based on V8 heap usage against heap_size_limit.
 // Threat model: when GC thrashing starts (>90% heap limit), throughput drops to 0
@@ -869,11 +983,27 @@ async function startMonitor(options, stats, dailyAlerts, recentlyScanned, downlo
   // Backpressure: poll() skips when queue >= 30K or memory pressure >= CRITICAL (90%).
   // Adaptive concurrency adjusts scan throughput to match ingestion rate.
   let pollInProgress = false;
+  let pollStartedAt = 0;
+  let backoffUntil = 0;
   pollIntervalHandle = setInterval(async () => {
-    if (!running || pollInProgress) return;
+    if (!running) return;
+    // Backoff window after consecutive total-registry failures. Hoisted out of
+    // poll() (it used to `await sleep(backoff)` while holding pollInProgress, up
+    // to POLL_MAX_BACKOFF=16min) so the watchdog can stay sized to poll *work*.
+    if (Date.now() < backoffUntil) return;
+    // Skip if a cycle is already in flight, unless the flag is stale — a
+    // backstop for any hang path that bypasses runPollCycle()'s watchdog.
+    const { skip, forceReset } = shouldSkipPoll(pollInProgress, pollStartedAt, Date.now(), POLL_WATCHDOG_MS, POLL_INTERVAL);
+    if (forceReset) {
+      console.warn(`[MONITOR] Poll flag stuck for ${((Date.now() - pollStartedAt) / 1000).toFixed(0)}s — force-resetting`);
+      pollInProgress = false;
+    } else if (skip) {
+      return;
+    }
     pollInProgress = true;
+    pollStartedAt = Date.now();
     try {
-      await poll(state, scanQueue, stats);
+      await runPollCycle(state, scanQueue, stats);
       // Atomicity: persist queue + seq together after each poll
       persistQueue(scanQueue, state);
       saveNpmSeq(state.npmLastSeq);
@@ -881,6 +1011,14 @@ async function startMonitor(options, stats, dailyAlerts, recentlyScanned, downlo
       if (scanQueue.length > QUEUE_WARNING_THRESHOLD) {
         console.log(`[MONITOR] WARNING: scan queue depth ${scanQueue.length} — processing may be lagging behind ingestion`);
       }
+      // Apply hoisted poll backoff (set after consecutive total-registry failures).
+      const backoffMs = getPollBackoffMs();
+      if (backoffMs > 0) {
+        backoffUntil = Date.now() + backoffMs;
+        console.log(`[MONITOR] Poll backoff: skipping ticks for ${(backoffMs / 1000).toFixed(0)}s after consecutive registry failures`);
+      } else {
+        backoffUntil = 0;
+      }
     } catch (err) {
       console.error('[MONITOR] Poll error (interval):', err.message);
     } finally {
@@ -934,6 +1072,11 @@ async function startMonitor(options, stats, dailyAlerts, recentlyScanned, downlo
       // P1.0: persist the same sample as a time series for offline leak localisation.
       appendMemTrend(currentMem, getActiveWorkers(), scanQueue.length);
 
+      // Heap diagnostics (restart root-cause): cheap heap-spaces breakdown
+      // (retention vs churn) + opt-in one-shot snapshot at MUADDIB_HEAPSNAPSHOT_MB.
+      console.log(`[MONITOR] HEAP-SPACES: ${formatHeapSpaces(v8.getHeapSpaceStatistics())}`);
+      maybeHeapSnapshot(Number(heapUsedMB));
+
       // Graduated response at HIGH+
       if (pressureLevel >= MEMORY_PRESSURE_LEVELS.HIGH) {
         handleMemoryPressure(pressureLevel, heapRatio, recentlyScanned, downloadsCache, scanQueue);
@@ -995,6 +1138,11 @@ module.exports = {
   recordRestart,
   countRecentRestarts,
   POLL_INTERVAL,
+  POLL_WATCHDOG_MS,
+  runPollCycle,
+  shouldSkipPoll,
+  shouldSnapshot,
+  formatHeapSpaces,
   PROCESS_LOOP_INTERVAL,
   QUEUE_WARNING_THRESHOLD,
   QUEUE_PERSIST_INTERVAL,

package/src/monitor/ingestion.js

@@ -38,6 +38,7 @@ const SELF_PACKAGE_NAME = require('../../package.json').name;
 
 const POLL_INTERVAL = 60_000;
 const POLL_MAX_BACKOFF = 960_000; // 16 minutes max backoff
+const MAX_RESPONSE_BYTES = 64 * 1024 * 1024; // OOM guard: cap a single buffered HTTP (JSON/XML metadata) response at 64MB
 
 // --- Mutable state ---
 let consecutivePollErrors = 0;
@@ -48,7 +49,11 @@ let consecutivePollErrors = 0;
 // pollPyPIChangelog. Kept tiny on purpose — only network I/O lives here.
 const _deps = {
   httpsPost: null, // populated below once httpsPost is defined
-  httpsGet: null   // populated below; used by npm pollers so tests can stub
+  httpsGet: null,  // populated below; used by npm pollers so tests can stub
+  // Low-level client (https.get / https.request). Routing through _deps lets a
+  // test inject a fake req/res to exercise the absolute-deadline timer without
+  // real TLS. Production always uses the real `https` module.
+  https
 };
 
 function getConsecutivePollErrors() {
@@ -59,36 +64,71 @@ function setConsecutivePollErrors(val) {
   consecutivePollErrors = val;
 }
 
-// --- Utility ---
-
-function sleep(ms) {
-  return new Promise((resolve) => setTimeout(resolve, ms));
+/**
+ * Backoff (ms) the poll scheduler should wait before its next cycle, derived
+ * from consecutive total-registry failures. Returns 0 when healthy or after a
+ * single failure; otherwise exponential POLL_INTERVAL * 2^(n-1), capped at
+ * POLL_MAX_BACKOFF (16min). Pure read of module state — poll() never sleeps;
+ * the scheduler (daemon.js) owns the wait. See poll() for why the sleep was
+ * hoisted out (it used to hold pollInProgress for up to 16min).
+ * @returns {number}
+ */
+function getPollBackoffMs() {
+  if (consecutivePollErrors <= 1) return 0;
+  return Math.min(POLL_INTERVAL * Math.pow(2, consecutivePollErrors - 1), POLL_MAX_BACKOFF);
 }
 
 // --- HTTP helpers ---
 
-function httpsGet(url, timeoutMs = 30_000) {
+function httpsGet(url, timeoutMs = 30_000, deadlineMs = Math.max(timeoutMs * 2, 90_000)) {
   return new Promise((resolve, reject) => {
-    const req = https.get(url, { timeout: timeoutMs }, (res) => {
+    let settled = false;
+    let req;
+    // Absolute deadline. Node's `{ timeout }` option is a socket-INACTIVITY
+    // timeout, not an overall deadline: a response whose body trickles forever
+    // (heartbeat/keep-alive bytes, or a long-poll feed that never sends 'end')
+    // keeps the socket "active", so the inactivity timeout never fires and this
+    // promise never settles — wedging the poll loop. The deadline bounds the
+    // WHOLE request+body and destroys the socket so it can't leak.
+    const deadline = setTimeout(() => {
+      if (req) req.destroy(new Error(`Overall deadline (${Math.round(deadlineMs / 1000)}s) exceeded for ${url}`));
+    }, deadlineMs);
+    const done = (err, value) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(deadline);
+      if (err) reject(err); else resolve(value);
+    };
+    req = _deps.https.get(url, { timeout: timeoutMs }, (res) => {
       if (res.statusCode === 301 || res.statusCode === 302) {
         res.resume();
         const location = res.headers.location;
-        if (!location) return reject(new Error(`Redirect without Location for ${url}`));
-        return httpsGet(location, timeoutMs).then(resolve, reject);
+        if (!location) return done(new Error(`Redirect without Location for ${url}`));
+        // Hand the deadline off to the recursive call, which has its own.
+        settled = true;
+        clearTimeout(deadline);
+        return httpsGet(location, timeoutMs, deadlineMs).then(resolve, reject);
       }
       if (res.statusCode < 200 || res.statusCode >= 300) {
         res.resume();
-        return reject(new Error(`HTTP ${res.statusCode} for ${url}`));
+        return done(new Error(`HTTP ${res.statusCode} for ${url}`));
       }
       const chunks = [];
-      res.on('data', (chunk) => chunks.push(chunk));
-      res.on('end', () => resolve(Buffer.concat(chunks).toString('utf8')));
-      res.on('error', reject);
+      let total = 0;
+      res.on('data', (chunk) => {
+        total += chunk.length;
+        if (total > MAX_RESPONSE_BYTES) {
+          req.destroy(new Error(`Response exceeded ${MAX_RESPONSE_BYTES} bytes for ${url}`));
+          return;
+        }
+        chunks.push(chunk);
+      });
+      res.on('end', () => done(null, Buffer.concat(chunks).toString('utf8')));
+      res.on('error', (err) => done(err));
     });
-    req.on('error', reject);
+    req.on('error', (err) => done(err));
     req.on('timeout', () => {
-      req.destroy();
-      reject(new Error(`Timeout for ${url}`));
+      req.destroy(new Error(`Timeout for ${url}`));
     });
   });
 }
@@ -97,7 +137,7 @@ function httpsGet(url, timeoutMs = 30_000) {
  * Minimal HTTPS POST. Used for PyPI XML-RPC; kept inside the ingestion module
  * (rather than pulled into shared/) because XML-RPC is its only consumer today.
  */
-function httpsPost(url, body, headers = {}, timeoutMs = 30_000) {
+function httpsPost(url, body, headers = {}, timeoutMs = 30_000, deadlineMs = Math.max(timeoutMs * 2, 90_000)) {
   return new Promise((resolve, reject) => {
     const u = new URL(url);
     const options = {
@@ -112,20 +152,40 @@ function httpsPost(url, body, headers = {}, timeoutMs = 30_000) {
         ...headers
       }
     };
-    const req = https.request(options, (res) => {
+    let settled = false;
+    let req;
+    // Absolute deadline — see httpsGet for the rationale (inactivity timeout is
+    // not an overall deadline; a trickling body would hang forever otherwise).
+    const deadline = setTimeout(() => {
+      if (req) req.destroy(new Error(`Overall deadline (${Math.round(deadlineMs / 1000)}s) exceeded for POST ${url}`));
+    }, deadlineMs);
+    const done = (err, value) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(deadline);
+      if (err) reject(err); else resolve(value);
+    };
+    req = _deps.https.request(options, (res) => {
       if (res.statusCode < 200 || res.statusCode >= 300) {
         res.resume();
-        return reject(new Error(`HTTP ${res.statusCode} for POST ${url}`));
+        return done(new Error(`HTTP ${res.statusCode} for POST ${url}`));
       }
       const chunks = [];
-      res.on('data', (chunk) => chunks.push(chunk));
-      res.on('end', () => resolve(Buffer.concat(chunks).toString('utf8')));
-      res.on('error', reject);
+      let total = 0;
+      res.on('data', (chunk) => {
+        total += chunk.length;
+        if (total > MAX_RESPONSE_BYTES) {
+          req.destroy(new Error(`Response exceeded ${MAX_RESPONSE_BYTES} bytes for POST ${url}`));
+          return;
+        }
+        chunks.push(chunk);
+      });
+      res.on('end', () => done(null, Buffer.concat(chunks).toString('utf8')));
+      res.on('error', (err) => done(err));
     });
-    req.on('error', reject);
+    req.on('error', (err) => done(err));
     req.on('timeout', () => {
-      req.destroy();
-      reject(new Error(`Timeout for POST ${url}`));
+      req.destroy(new Error(`Timeout for POST ${url}`));
     });
     req.write(body);
     req.end();
@@ -1292,13 +1352,15 @@ async function poll(state, scanQueue, stats) {
     pollPyPI(state, scanQueue, stats)
   ]);
 
-  // Track consecutive poll failures for backoff
+  // Track consecutive poll failures. The backoff WAIT is applied by the
+  // scheduler (daemon.js, via getPollBackoffMs()), NOT here: sleeping inside
+  // poll() used to hold pollInProgress for up to POLL_MAX_BACKOFF (16min),
+  // stalling ingestion and forcing the poll watchdog to be sized above the
+  // backoff. poll() stays sleep-free so the watchdog bounds poll *work* only.
   if (npmCount === -1 && pypiCount === -1) {
     consecutivePollErrors++;
     if (consecutivePollErrors > 1) {
-      const backoff = Math.min(POLL_INTERVAL * Math.pow(2, consecutivePollErrors - 1), POLL_MAX_BACKOFF);
-      console.log(`[MONITOR] Both registries failed (${consecutivePollErrors}x) — backing off ${(backoff / 1000).toFixed(0)}s`);
-      await sleep(backoff);
+      console.log(`[MONITOR] Both registries failed (${consecutivePollErrors}x) — scheduler will back off ${(getPollBackoffMs() / 1000).toFixed(0)}s`);
     }
   } else {
     consecutivePollErrors = 0;
@@ -1314,10 +1376,12 @@ module.exports = {
   SELF_PACKAGE_NAME,
   POLL_INTERVAL,
   POLL_MAX_BACKOFF,
+  MAX_RESPONSE_BYTES,
 
   // Mutable state
   getConsecutivePollErrors,
   setConsecutivePollErrors,
+  getPollBackoffMs,
 
   // HTTP helpers
   httpsGet,