muaddib-scanner 2.11.134 → 2.11.136

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "muaddib-scanner",
-  "version": "2.11.134",
+  "version": "2.11.136",
   "description": "Supply-chain threat detection & response for npm & PyPI/Python",
   "main": "src/index.js",
   "bin": {

package/{self-scan-v2.11.134.json → self-scan-v2.11.136.json}

@@ -1,6 +1,6 @@
 {
   "target": "node_modules",
-  "timestamp": "2026-06-28T13:13:44.321Z",
+  "timestamp": "2026-06-30T19:55:27.666Z",
   "threats": [
     {
       "type": "string_mutation_obfuscation",

package/src/monitor/daemon.js

@@ -1040,6 +1040,8 @@ async function startMonitor(options, stats, dailyAlerts, recentlyScanned, downlo
     } catch (e) {
       console.error(`[MONITOR] Shutdown in-flight spill failed: ${e.message}`);
     }
+    // Terminate the off-thread extraction worker pool (bounded-resource teardown).
+    try { await require('../shared/extract-pool.js').destroyExtractPool(); } catch { /* best-effort */ }
     // Persist remaining queue items so they survive the restart
     persistQueue(scanQueue, state);
     saveRecentlyScanned(recentlyScanned); // Persist dedup set too (avoid re-scan storm on restart)

package/src/monitor/ingestion.js

@@ -622,6 +622,100 @@ function isPrereleaseVersion(v) {
   return /-/.test(String(v == null ? '' : v).split('+')[0]);
 }
 
+// Derive the capture-at-publish trigger for the fast-takedown class (Leo/Miasma)
+// from a registry packument. Shared by preResolveNpmBatch (Stage 2.1) and the
+// real-time capture lane so both compute the SAME signal: ATO = published version
+// != dist-tags.latest, excluding prereleases (isPrereleaseVersion); burst =
+// >= BURST_MIN_VERSIONS_PREFETCH versions in the recent window. Returns the cache
+// trigger object (shouldCache:true) or null. `name` is passed through so a package
+// that is ALSO an ioc/typosquat match still gets the right (higher-priority) reason.
+function evaluateBurstAtoTrigger(name, npmInfo, version) {
+  if (!npmInfo) return null;
+  const atoSignal = !!(npmInfo.latestTagVersion && version &&
+    version !== npmInfo.latestTagVersion && !isPrereleaseVersion(version));
+  const burstCount = Number.isFinite(npmInfo.recentWindowCount)
+    ? npmInfo.recentWindowCount
+    : ((Array.isArray(npmInfo.recentVersions) ? npmInfo.recentVersions.length : 0) + 1);
+  const isBurst = burstCount >= BURST_MIN_VERSIONS_PREFETCH;
+  if (!atoSignal && !isBurst) return null;
+  const trig = evaluateCacheTrigger(name, null, null, { atoSignal, isBurst });
+  return trig.shouldCache ? trig : null;
+}
+
+// Fix C — real-time capture lane. preResolveNpmBatch sheds the packument prefetch
+// when the scan queue is deep (queue > PRE_RESOLVE_SHED_QUEUE — the chronic backlog
+// state). Under shed nothing is prefetched AT ALL: schedulePrefetch needs a resolved
+// tarballUrl, which the shed never fetched — so even ioc/typosquat/first_publish items
+// (flagged name-only at ingestion) are not captured, and burst/ATO is never even
+// computed. The fast-takedown tarballs then 404 before the backlogged scan reaches
+// them (the Leo/Miasma gap Fix A meant to close). This lane resolves a BOUNDED subset
+// of the still-unresolved items REGARDLESS of the shed — prioritizing the already
+// name-flagged (known high-risk, FP-free), then filling the budget to DISCOVER
+// burst/ATO — and enriches each item (version/tarballUrl/_npmInfo/_cacheTrigger) IN
+// PLACE so (a) the schedulePrefetch pass that immediately follows captures the tarball
+// at publish, and (b) the already-enqueued item (enqueueScan pushes by reference)
+// carries the real version so scanPackage's cache-hit (keyed by name@version) finds it.
+//
+// Bounded + rate-limited + flag-gated so it can be turned on only once throughput /
+// OOM headroom allows: OFF unless MUADDIB_REALTIME_PREFETCH=1; at most
+// MUADDIB_REALTIME_PREFETCH_MAX items/cycle (default 30); registry calls go through
+// getNpmLatestTarball's shared semaphore (honors the 429 brain). npm-only for now.
+const REALTIME_PREFETCH_CONCURRENCY = 4;
+function realtimePrefetchEnabled() {
+  return process.env.MUADDIB_REALTIME_PREFETCH === '1';
+}
+function realtimePrefetchMax() {
+  const n = parseInt(process.env.MUADDIB_REALTIME_PREFETCH_MAX, 10);
+  return Number.isFinite(n) && n >= 0 ? n : 30;
+}
+async function realtimePrefetchHighRisk(items, stats) {
+  if (!realtimePrefetchEnabled()) return;
+  const cap = realtimePrefetchMax();
+  if (!cap || !Array.isArray(items) || items.length === 0) return;
+  // Act only on items the shed (or a resolve failure) left UNRESOLVED (no _npmInfo).
+  // When preResolveNpmBatch did NOT shed, every item already has _npmInfo and this
+  // selects nothing (no duplicate fetch). Flagged first (ioc/typo/first_publish —
+  // known high-risk, just needs its tarballUrl resolved to be prefetched), then
+  // unflagged (candidates for burst/ATO discovery).
+  const flagged = [];
+  const unflagged = [];
+  for (const it of items) {
+    if (!it || it.ecosystem !== 'npm' || !it.name || it._npmInfo) continue;
+    (it._cacheTrigger ? flagged : unflagged).push(it);
+  }
+  const selected = flagged.concat(unflagged).slice(0, cap);
+  if (selected.length === 0) return;
+  const overflow = (flagged.length + unflagged.length) - selected.length;
+  if (overflow > 0) {
+    // No silent caps (CLAUDE.md): a burst wider than the budget is only partially captured.
+    console.warn(`[MONITOR] REALTIME PREFETCH: ${flagged.length + unflagged.length} unresolved high-risk candidates, capped at ${cap} this cycle (${overflow} deferred to lazy scan)`);
+    if (stats) stats.realtimePrefetchCapped = (stats.realtimePrefetchCapped || 0) + overflow;
+  }
+  let idx = 0;
+  const worker = async () => {
+    while (idx < selected.length) {
+      const it = selected[idx++];
+      try {
+        const npmInfo = await getNpmLatestTarball(it.name);
+        if (!npmInfo || !npmInfo.tarball) continue;
+        // Enrich in place — the item is already enqueued by reference (shed path), so
+        // this also lets the eventual scan reuse the URL + hit the prefetch cache.
+        it.tarballUrl = it.tarballUrl || npmInfo.tarball;
+        if (!it.version) it.version = npmInfo.version || '';
+        it._npmInfo = npmInfo;
+        if (!it._cacheTrigger) {
+          const trig = evaluateBurstAtoTrigger(it.name, npmInfo, it.version);
+          if (trig) it._cacheTrigger = trig;
+        }
+        if (it._cacheTrigger && stats) stats.realtimePrefetchFlagged = (stats.realtimePrefetchFlagged || 0) + 1;
+      } catch {
+        if (stats) stats.realtimePrefetchFailed = (stats.realtimePrefetchFailed || 0) + 1;
+      }
+    }
+  };
+  await Promise.all(Array.from({ length: Math.min(REALTIME_PREFETCH_CONCURRENCY, selected.length) }, worker));
+}
+
 async function preResolveNpmBatch(items, stats, scanQueue) {
   if (!items || items.length === 0) return;
   const start = Date.now();
@@ -676,17 +770,8 @@ async function preResolveNpmBatch(items, stats, scanQueue) {
           // higher-priority (ioc/typosquat) trigger already fired. queue.js still computes
           // the same signals later for scan-queue protection + extras enqueue (idempotent).
           if (!item._cacheTrigger) {
-            const atoSignal = !!(npmInfo.latestTagVersion && item.version &&
-              item.version !== npmInfo.latestTagVersion &&
-              !isPrereleaseVersion(item.version));
-            const burstCount = Number.isFinite(npmInfo.recentWindowCount)
-              ? npmInfo.recentWindowCount
-              : ((Array.isArray(npmInfo.recentVersions) ? npmInfo.recentVersions.length : 0) + 1);
-            const isBurst = burstCount >= BURST_MIN_VERSIONS_PREFETCH;
-            if (atoSignal || isBurst) {
-              const trig = evaluateCacheTrigger(item.name, null, null, { atoSignal, isBurst });
-              if (trig.shouldCache) item._cacheTrigger = trig;
-            }
+            const trig = evaluateBurstAtoTrigger(item.name, npmInfo, item.version);
+            if (trig) item._cacheTrigger = trig;
           }
           resolved++;
         } else {
@@ -962,6 +1047,12 @@ async function pollNpmChanges(state, scanQueue, stats) {
     // resolveTarballAndScan() picks up the slack — zero scan loss.
     await preResolveNpmBatch(newItems, stats, scanQueue);
 
+    // Fix C — real-time capture lane: when preResolveNpmBatch shed under backlog,
+    // resolve+flag a bounded high-risk subset anyway so the schedulePrefetch below
+    // still captures them. No-op when not shedding or when MUADDIB_REALTIME_PREFETCH
+    // is off (default). See realtimePrefetchHighRisk.
+    await realtimePrefetchHighRisk(newItems, stats);
+
     // Capture-at-publish (Miasma / Leo, June 2026): prefetch the high-value
     // subset's tarballs into the local cache NOW — before the (often backlogged)
     // scan downloads them — so we win the race against fast-takedown unpublishes.
@@ -1604,6 +1695,8 @@ module.exports = {
   preResolvePyPIBatch,
   preResolveShouldShed,
   isPrereleaseVersion,
+  evaluateBurstAtoTrigger,
+  realtimePrefetchHighRisk,
 
   // RSS parsing
   parseNpmRss,

package/src/monitor/queue.js

@@ -12,7 +12,8 @@ const os = require('os');
 const { Worker } = require('worker_threads');
 const { runSandbox, tryAcquireSandboxSlot } = require('../sandbox/index.js');
 const { sendWebhook } = require('../webhook.js');
-const { downloadToFile, extractArchive, extractArchiveOffThread, sanitizePackageName } = require('../shared/download.js');
+const { downloadToFile, extractArchive, sanitizePackageName } = require('../shared/download.js');
+const { extractInPool } = require('../shared/extract-pool.js');
 const { MAX_TARBALL_SIZE, getMaxFileSize } = require('../shared/constants.js');
 const { acquireRegistrySlot, releaseRegistrySlot, awaitRateToken: awaitRateTokenForWorker, signal429: signal429ForWorker } = require('../shared/http-limiter.js');
 const { loadCachedIOCs } = require('../ioc/updater.js');
@@ -179,21 +180,25 @@ const STATIC_SCAN_TIMEOUT_MS = 45_000; // 45s for static analysis only
 const LARGE_PACKAGE_SIZE = 10 * 1024 * 1024; // 10MB
 const RECENTLY_SCANNED_MAX = 50_000; // FIFO cap for the dedup Set (P0c — bounded resource)
 
-// OOM fix (2026-06-19): archives larger than this (COMPRESSED, on-disk size) are
-// extracted off the main thread in a worker, so the synchronous extractor
-// (adm-zip extractAllTo / execFileSync tar) can no longer wedge the event loop and
-// starve the RSS breaker / memory governor / EMERGENCY purge (all main-thread
-// timers) → cgroup OOM. Confirmed culprit: data/loop-stalls.jsonl (extract:* up to
-// 148s). Small archives extract inline — a worker spawn costs more than their
-// sub-100ms extraction. Env-tunable via MUADDIB_INLINE_EXTRACT_MB.
-const INLINE_EXTRACT_MAX_BYTES = (parseInt(process.env.MUADDIB_INLINE_EXTRACT_MB, 10) || 4) * 1024 * 1024;
-
-// Extract inline for small archives, off-thread for large ones. compressedSize is
-// the on-disk tarball size (reliable, unlike the registry unpackedSize metadata).
-// Always returns a Promise so the call sites can uniformly `await`.
+// Event-loop-stall fix (2026-06-30): ALL archive extraction runs off the main
+// thread via a persistent worker pool (src/shared/extract-pool.js). The previous
+// OOM fix only offloaded archives > 4MB (spawn-per-call) and extracted everything
+// smaller INLINE — but adm-zip extractAllTo is fully synchronous and slow on
+// many-file trees regardless of size, so those inline extractions still wedged the
+// loop for seconds-to-minutes (data/loop-stalls.jsonl: extract:prework up to 229s;
+// a 2MB many-file package blocked 5s+), starving the RSS breaker / memory governor
+// / EMERGENCY purge (all main-thread timers) → restart. The pool's reusable workers
+// pay no per-package spawn cost, so there is no longer a reason to extract inline.
+// MUADDIB_INLINE_EXTRACT_MB keeps a small inline escape hatch (default 0 = always
+// pool). compressedSize is the on-disk tarball size (reliable, unlike registry
+// unpackedSize metadata).
+const INLINE_EXTRACT_MAX_BYTES = (parseInt(process.env.MUADDIB_INLINE_EXTRACT_MB, 10) || 0) * 1024 * 1024;
+
+// Always returns a Promise so call sites uniformly `await`. Pool for anything above
+// the (default 0) inline window; the tiny inline path stays available for operators.
 function extractGated(archivePath, destDir, compressedSize) {
   if (compressedSize > INLINE_EXTRACT_MAX_BYTES) {
-    return extractArchiveOffThread(archivePath, destDir);
+    return extractInPool(archivePath, destDir);
   }
   return Promise.resolve(extractArchive(archivePath, destDir));
 }

package/src/shared/extract-pool-worker.js

@@ -0,0 +1,30 @@
+'use strict';
+
+/**
+ * Persistent worker for the extraction pool (src/shared/extract-pool.js).
+ *
+ * Unlike extract-worker.js (one-shot, used by download.js extractArchiveOffThread),
+ * this worker stays alive and serves a request/response loop: one extraction per
+ * message, reusing the loaded extractArchive + adm-zip across packages so the pool
+ * pays no per-package Worker-spawn cost. All extraction hardening (zip-slip,
+ * zip-bomb uncompressed-size cap) lives in extractArchive and runs HERE, off the
+ * parent's event loop. Each reply echoes the job id so the pool matches it to the
+ * caller. Never throws to the thread — failures come back as { ok:false, error }.
+ *
+ * Message in:  { id, archivePath, destDir, format? }
+ * Message out: { id, ok:true, dir } | { id, ok:false, error }
+ */
+
+const { parentPort } = require('worker_threads');
+const { extractArchive } = require('./download.js');
+
+parentPort.on('message', (job) => {
+  if (!job || typeof job.id === 'undefined') return;
+  try {
+    const opts = job.format ? { format: job.format } : {};
+    const dir = extractArchive(job.archivePath, job.destDir, opts);
+    parentPort.postMessage({ id: job.id, ok: true, dir });
+  } catch (err) {
+    parentPort.postMessage({ id: job.id, ok: false, error: err && err.message ? err.message : String(err) });
+  }
+});

package/src/shared/extract-pool.js

@@ -0,0 +1,217 @@
+'use strict';
+
+/**
+ * Persistent worker-thread pool for off-main-thread archive extraction.
+ *
+ * Why a pool, not download.js extractArchiveOffThread (spawn-per-call): the
+ * monitor extracts thousands of small packages on its main thread. The
+ * spawn-per-call offloader pays a fresh Worker (V8 isolate + module load,
+ * ~10-40ms) every time, which is why queue.js gated it to archives > 4MB and
+ * ran everything smaller INLINE (synchronous extractArchive). Those inline
+ * extractions are exactly what wedge the event loop: adm-zip `extractAllTo` is
+ * fully synchronous and slow on many-file trees regardless of total size
+ * (data/loop-stalls.jsonl: `extract:prework` up to 229s; a 2MB many-file
+ * package blocks 5s+). A wedged loop starves the RSS breaker / memory governor /
+ * EMERGENCY purge (all main-thread timers) → restart.
+ *
+ * A pool of N reusable workers removes BOTH costs: no extraction ever runs on
+ * the main thread (at any size), and there is no per-package spawn. The size
+ * gate in queue.js then becomes unnecessary (its inline window defaults to 0).
+ *
+ * Each worker runs extract-pool-worker.js — a request/response loop that calls
+ * the SAME extractArchive, so all hardening (zip-slip, zip-bomb uncompressed
+ * cap) stays centralized there and runs in the worker.
+ *
+ * Bounded + crash-resilient (CLAUDE.md "Production Engineering"):
+ *  - POOL_SIZE workers max, spawned lazily on demand; idle workers are `unref`'d
+ *    so the pool never keeps the process alive on its own.
+ *  - One job per worker; excess jobs wait in a FIFO bounded to MAX_PENDING —
+ *    past the cap a job rejects immediately (the caller treats it as an extract
+ *    failure → size_skip, ledgered) so a slow extractor cannot grow memory
+ *    without bound.
+ *  - Per-job timeout terminates and drops a wedged worker and rejects that job —
+ *    a pathological archive cannot pin a slot forever.
+ *  - A worker that errors or exits mid-job rejects its in-flight job and is
+ *    removed; the next dispatch lazily respawns up to POOL_SIZE.
+ */
+
+const path = require('path');
+
+const POOL_SIZE = (() => {
+  const v = parseInt(process.env.MUADDIB_EXTRACT_POOL_SIZE, 10);
+  if (Number.isFinite(v) && v >= 1) return v;
+  let cores = 4;
+  try { cores = require('os').cpus().length || 4; } catch { /* default 4 */ }
+  return Math.max(2, Math.min(4, cores - 2));
+})();
+
+const JOB_TIMEOUT_MS = (() => {
+  const v = parseInt(process.env.MUADDIB_EXTRACT_POOL_TIMEOUT_MS, 10);
+  return Number.isFinite(v) && v > 0 ? v : 120_000;
+})();
+
+const MAX_PENDING = (() => {
+  const v = parseInt(process.env.MUADDIB_EXTRACT_POOL_MAX_PENDING, 10);
+  return Number.isFinite(v) && v > 0 ? v : 512;
+})();
+
+const WORKER_FILE = path.join(__dirname, 'extract-pool-worker.js');
+
+// Pool state (lazy-init on first extractInPool). Each worker entry:
+// { worker, busy, job }. `pending` is the FIFO of jobs waiting for a free slot.
+let workers = [];
+let pending = [];
+let jobSeq = 0;
+let destroyed = false;
+
+function _settle(job, err, dir) {
+  if (job.settled) return;
+  job.settled = true;
+  if (job.timer) { clearTimeout(job.timer); job.timer = null; }
+  if (err) job.reject(err);
+  else job.resolve(dir);
+}
+
+function _removeWorker(entry) {
+  const i = workers.indexOf(entry);
+  if (i !== -1) workers.splice(i, 1);
+}
+
+function _spawnWorker() {
+  const { Worker } = require('worker_threads');
+  const entry = { worker: null, busy: false, job: null };
+  const w = new Worker(WORKER_FILE);
+  entry.worker = w;
+  // Idle workers must not keep the process alive; an in-flight job is kept alive
+  // by the caller's awaited Promise, not by the worker handle.
+  if (typeof w.unref === 'function') w.unref();
+  w.on('message', (msg) => _onMessage(entry, msg));
+  w.on('error', (err) => _onWorkerError(entry, err));
+  w.on('exit', (code) => _onWorkerExit(entry, code));
+  workers.push(entry);
+  return entry;
+}
+
+function _onMessage(entry, msg) {
+  const job = entry.job;
+  entry.busy = false;
+  entry.job = null;
+  if (job && msg && msg.id === job.id) {
+    if (msg.ok) _settle(job, null, msg.dir);
+    else _settle(job, new Error(msg.error || 'extract-pool: worker reported failure'));
+  }
+  _dispatch();
+}
+
+function _onWorkerError(entry, err) {
+  const job = entry.job;
+  entry.busy = false;
+  entry.job = null;
+  _removeWorker(entry);
+  if (job) _settle(job, err instanceof Error ? err : new Error(String(err)));
+  _dispatch();
+}
+
+function _onWorkerExit(entry, code) {
+  const job = entry.job;
+  entry.busy = false;
+  entry.job = null;
+  _removeWorker(entry);
+  if (job) _settle(job, new Error(`extract-pool: worker exited (${code}) mid-job`));
+  _dispatch();
+}
+
+function _assign(entry, job) {
+  entry.busy = true;
+  entry.job = job;
+  job.timer = setTimeout(() => {
+    // Worker is wedged on a pathological archive: terminate it (best-effort) and
+    // drop it from the pool, reject the job, let _dispatch respawn lazily.
+    entry.busy = false;
+    entry.job = null;
+    _removeWorker(entry);
+    try { entry.worker.terminate(); } catch { /* already gone */ }
+    _settle(job, new Error(`extract-pool: timeout after ${JOB_TIMEOUT_MS}ms: ${path.basename(job.archivePath)}`));
+    _dispatch();
+  }, JOB_TIMEOUT_MS);
+  if (job.timer && typeof job.timer.unref === 'function') job.timer.unref();
+  entry.worker.postMessage({ id: job.id, archivePath: job.archivePath, destDir: job.destDir, format: job.format });
+}
+
+function _dispatch() {
+  if (destroyed) return;
+  while (pending.length > 0) {
+    let entry = workers.find((e) => !e.busy);
+    if (!entry) {
+      if (workers.length < POOL_SIZE) entry = _spawnWorker();
+      else break; // all workers busy and at cap — wait for a completion
+    }
+    _assign(entry, pending.shift());
+  }
+}
+
+/**
+ * Extract an archive off the main thread via the pool. Same contract as
+ * download.js extractArchive (resolves to the extracted package root) but never
+ * blocks the caller's event loop.
+ *
+ * @param {string} archivePath
+ * @param {string} destDir  - must already exist
+ * @param {Object} [options]
+ * @param {'targz'|'zip'} [options.format] - override auto-detection
+ * @returns {Promise<string>} extracted package root
+ */
+function extractInPool(archivePath, destDir, options = {}) {
+  return new Promise((resolve, reject) => {
+    if (destroyed) { reject(new Error('extract-pool: pool destroyed')); return; }
+    if (pending.length >= MAX_PENDING) {
+      reject(new Error(`extract-pool: pending queue full (${MAX_PENDING}) — extraction is not keeping up`));
+      return;
+    }
+    const job = {
+      id: ++jobSeq,
+      archivePath,
+      destDir,
+      format: (options && options.format) || null,
+      resolve,
+      reject,
+      settled: false,
+      timer: null,
+    };
+    pending.push(job);
+    _dispatch();
+  });
+}
+
+/**
+ * Terminate all workers and reject any in-flight / pending jobs. Idempotent and
+ * reusable: after it resolves the pool lazily re-inits on the next extractInPool
+ * (the daemon calls this once during gracefulShutdown; tests call it between
+ * cases).
+ * @returns {Promise<void>}
+ */
+async function destroyExtractPool() {
+  destroyed = true;
+  const inflight = workers.slice();
+  const queued = pending.slice();
+  workers = [];
+  pending = [];
+  for (const job of queued) _settle(job, new Error('extract-pool: pool destroyed'));
+  await Promise.all(inflight.map((entry) => {
+    if (entry.job) _settle(entry.job, new Error('extract-pool: pool destroyed'));
+    try { return entry.worker.terminate(); } catch { return Promise.resolve(); }
+  }));
+  destroyed = false;
+}
+
+/** Observability snapshot (live workers / busy / pending), all bounded. */
+function getPoolStats() {
+  return {
+    size: workers.length,
+    max: POOL_SIZE,
+    busy: workers.filter((e) => e.busy).length,
+    pending: pending.length,
+  };
+}
+
+module.exports = { extractInPool, destroyExtractPool, getPoolStats, POOL_SIZE };