muaddib-scanner 2.11.98 → 2.11.100

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "muaddib-scanner",
-  "version": "2.11.98",
+  "version": "2.11.100",
   "description": "Supply-chain threat detection & response for npm & PyPI/Python",
   "main": "src/index.js",
   "bin": {

package/{self-scan-v2.11.98.json → self-scan-v2.11.100.json}

@@ -1,6 +1,6 @@
 {
   "target": "node_modules",
-  "timestamp": "2026-06-11T15:36:15.399Z",
+  "timestamp": "2026-06-11T17:09:48.912Z",
   "threats": [
     {
       "type": "string_mutation_obfuscation",

package/src/monitor/heavy-lane.js

@@ -0,0 +1,165 @@
+'use strict';
+
+/**
+ * Heavy-lane semaphore (C2, 2026-06-11) — bound the daemon's RSS by limiting
+ * how many MEMORY-heavy static scans run concurrently.
+ *
+ * Measured (worker-mem.jsonl, n=461 workers): per-worker isolate heap peaks
+ * are BIMODAL — p50 = 12MB, but 12.6% of scans jump straight to 0.9-2.1GB
+ * (giant minified JS bundles; the AST cache accumulates across every parsable
+ * file, executor.js only skips files > getMaxFileSize() individually). With
+ * 8 concurrent workers a handful of heavies coincide → process RSS > the
+ * 8.5GB breaker → EMERGENCY. The heavies are identifiable BEFORE the worker
+ * spawns (total parsable-JS bytes on disk), so instead of killing them (a
+ * 768MB worker cap would cost 12% of coverage) we serialize them: at most
+ * MUADDIB_HEAVY_SCAN_MAX run at once, lights are NEVER blocked.
+ * Worst-case RSS ≈ baseline 2GB + 2×2GB heavies + N×12MB lights ≈ 5-6GB.
+ *
+ * Same {active, queue[]} semaphore pattern as src/shared/http-limiter.js and
+ * the sandbox slots (src/sandbox/index.js), plus two extensions those never
+ * needed: an abort-aware acquire and a wait-timeout. Both MUST remove their
+ * waiter from the queue on the way out — a release would otherwise hand the
+ * slot to a dead waiter and leak it permanently.
+ */
+
+// Max number of HEAVY_LANE_WAIT_TIMEOUT requeues before an item's final pass
+// runs without the wait bound (abort-aware only, still bounded by the outer
+// SCAN_TIMEOUT_MS). Guarantees an item cannot loop in the queue forever.
+const HEAVY_REQUEUE_MAX = 3;
+
+// Env knobs (read at call time so tests can flip them around resetHeavyLane()):
+// - MUADDIB_HEAVY_SCAN_MAX: concurrent heavy scans (default 2, 0 = lane off)
+// - MUADDIB_HEAVY_SCAN_BYTES: heavy threshold on total parsable-JS bytes.
+//   Default 3 MiB — the measured distribution has a HOLE between light
+//   (≤12MB heap ⇔ <~1MB JS) and heavy (≥512MB heap ⇔ ≥~8MB JS); 3 MiB sits
+//   in the hole with ~3× margin both ways. A false-heavy costs a short wait;
+//   a false-light risks an EMERGENCY — hence the deliberately low default.
+// - MUADDIB_HEAVY_WAIT_MAX_MS: wait bound before requeue (default 120s —
+//   ~2.5 slot services of 45s, leaves >150s of the 300s scan budget).
+function heavyScanMax() {
+  const v = parseInt(process.env.MUADDIB_HEAVY_SCAN_MAX, 10);
+  return Number.isFinite(v) && v >= 0 ? v : 2;
+}
+
+function heavyScanBytesThreshold() {
+  const v = parseInt(process.env.MUADDIB_HEAVY_SCAN_BYTES, 10);
+  return Number.isFinite(v) && v > 0 ? v : 3 * 1024 * 1024;
+}
+
+function heavyWaitMaxMs() {
+  const v = parseInt(process.env.MUADDIB_HEAVY_WAIT_MAX_MS, 10);
+  return Number.isFinite(v) && v >= 0 ? v : 120000;
+}
+
+const _lane = { active: 0, queue: [] };
+
+/**
+ * Pure classifier. `truncated` (the bounded measurement walk overflowed its
+ * depth/file caps) classifies heavy by default — defensive: an unmeasurable
+ * package is exactly the kind that blows a worker. Compares weightedJsBytes
+ * (plain + ×12 minified — see measureJsWeight in queue.js: raw bytes alone
+ * missed the minified explosions, powerlines 517KB → 1151MB heap) and falls
+ * back to totalJsBytes for callers that don't weight.
+ * @param {{totalJsBytes: number, weightedJsBytes?: number, truncated: boolean}|null} weight
+ * @param {number} [thresholdBytes]
+ */
+function isHeavyScan(weight, thresholdBytes = heavyScanBytesThreshold()) {
+  if (!weight) return false;
+  if (weight.truncated) return true;
+  const effective = Number.isFinite(weight.weightedJsBytes) ? weight.weightedJsBytes : (weight.totalJsBytes || 0);
+  return effective >= thresholdBytes;
+}
+
+/**
+ * Acquire a heavy-lane slot. Resolves true when a slot is held, false when
+ * the lane is disabled (MUADDIB_HEAVY_SCAN_MAX=0 — nothing to release).
+ * FIFO when saturated.
+ *
+ * @param {Object} [opts]
+ * @param {AbortSignal} [opts.signal] - outer scan abort: rejects err.code='ABORT_ERR'
+ * @param {number} [opts.maxWaitMs] - wait bound; 0/absent = unbounded.
+ *   On expiry rejects err.code='HEAVY_LANE_WAIT_TIMEOUT' (caller requeues).
+ * @returns {Promise<boolean>}
+ */
+function acquireHeavySlot(opts = {}) {
+  const max = heavyScanMax();
+  if (max === 0) return Promise.resolve(false);
+  if (_lane.active < max) {
+    _lane.active++;
+    return Promise.resolve(true);
+  }
+  const { signal, maxWaitMs } = opts;
+  return new Promise((resolve, reject) => {
+    let timer = null;
+    const cleanup = () => {
+      if (timer) { clearTimeout(timer); timer = null; }
+      if (signal) { try { signal.removeEventListener('abort', onAbort); } catch { /* not added */ } }
+    };
+    const waiter = () => {
+      cleanup();
+      resolve(true); // slot transferred by releaseHeavySlot (active unchanged)
+    };
+    // Leaving the queue WITHOUT being woken: splice the waiter out, or the
+    // next release hands the slot to this dead waiter and leaks it (trap #1).
+    const bail = (err) => {
+      const i = _lane.queue.indexOf(waiter);
+      if (i === -1) return; // already woken — the release path owns the slot
+      _lane.queue.splice(i, 1);
+      cleanup();
+      reject(err);
+    };
+    const onAbort = () => {
+      const err = new Error('Heavy-lane wait aborted (outer scan timeout)');
+      err.code = 'ABORT_ERR';
+      bail(err);
+    };
+    // Push BEFORE wiring abort/timeout: bail() rejects only when it finds the
+    // waiter in the queue (its index check guards the already-woken race) —
+    // a pre-aborted signal firing before the push would otherwise bail into
+    // the guard and leave the promise forever pending.
+    _lane.queue.push(waiter);
+    if (signal) {
+      if (signal.aborted) { onAbort(); return; }
+      signal.addEventListener('abort', onAbort, { once: true });
+    }
+    if (Number.isFinite(maxWaitMs) && maxWaitMs > 0) {
+      // Deliberately NOT unref'd: a pending acquire is active work (a scan
+      // holding tmp disk and a pool slot) — it must keep the process alive.
+      timer = setTimeout(() => {
+        const err = new Error(`Heavy-lane slot not acquired within ${maxWaitMs}ms`);
+        err.code = 'HEAVY_LANE_WAIT_TIMEOUT';
+        bail(err);
+      }, maxWaitMs);
+    }
+  });
+}
+
+function releaseHeavySlot() {
+  if (_lane.queue.length > 0) {
+    const next = _lane.queue.shift();
+    next(); // transfers the slot to the next waiter (active count unchanged)
+  } else if (_lane.active > 0) {
+    _lane.active--;
+  }
+}
+
+function getHeavyLaneState() {
+  return { active: _lane.active, waiting: _lane.queue.length, max: heavyScanMax() };
+}
+
+/** Test helper — same role as resetSandboxLimiter in src/sandbox/index.js. */
+function resetHeavyLane() {
+  _lane.active = 0;
+  _lane.queue.length = 0;
+}
+
+module.exports = {
+  acquireHeavySlot,
+  releaseHeavySlot,
+  isHeavyScan,
+  getHeavyLaneState,
+  resetHeavyLane,
+  heavyScanBytesThreshold,
+  heavyWaitMaxMs,
+  HEAVY_REQUEUE_MAX
+};

package/src/monitor/queue.js

@@ -13,13 +13,14 @@ const { Worker } = require('worker_threads');
 const { runSandbox, tryAcquireSandboxSlot } = require('../sandbox/index.js');
 const { sendWebhook } = require('../webhook.js');
 const { downloadToFile, extractArchive, sanitizePackageName } = require('../shared/download.js');
-const { MAX_TARBALL_SIZE } = require('../shared/constants.js');
+const { MAX_TARBALL_SIZE, getMaxFileSize } = require('../shared/constants.js');
 const { acquireRegistrySlot, releaseRegistrySlot } = require('../shared/http-limiter.js');
 const { loadCachedIOCs } = require('../ioc/updater.js');
 const { scanPackageJson } = require('../scanner/package.js');
 const { scanShellScripts } = require('../scanner/shell.js');
 const { buildTrainingRecord } = require('../ml/feature-extractor.js');
 const { appendWorkerMem } = require('./worker-mem.js');
+const { acquireHeavySlot, releaseHeavySlot, isHeavyScan, getHeavyLaneState, heavyWaitMaxMs, HEAVY_REQUEUE_MAX } = require('./heavy-lane.js');
 const { appendRecord: appendTrainingRecord, relabelRecords } = require('../ml/jsonl-writer.js');
 
 // From ./state.js
@@ -305,6 +306,98 @@ function countPackageFiles(dir) {
   return { fileCountTotal, hasTests };
 }
 
+// C2 heavy-lane measurement bounds. Distinct from countPackageFiles (whose
+// depth cap of 5 is an ML-feature contract — do not touch it).
+const JS_WEIGHT_MAX_DEPTH = 8;
+const JS_WEIGHT_MAX_FILES = 2000;
+const JS_WEIGHT_FILE_PATTERN = /\.(?:[cm]?js|[jt]sx?)$/i;
+// Minified JS expands SUPER-linearly in the worker (live counter-examples
+// from the 16:18 rollout, 2026-06-11: powerlines = 517KB JS of which 449KB
+// minified → 1151MB heap, ~2300×; @lethevimlet/sshift = ~1.9MB minified →
+// 1.38GB — both sailed under the raw-bytes threshold as 'light'). Plain
+// source stays roughly linear (the 12MB-heap mode of the bimodal
+// distribution). So minified bytes count ×12 toward the heavy threshold —
+// ≥~256KB of minified JS crosses the 3MiB default. Detection: average line
+// length over the first 4KB; plain code sits at 40-120 chars, minified
+// bundles at 800+ (often a single line). 250 splits cleanly even when a
+// license header pads the probe window.
+const JS_MINIFIED_WEIGHT = 12;
+const JS_MINIFIED_AVG_LINE = 250;
+const JS_MINIFIED_PROBE_BYTES = 4096;
+
+/** Probe the first 4KB of a file (never loads the rest) for minification. */
+function probeIsMinified(filePath) {
+  let fd = null;
+  try {
+    fd = fs.openSync(filePath, 'r');
+    const buf = Buffer.alloc(JS_MINIFIED_PROBE_BYTES);
+    const n = fs.readSync(fd, buf, 0, JS_MINIFIED_PROBE_BYTES, 0);
+    if (n <= 0) return false;
+    const head = buf.toString('utf8', 0, n);
+    return (head.length / head.split('\n').length) > JS_MINIFIED_AVG_LINE;
+  } catch {
+    return false;
+  } finally {
+    if (fd !== null) { try { fs.closeSync(fd); } catch { /* best-effort */ } }
+  }
+}
+
+/**
+ * Measure how much parsable JS a package carries — the heavy-lane
+ * classification signal. The per-worker isolate heap is driven by the SUM of
+ * AST-parsed JS bytes (executor.js skips files > getMaxFileSize()
+ * individually, but the AST cache accumulates across files), so we sum the
+ * on-disk sizes of parsable JS files, skipping the ones the executor will
+ * skip anyway. NEVER use meta.unpackedSize for this — it is absent for PyPI
+ * and part of npm (the `|| 0` hole that lets giant bundles bypass the C1
+ * size-cap in the first place).
+ *
+ * Bounded walk; an overflow (depth/file caps) returns truncated:true, which
+ * isHeavyScan classifies heavy by default.
+ *
+ * weightedJsBytes = plain bytes + JS_MINIFIED_WEIGHT × minified bytes — the
+ * value isHeavyScan compares against the threshold (raw bytes alone missed
+ * the minified explosions, see JS_MINIFIED_WEIGHT above).
+ *
+ * @param {string} dir - extracted package directory
+ * @returns {{ totalJsBytes: number, minifiedJsBytes: number, weightedJsBytes: number, maxJsFileBytes: number, truncated: boolean }}
+ */
+function measureJsWeight(dir) {
+  let totalJsBytes = 0;
+  let minifiedJsBytes = 0;
+  let maxJsFileBytes = 0;
+  let seen = 0;
+  let truncated = false;
+  const perFileCap = getMaxFileSize();
+
+  function walk(current, depth) {
+    if (truncated) return;
+    if (depth > JS_WEIGHT_MAX_DEPTH) { truncated = true; return; }
+    let entries;
+    try { entries = fs.readdirSync(current, { withFileTypes: true }); } catch { return; }
+    for (const entry of entries) {
+      if (truncated) return;
+      if (entry.isDirectory()) {
+        if (ML_EXCLUDED_DIRS.has(entry.name)) continue;
+        walk(path.join(current, entry.name), depth + 1);
+      } else if (entry.isFile() && JS_WEIGHT_FILE_PATTERN.test(entry.name)) {
+        if (++seen > JS_WEIGHT_MAX_FILES) { truncated = true; return; }
+        const filePath = path.join(current, entry.name);
+        let size;
+        try { size = fs.statSync(filePath).size; } catch { continue; }
+        if (size > perFileCap) continue; // executor skips these — they never reach the AST
+        totalJsBytes += size;
+        if (probeIsMinified(filePath)) minifiedJsBytes += size;
+        if (size > maxJsFileBytes) maxJsFileBytes = size;
+      }
+    }
+  }
+
+  walk(dir, 0);
+  const weightedJsBytes = (totalJsBytes - minifiedJsBytes) + JS_MINIFIED_WEIGHT * minifiedJsBytes;
+  return { totalJsBytes, minifiedJsBytes, weightedJsBytes, maxJsFileBytes, truncated };
+}
+
 /**
  * Pure classifier: is this a prebuilt native-binary platform shard (the kind that
  * hangs the sandbox install and always times out INCONCLUSIVE)? No I/O — the parsed
@@ -435,6 +528,7 @@ function runScanInWorker(extractedDir, timeoutMs, scanContext = null, signal = n
     appendWorkerMem({
       ev: 'spawn', tid: _wmTid,
       name: _sc.name, version: _sc.version, ecosystem: _sc.ecosystem,
+      lane: _sc._lane, jsBytes: _sc._jsBytes, jsMin: _sc._jsMin,
       rss: process.memoryUsage().rss
     });
 
@@ -645,6 +739,18 @@ async function scanPackage(name, version, ecosystem, tarballUrl, registryMeta, s
     // ML Phase 2a: Count JS files and detect test presence for enriched features
     const { fileCountTotal, hasTests } = countPackageFiles(extractedDir);
 
+    // C2 heavy-lane classification (see heavy-lane.js header): measured on
+    // disk, after extraction — registry metadata is not trustworthy here.
+    // Measurement failure falls back to the compressed tarball size
+    // (conservative: never silently far under the real JS weight).
+    let jsWeight;
+    try {
+      jsWeight = measureJsWeight(extractedDir);
+    } catch {
+      jsWeight = { totalJsBytes: fileSize, maxJsFileBytes: 0, truncated: false };
+    }
+    const lane = isHeavyScan(jsWeight) ? 'heavy' : 'light';
+
     // Hoisted before the worker spawn (per-worker 429-storm fix): fetch the npm
     // registry metadata ONCE on the main thread. The shared http-limiter coordinates
     // it and the temporal cache is warm (npm-registry.js reads it first), so only
@@ -663,6 +769,28 @@ async function scanPackage(name, version, ecosystem, tarballUrl, registryMeta, s
       }
     }
 
+    // C2 heavy-lane: serialize the memory-heavy scans. Acquired AFTER the
+    // registry fetch above (never hold the slot during network I/O); released
+    // in the finally right after the static scan — the slot covers ONLY the
+    // worker's lifetime (≤ STATIC_SCAN_TIMEOUT_MS), not the sandbox (which
+    // has its own semaphore and runs outside the daemon's heap).
+    let heavySlotHeld = false;
+    if (lane === 'heavy') {
+      stats.heavyScans = (stats.heavyScans || 0) + 1;
+      const laneState = getHeavyLaneState();
+      if (laneState.max > 0 && laneState.active >= laneState.max) {
+        stats.heavyLaneWaits = (stats.heavyLaneWaits || 0) + 1;
+        console.log(`[MONITOR] HEAVY_LANE: ${name}@${version} waiting for a slot (${(jsWeight.totalJsBytes / 1024 / 1024).toFixed(1)}MB JS, active=${laneState.active}, waiting=${laneState.waiting})`);
+      }
+      // After HEAVY_REQUEUE_MAX requeues the final pass waits unbounded
+      // (abort-aware only, still under the outer SCAN_TIMEOUT_MS) so an item
+      // cannot loop in the queue forever.
+      const lastPass = (meta._heavyRetries || 0) >= HEAVY_REQUEUE_MAX;
+      const waitStart = Date.now();
+      heavySlotHeld = await acquireHeavySlot({ signal, maxWaitMs: lastPass ? 0 : heavyWaitMaxMs() });
+      stats.heavyLaneWaitMsTotal = (stats.heavyLaneWaitMsTotal || 0) + (Date.now() - waitStart);
+    }
+
     let result;
     try {
       // scanContext: feeds monitor-side info (name/version/ecosystem) and the
@@ -679,7 +807,13 @@ async function scanPackage(name, version, ecosystem, tarballUrl, registryMeta, s
         // Stage 2: set by processQueueItem when MUADDIB_TRIAGE_MODE=enforce.
         // Defaults to 'full' so any CLI/test caller that bypasses triage gets
         // the full 20-scanner pipeline (unchanged behaviour).
-        scanMode: (meta && meta.scanMode) || 'full'
+        scanMode: (meta && meta.scanMode) || 'full',
+        // C2 observability: lane + JS weight flow into the worker-mem spawn
+        // event (runScanInWorker) so lane×heap-peak cross-checks are possible
+        // post-rollout (hard criterion: zero 'light' scans peaking >512MB).
+        _lane: lane,
+        _jsBytes: jsWeight.totalJsBytes,
+        _jsMin: jsWeight.minifiedJsBytes || 0
       };
       // Hand the main-thread-fetched metadata to the worker so its processor skips
       // the per-worker getPackageMetadata fetch (429-storm fix). npm only; the key
@@ -705,6 +839,10 @@ async function scanPackage(name, version, ecosystem, tarballUrl, registryMeta, s
         return { sandboxResult: null, staticClean: false };
       }
       throw staticErr;
+    } finally {
+      // Single release point — success, static timeout, EMERGENCY terminate
+      // and abort all funnel through here exactly once (heavySlotHeld guard).
+      if (heavySlotHeld) { releaseHeavySlot(); heavySlotHeld = false; }
     }
 
     // Phase 3 signal — agent-supply-chain lens. Pure observability, no scoring impact.
@@ -1285,6 +1423,11 @@ async function scanPackage(name, version, ecosystem, tarballUrl, registryMeta, s
       }
     }
   } catch (err) {
+    // C2 heavy-lane: a wait-timeout is NOT a scan failure — processQueueItem
+    // requeues the item (bounded by HEAVY_REQUEUE_MAX). Re-throw BEFORE any
+    // error accounting: this catch otherwise swallows everything into the
+    // 'scan_error' ledger path and the requeue would never happen.
+    if (err && err.code === 'HEAVY_LANE_WAIT_TIMEOUT') throw err;
     recordError(err, stats);
     stats.scanned++;
     stats.totalTimeMs += Date.now() - startTime;
@@ -1376,6 +1519,22 @@ async function processQueueItem(item, stats, dailyAlerts, recentlyScanned, downl
       })
     ]);
   } catch (err) {
+    // C2 heavy-lane: the bounded wait expired while the heavy slots were
+    // saturated (typical under a spill-drain burst). Not a failure — put the
+    // item back at the queue tail (natural backoff) up to HEAVY_REQUEUE_MAX
+    // passes; scanPackage runs the final pass without the wait bound. Note:
+    // _heavyRetries does not survive a spill (spillItems strips non-re-enqueue
+    // fields) — acceptable, the spill drain runs in calm windows anyway.
+    if (err && err.code === 'HEAVY_LANE_WAIT_TIMEOUT') {
+      const decision = computeHeavyRequeue(item);
+      if (decision.requeue) {
+        stats.heavyLaneRequeues = (stats.heavyLaneRequeues || 0) + 1;
+        console.log(`[MONITOR] HEAVY_LANE: requeued ${item.name}@${item.version || '?'} (wait-timeout pass ${decision.retries}/${HEAVY_REQUEUE_MAX})`);
+        enqueueScan(scanQueue, item, stats);
+        return;
+      }
+      // Safety net — should be unreachable (the last pass waits unbounded).
+    }
     recordError(err, stats);
     console.error(`[MONITOR] Queue error for ${item.name}: ${err.message}`);
     // IOC fallback: if scan failed for a known malicious package, send P1 alert.
@@ -1450,6 +1609,18 @@ function computeWorkersToSpawn(targetConcurrency, activeWorkers, queueLength) {
   return Math.max(0, Math.min(targetConcurrency - activeWorkers, queueLength));
 }
 
+/**
+ * Pure requeue decision for a heavy-lane wait-timeout (same extraction
+ * rationale as computeWorkersToSpawn). Mutates item._heavyRetries; once the
+ * counter passes HEAVY_REQUEUE_MAX the item is NOT requeued again — its next
+ * pass through scanPackage waits unbounded instead.
+ */
+function computeHeavyRequeue(item) {
+  const retries = (item._heavyRetries || 0) + 1;
+  item._heavyRetries = retries;
+  return { requeue: retries <= HEAVY_REQUEUE_MAX, retries };
+}
+
 // ── RSS-aware worker admission (P1 OOM durable fix) ──
 // The pressure breaker is reactive: it stops spawning at HIGH, but the workers already in
 // flight overshoot RSS by ~2GB (each isolate + gVisor sandbox ~0.55GB, draining up to
@@ -1795,7 +1966,10 @@ async function resolveTarballAndScan(item, stats, dailyAlerts, recentlyScanned,
     registryScripts: item.registryScripts || null,
     _cacheTrigger: item._cacheTrigger || null,
     fastTrack: item.fastTrack || false,
-    scanMode: effectiveScanMode
+    scanMode: effectiveScanMode,
+    // C2 heavy-lane: pass count set by computeHeavyRequeue — at
+    // HEAVY_REQUEUE_MAX the final pass waits for its slot unbounded.
+    _heavyRetries: item._heavyRetries || 0
   }, stats, dailyAlerts, recentlyScanned, downloadsCache, scanQueue, sandboxAvailable, signal);
   const sandboxResult = scanResult && scanResult.sandboxResult;
   const staticClean = scanResult && scanResult.staticClean;
@@ -1917,6 +2091,8 @@ module.exports = {
   isBundledToolingOnly,
   recordTrainingSample,
   countPackageFiles,
+  measureJsWeight,
+  computeHeavyRequeue,
   classifyNativeShard,
   shouldSkipSandbox,
   runScanInWorker,