muaddib-scanner 2.11.95 → 2.11.97

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "muaddib-scanner",
-  "version": "2.11.95",
+  "version": "2.11.97",
   "description": "Supply-chain threat detection & response for npm & PyPI/Python",
   "main": "src/index.js",
   "bin": {

package/{self-scan-v2.11.95.json → self-scan-v2.11.97.json}

@@ -1,6 +1,6 @@
 {
   "target": "node_modules",
-  "timestamp": "2026-06-11T13:00:12.001Z",
+  "timestamp": "2026-06-11T15:01:04.220Z",
   "threats": [
     {
       "type": "string_mutation_obfuscation",

package/src/monitor/queue.js

@@ -19,6 +19,7 @@ const { loadCachedIOCs } = require('../ioc/updater.js');
 const { scanPackageJson } = require('../scanner/package.js');
 const { scanShellScripts } = require('../scanner/shell.js');
 const { buildTrainingRecord } = require('../ml/feature-extractor.js');
+const { appendWorkerMem } = require('./worker-mem.js');
 const { appendRecord: appendTrainingRecord, relabelRecords } = require('../ml/jsonl-writer.js');
 
 // From ./state.js
@@ -426,6 +427,17 @@ function runScanInWorker(extractedDir, timeoutMs, scanContext = null, signal = n
     const _sc = scanContext || {};
     _liveWorkers.set(worker, { name: _sc.name, version: _sc.version, ecosystem: _sc.ecosystem });
 
+    // Off-heap attribution (worker-mem.jsonl, gated MUADDIB_WORKER_MEM=1):
+    // process RSS around each worker's lifetime. tid captured now — after
+    // 'exit' worker.threadId becomes -1.
+    const _wmTid = worker.threadId;
+    const _wmSpawnedAt = Date.now();
+    appendWorkerMem({
+      ev: 'spawn', tid: _wmTid,
+      name: _sc.name, version: _sc.version, ecosystem: _sc.ecosystem,
+      rss: process.memoryUsage().rss
+    });
+
     let settled = false;
     let timer = null;
     const done = (fn) => {
@@ -462,9 +474,19 @@ function runScanInWorker(extractedDir, timeoutMs, scanContext = null, signal = n
 
     worker.on('error', (err) => done(() => reject(err)));
 
-    worker.on('exit', (code) => done(() => {
-      if (code !== 0) reject(new Error(`Worker exited with code ${code}`));
-    }));
+    worker.on('exit', (code) => {
+      // 'exit' fires exactly once per worker (even after terminate/error), so
+      // it is the one reliable place to close the spawn/exit RSS pair.
+      appendWorkerMem({
+        ev: 'exit', tid: _wmTid,
+        name: _sc.name, version: _sc.version, code,
+        durMs: Date.now() - _wmSpawnedAt,
+        rss: process.memoryUsage().rss
+      });
+      done(() => {
+        if (code !== 0) reject(new Error(`Worker exited with code ${code}`));
+      });
+    });
   });
 }
 

package/src/monitor/spill.js

@@ -83,9 +83,24 @@ function _writeEntries(file, entries) {
   fs.renameSync(tmp, file);
 }
 
+// Conservative upper bound on a spilled line's byte size (the SPILL_FIELDS
+// record + ts + newline run ~150-250 bytes). Used for the O(1) stat-gated
+// compaction trigger below — overestimating only makes compaction fire a bit
+// late, never early, so the file ceiling stays ~MUADDIB_SPILL_MAX.
+const SPILL_LINE_BYTES_EST = 256;
+
 /**
  * Append evicted queue items to the backlog. Never throws; on write failure the
  * caller's fallback is the pre-spill behavior (drop, ledgered).
+ *
+ * HOT PATH — runs INSIDE the EMERGENCY memory handler (evictFromScanQueueBulk),
+ * so it MUST be append-only and allocation-free beyond the write buffer. The
+ * 2026-06-11 freeze was caused by calling _compactBacklog (which reads + parses
+ * the WHOLE backlog) on every spill: a large allocation during a reclaim stall
+ * that wedged the handler before it could free RSS. Compaction now fires ONLY
+ * when a cheap statSync shows the file is genuinely near the cap (normally
+ * never during an EMERGENCY — the backlog is far below the byte budget there),
+ * and the calm-time drain also keeps it bounded.
  * @param {Array<object>} items evicted scan-queue items
  * @returns {number} how many items were actually persisted
  */
@@ -107,7 +122,11 @@ function spillItems(items) {
       written++;
     }
     if (buf) fs.appendFileSync(file, buf, 'utf8');
-    _compactBacklog(file);
+    // O(1) stat-gated compaction: only read+rewrite the file when it is actually
+    // near the cap. NO whole-file read on the normal EMERGENCY spill path.
+    let size = 0;
+    try { size = fs.statSync(file).size; } catch { /* fresh file */ }
+    if (size > _maxEntries() * SPILL_LINE_BYTES_EST) _compactBacklog(file);
   } catch {
     return 0; // degrade to drop-with-ledger at the call site
   }

package/src/monitor/worker-mem.js

@@ -0,0 +1,72 @@
+'use strict';
+
+/**
+ * Per-worker memory instrumentation (off-heap RSS attribution, 2026-06).
+ *
+ * The EMERGENCY breaker fires on process RSS while the heap sits at ~15% —
+ * the driver is off-heap (malloc arenas + tarball Buffers) and mem-trend.jsonl
+ * only samples the whole process. This module attributes memory to individual
+ * scan workers / packages so the worker_threads → child_process decision can
+ * be made on data:
+ *   H1: RSS stays high AFTER workers die  → arenas never returned to the OS
+ *   H2: RSS peaks only WHILE workers live → concurrent in-flight peak
+ *
+ * Producers:
+ *   - queue.js (parent): ev:'spawn' / ev:'exit' around each scan worker,
+ *     with process-wide RSS (delta attributable per package, noisy but
+ *     aggregable over 24-48h).
+ *   - scan-worker.js (worker): ev:'sample' every sampleIntervalMs() with the
+ *     isolate-local heapUsed/external/arrayBuffers (rss there is process-wide).
+ *
+ * Same hot-path safety rules as spill.js (2026-06-11 prod-freeze lesson):
+ * append-only, stat-gated O(1) rotation, never read the file back, never throw.
+ * OFF unless MUADDIB_WORKER_MEM=1 (staged rollout, same pattern as
+ * MUADDIB_WORKER_MAX_OLD_MB) so tests and CLI runs never touch data/.
+ */
+
+const fs = require('fs');
+const path = require('path');
+
+const DEFAULT_FILE = path.join(__dirname, '..', '..', 'data', 'worker-mem.jsonl');
+const DEFAULT_MAX_MB = 64;        // rotate past 64MB (file + .1 = 128MB worst case, ~2.5 days at concurrency 8)
+const DEFAULT_SAMPLE_MS = 10000;  // per-worker isolate sample cadence
+
+function workerMemEnabled() {
+  return process.env.MUADDIB_WORKER_MEM === '1';
+}
+
+function workerMemFile() {
+  return process.env.MUADDIB_WORKER_MEM_FILE || DEFAULT_FILE;
+}
+
+/** 0 = sampling disabled (instrumentation off, or explicit MUADDIB_WORKER_MEM_SAMPLE_MS=0). */
+function sampleIntervalMs() {
+  if (!workerMemEnabled()) return 0;
+  const v = parseInt(process.env.MUADDIB_WORKER_MEM_SAMPLE_MS, 10);
+  if (Number.isFinite(v) && v >= 0) return v;
+  return DEFAULT_SAMPLE_MS;
+}
+
+/**
+ * Append one instrumentation entry (ts stamped here). Bounded resource
+ * (CLAUDE.md §2): stat-gated truncate-rotate, no read-back on the hot path.
+ * @returns {boolean} true if a line was written
+ */
+function appendWorkerMem(entry) {
+  if (!workerMemEnabled()) return false;
+  try {
+    const file = workerMemFile();
+    const maxMb = parseInt(process.env.MUADDIB_WORKER_MEM_MAX_MB, 10);
+    const maxBytes = (Number.isFinite(maxMb) && maxMb > 0 ? maxMb : DEFAULT_MAX_MB) * 1024 * 1024;
+    try {
+      const st = fs.statSync(file);
+      if (st.size > maxBytes) fs.renameSync(file, file + '.1');
+    } catch { /* no file yet — fine */ }
+    fs.appendFileSync(file, JSON.stringify({ ts: new Date().toISOString(), ...entry }) + '\n', 'utf8');
+    return true;
+  } catch { /* instrumentation must never crash the daemon or a worker */
+    return false;
+  }
+}
+
+module.exports = { appendWorkerMem, sampleIntervalMs, workerMemEnabled, workerMemFile };

package/src/pipeline/scan-worker.js

@@ -12,7 +12,7 @@
  *   parentPort.postMessage({ type: 'error', message: string })
  */
 
-const { parentPort, workerData } = require('worker_threads');
+const { parentPort, workerData, threadId } = require('worker_threads');
 
 if (!parentPort) {
   // Not running as a worker — exit gracefully
@@ -20,17 +20,38 @@ if (!parentPort) {
 }
 
 const { run } = require('../index.js');
+const { appendWorkerMem, sampleIntervalMs } = require('../monitor/worker-mem.js');
 
 (async () => {
+  // Off-heap attribution samples (worker-mem.jsonl): heapUsed/external/
+  // arrayBuffers are isolate-local here, rss is process-wide. The samples MUST
+  // NOT go through parentPort — the parent settles the scan promise on the
+  // first message it receives (queue.js done()), so a sample message would
+  // hang the scan forever. unref() so the timer never keeps the worker alive.
+  const scanContext = workerData.scanContext || {};
+  const everyMs = sampleIntervalMs();
+  let sampler = null;
+  if (everyMs > 0) {
+    sampler = setInterval(() => {
+      const m = process.memoryUsage();
+      appendWorkerMem({
+        ev: 'sample', tid: threadId,
+        name: scanContext.name, version: scanContext.version,
+        heapUsed: m.heapUsed, external: m.external, arrayBuffers: m.arrayBuffers, rss: m.rss
+      });
+    }, everyMs);
+    sampler.unref();
+  }
   try {
     // scanContext (optional) carries monitor-side info that opt-in scanners need
     // (e.g. trusted-dep-diff requires package name + version to query the registry).
     // It is spread INTO the pipeline options, but `_capture: true` always wins so
     // the worker keeps returning the result object — never prints.
-    const scanContext = workerData.scanContext || {};
     const result = await run(workerData.extractedDir, { ...scanContext, _capture: true });
     parentPort.postMessage({ type: 'result', data: result });
   } catch (err) {
     parentPort.postMessage({ type: 'error', message: err.message || String(err) });
+  } finally {
+    if (sampler) clearInterval(sampler);
   }
 })();