muaddib-scanner 2.11.162 → 2.11.164

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "muaddib-scanner",
-  "version": "2.11.162",
+  "version": "2.11.164",
   "description": "Supply-chain threat detection & response for npm & PyPI/Python",
   "main": "src/index.js",
   "bin": {

package/{self-scan-v2.11.162.json → self-scan-v2.11.164.json}

@@ -1,6 +1,6 @@
 {
   "target": "node_modules",
-  "timestamp": "2026-07-10T07:45:05.079Z",
+  "timestamp": "2026-07-10T09:30:28.827Z",
   "threats": [
     {
       "type": "string_mutation_obfuscation",

package/src/monitor/state.js

@@ -24,9 +24,22 @@
 
 const fs = require('fs');
 const path = require('path');
+const readline = require('readline');
 const { isMainThread, threadId } = require('worker_threads');
 const { sanitizePackageName } = require('../shared/download.js');
 
+// Event-loop stall attribution (event-loop-monitor.js): the periodic JSONL
+// compactions below are O(file-size) SYNCHRONOUS rewrites that run on the monitor
+// main thread. At the 500K-entry scan-ledger steady state this is the leading
+// suspect for the multi-minute loop stalls that blind the RSS breaker — yet it was
+// uninstrumented, so a stall in it mis-attributed to a stale prior breadcrumb
+// (docker:rm with durationMs << lagMs). Wrapping each compaction NAMES it: a stall
+// now logs op.label='compact:*' with durationMs ~ lagMs. Guarded require so state.js
+// stays usable from the CLI / scan-worker where the sampler isn't running
+// (runInstrumented is then a transparent passthrough). Never throws.
+let _runInstrumented = (label, meta, fn) => fn();
+try { ({ runInstrumented: _runInstrumented } = require('./event-loop-monitor.js')); } catch { /* sampler unavailable — passthrough */ }
+
 // --- File path constants ---
 
 const STATE_FILE = path.join(__dirname, '..', '..', 'data', 'monitor-state.json');
@@ -761,23 +774,26 @@ function loadTemporalDetections() {
  * runStateMigrations to enforce caps after migration.
  */
 function _compactTemporalDetectionsJsonl() {
-  try {
-    const total = _countJsonlLines(TEMPORAL_DETECTIONS_FILE);
-    if (total <= MAX_TEMPORAL_DETECTIONS) return;
-    const toDrop = total - MAX_TEMPORAL_DETECTIONS;
-    let skipped = 0;
-    const kept = [];
-    _iterateJsonlSync(TEMPORAL_DETECTIONS_FILE, (entry) => {
-      if (skipped < toDrop) { skipped++; return; }
-      kept.push(JSON.stringify(entry));
-    });
-    const tmpFile = TEMPORAL_DETECTIONS_FILE + '.tmp';
-    fs.writeFileSync(tmpFile, kept.length ? kept.join('\n') + '\n' : '', 'utf8');
-    fs.renameSync(tmpFile, TEMPORAL_DETECTIONS_FILE);
-    console.log(`[MONITOR] COMPACT temporal-detections: ${total} -> ${kept.length} entries`);
-  } catch (err) {
-    console.error(`[MONITOR] Temporal detections compaction failed: ${err.message}`);
-  }
+  // Same O(file) synchronous rewrite shape as the scan-ledger/detections compactions.
+  _runInstrumented('compact:temporal-detections', { cap: MAX_TEMPORAL_DETECTIONS }, () => {
+    try {
+      const total = _countJsonlLines(TEMPORAL_DETECTIONS_FILE);
+      if (total <= MAX_TEMPORAL_DETECTIONS) return;
+      const toDrop = total - MAX_TEMPORAL_DETECTIONS;
+      let skipped = 0;
+      const kept = [];
+      _iterateJsonlSync(TEMPORAL_DETECTIONS_FILE, (entry) => {
+        if (skipped < toDrop) { skipped++; return; }
+        kept.push(JSON.stringify(entry));
+      });
+      const tmpFile = TEMPORAL_DETECTIONS_FILE + '.tmp';
+      fs.writeFileSync(tmpFile, kept.length ? kept.join('\n') + '\n' : '', 'utf8');
+      fs.renameSync(tmpFile, TEMPORAL_DETECTIONS_FILE);
+      console.log(`[MONITOR] COMPACT temporal-detections: ${total} -> ${kept.length} entries`);
+    } catch (err) {
+      console.error(`[MONITOR] Temporal detections compaction failed: ${err.message}`);
+    }
+  });
 }
 
 // --- State persistence ---
@@ -996,28 +1012,32 @@ function getDetectionStats() {
  * stays consistent. No-op when the file is already under cap.
  */
 function _compactDetectionsJsonl() {
-  try {
-    const total = _countJsonlLines(DETECTIONS_FILE);
-    if (total <= MAX_DETECTIONS) return;
-    const toDrop = total - MAX_DETECTIONS;
-    let skipped = 0;
-    const kept = [];
-    const newDedup = new Set();
-    _iterateJsonlSync(DETECTIONS_FILE, (entry) => {
-      if (skipped < toDrop) { skipped++; return; }
-      kept.push(JSON.stringify(entry));
-      if (entry && entry.package && entry.version) {
-        newDedup.add(`${entry.package}@${entry.version}`);
-      }
-    });
-    const tmpFile = DETECTIONS_FILE + '.tmp';
-    fs.writeFileSync(tmpFile, kept.length ? kept.join('\n') + '\n' : '', 'utf8');
-    fs.renameSync(tmpFile, DETECTIONS_FILE);
-    _detectionDedupSet = newDedup;
-    console.log(`[MONITOR] COMPACT detections: ${total} -> ${kept.length} entries`);
-  } catch (err) {
-    console.error(`[MONITOR] Detections compaction failed: ${err.message}`);
-  }
+  // Same O(file) synchronous rewrite as the scan-ledger (smaller cap, but same
+  // main-thread blocking shape). Named so a stall here can't hide behind a stale crumb.
+  _runInstrumented('compact:detections', { cap: MAX_DETECTIONS }, () => {
+    try {
+      const total = _countJsonlLines(DETECTIONS_FILE);
+      if (total <= MAX_DETECTIONS) return;
+      const toDrop = total - MAX_DETECTIONS;
+      let skipped = 0;
+      const kept = [];
+      const newDedup = new Set();
+      _iterateJsonlSync(DETECTIONS_FILE, (entry) => {
+        if (skipped < toDrop) { skipped++; return; }
+        kept.push(JSON.stringify(entry));
+        if (entry && entry.package && entry.version) {
+          newDedup.add(`${entry.package}@${entry.version}`);
+        }
+      });
+      const tmpFile = DETECTIONS_FILE + '.tmp';
+      fs.writeFileSync(tmpFile, kept.length ? kept.join('\n') + '\n' : '', 'utf8');
+      fs.renameSync(tmpFile, DETECTIONS_FILE);
+      _detectionDedupSet = newDedup;
+      console.log(`[MONITOR] COMPACT detections: ${total} -> ${kept.length} entries`);
+    } catch (err) {
+      console.error(`[MONITOR] Detections compaction failed: ${err.message}`);
+    }
+  });
 }
 
 // --- Per-scan ledger (Phase 0a: operational coverage observability) ---
@@ -1035,6 +1055,12 @@ const MAX_SCAN_LEDGER = (() => {
 })();
 const SCAN_LEDGER_COMPACT_INTERVAL = 2000;
 let _scanLedgerAppendedSinceCompact = 0;
+// Async-compaction coordination. `_scanLedgerCompacting` guards re-entrancy (a second
+// trigger while a compaction is in flight is skipped — the cap is soft, the next
+// interval retries). The promise lets tests + graceful shutdown await an in-flight
+// compaction settling. See _compactScanLedgerJsonl (async, streaming, race-safe).
+let _scanLedgerCompacting = false;
+let _scanLedgerCompactionPromise = Promise.resolve();
 
 // Terminal outcomes a dequeued package can reach. Unknown values normalize to 'clean'
 // so a typo at a call site can never crash the pipeline.
@@ -1105,7 +1131,10 @@ function appendScanLedger(e) {
     _scanLedgerAppendedSinceCompact++;
     if (_scanLedgerAppendedSinceCompact >= SCAN_LEDGER_COMPACT_INTERVAL) {
       _scanLedgerAppendedSinceCompact = 0;
-      _compactScanLedgerJsonl();
+      // Fire the async compaction (non-blocking — keeps the RSS-breaker poll loop
+      // reactive). Guarded + fully caught internally so it never rejects; stored so
+      // tests and graceful shutdown can await an in-flight compaction settling.
+      _scanLedgerCompactionPromise = _compactScanLedgerJsonl();
     }
   } catch (err) {
     if (err.code === 'EROFS' || err.code === 'EACCES' || err.code === 'EPERM') return;
@@ -1121,23 +1150,96 @@ function appendScanLedger(e) {
  * Compact the scan-ledger JSONL: keep only the most recent MAX_SCAN_LEDGER entries.
  * No-op when already under cap. Streams (never loads the whole file at once).
  */
-function _compactScanLedgerJsonl() {
+// Stream-count newline-terminated lines in [0, endByte) without loading the file or
+// parsing JSON. Async — yields between 64KB chunks so it never wedges the poll loop.
+function _streamCountLines(filePath, endByte) {
+  return new Promise((resolve, reject) => {
+    if (!endByte || endByte <= 0) { resolve(0); return; }
+    let count = 0;
+    const rs = fs.createReadStream(filePath, { start: 0, end: endByte - 1 });
+    rs.on('error', (e) => ((e && e.code === 'ENOENT') ? resolve(0) : reject(e)));
+    rs.on('data', (buf) => { for (let i = 0; i < buf.length; i++) if (buf[i] === 0x0a) count++; });
+    rs.on('end', () => resolve(count));
+  });
+}
+
+// Stream [0, endByte): skip the first `skip` lines, copy the rest VERBATIM (no parse,
+// no re-stringify) to tmp. Bounded memory: one line at a time + write backpressure
+// (pause the reader until the write buffer drains). Returns the kept line count.
+function _streamKeepTailToTmp(filePath, skip, endByte, tmpFile) {
+  return new Promise((resolve, reject) => {
+    let seen = 0, kept = 0, failed = false, draining = false;
+    const rs = fs.createReadStream(filePath, { encoding: 'utf8', start: 0, end: endByte - 1 });
+    const ws = fs.createWriteStream(tmpFile, { encoding: 'utf8' }); // truncates any orphan tmp
+    const rl = readline.createInterface({ input: rs, crlfDelay: Infinity });
+    const fail = (e) => { if (failed) return; failed = true; try { rl.close(); } catch { /* */ } try { rs.destroy(); } catch { /* */ } try { ws.destroy(); } catch { /* */ } reject(e); };
+    rs.on('error', fail); ws.on('error', fail);
+    rl.on('line', (line) => {
+      if (seen++ < skip) return;
+      if (!line) return;
+      const ok = ws.write(line + '\n');
+      kept++;
+      // Backpressure: pause the reader until the write buffer drains. The `draining`
+      // guard is required — readline emits lines ALREADY buffered from the current
+      // chunk synchronously even after pause(), so without it each such line would
+      // stack another 'drain' listener (MaxListenersExceededWarning + redundant resumes).
+      if (!ok && !draining) {
+        draining = true;
+        rl.pause();
+        ws.once('drain', () => { draining = false; rl.resume(); });
+      }
+    });
+    rl.on('close', () => { if (!failed) ws.end(() => resolve(kept)); });
+  });
+}
+
+// Async, streaming, race-safe compaction of the scan-ledger. Replaces the old
+// SYNCHRONOUS 127MB rewrite that wedged the main-thread poll loop for minutes — and
+// because the RSS circuit breaker runs on that same loop, a wedged loop is a BLIND
+// breaker → RSS climbs unchecked → cgroup OOM. Design (why it can't lose appends):
+//   • two async streaming passes over the immutable snapshot [0, preSize): count, then
+//     copy the last MAX lines to a .tmp. Bounded memory (no 500K-entry array / 127MB
+//     join). Non-blocking: appends keep flowing to the LIVE file during these passes.
+//   • the ONLY destructive step — fold the live tail [preSize, nowSize) onto .tmp, then
+//     rename .tmp over the ledger — runs with NO `await` between the tail read and the
+//     rename, so it is ATOMIC w.r.t. the single-threaded event loop: no concurrent
+//     appendScanLedger can interleave, hence the rename can never clobber an un-folded
+//     append. (appendFileSync writes whole lines, so preSize is always line-aligned.)
+//   • crash-safe: the live ledger is untouched until the atomic rename; a crash leaves
+//     it intact plus a stale .tmp (overwritten next run). No sidecar, no recovery hook.
+// Never throws (fire-and-forget from appendScanLedger).
+async function _compactScanLedgerJsonl() {
+  if (_scanLedgerCompacting) return;              // re-entrancy guard
+  _scanLedgerCompacting = true;
+  const tmpFile = SCAN_LEDGER_FILE + '.tmp';
   try {
-    const total = _countJsonlLines(SCAN_LEDGER_FILE);
+    if (!fs.existsSync(SCAN_LEDGER_FILE)) return;
+    const preSize = fs.statSync(SCAN_LEDGER_FILE).size; // immutable snapshot boundary
+    if (preSize === 0) return;
+    const total = await _streamCountLines(SCAN_LEDGER_FILE, preSize);
     if (total <= MAX_SCAN_LEDGER) return;
     const toDrop = total - MAX_SCAN_LEDGER;
-    let skipped = 0;
-    const kept = [];
-    _iterateJsonlSync(SCAN_LEDGER_FILE, (entry) => {
-      if (skipped < toDrop) { skipped++; return; }
-      kept.push(JSON.stringify(entry));
-    });
-    const tmpFile = SCAN_LEDGER_FILE + '.tmp';
-    fs.writeFileSync(tmpFile, kept.length ? kept.join('\n') + '\n' : '', 'utf8');
-    fs.renameSync(tmpFile, SCAN_LEDGER_FILE);
-    console.log(`[MONITOR] COMPACT scan-ledger: ${total} -> ${kept.length} entries`);
+    const t0 = Date.now();
+    const kept = await _streamKeepTailToTmp(SCAN_LEDGER_FILE, toDrop, preSize, tmpFile);
+    // ── SYNCHRONOUS finalize (atomic w.r.t. the event loop — no await below) ──
+    const nowSize = fs.statSync(SCAN_LEDGER_FILE).size;
+    let foldedBytes = 0;
+    if (nowSize > preSize) {                        // appends landed during the async passes
+      const fd = fs.openSync(SCAN_LEDGER_FILE, 'r');
+      try {
+        foldedBytes = nowSize - preSize;
+        const tail = Buffer.allocUnsafe(foldedBytes);
+        fs.readSync(fd, tail, 0, foldedBytes, preSize);
+        fs.appendFileSync(tmpFile, tail);           // preserve concurrent appends (whole lines)
+      } finally { fs.closeSync(fd); }
+    }
+    fs.renameSync(tmpFile, SCAN_LEDGER_FILE);        // atomic swap
+    console.log(`[MONITOR] COMPACT scan-ledger: ${total} -> ${kept} entries (async, ${Date.now() - t0}ms, +${foldedBytes}B live tail)`);
   } catch (err) {
     console.error(`[MONITOR] Scan-ledger compaction failed: ${err.message}`);
+    try { fs.unlinkSync(tmpFile); } catch { /* nothing to clean */ }
+  } finally {
+    _scanLedgerCompacting = false;
   }
 }
 
@@ -1856,6 +1958,7 @@ module.exports = {
   loadScanLedger,
   computeLedgerRollup,
   _compactScanLedgerJsonl,
+  _whenScanLedgerCompactionSettles: () => _scanLedgerCompactionPromise,
   getDetectionStats,
   runStateMigrations,
   // Internal — exported for tests and for the daemon hourly housekeeping.

package/src/sandbox/index.js

@@ -59,6 +59,18 @@ function dockerSync(label, meta, fn) {
   try { return fn(); } finally { _endOp(_crumb); }
 }
 
+// Same breadcrumb mechanism as dockerSync, for the SYNCHRONOUS post-container
+// result-handling that also runs on the main thread (the deferred sandbox worker is a
+// setInterval on the main loop — deferred-sandbox.js): JSON.parse of container stdout,
+// gVisor kernel-log parse/cleanup, preload-log analysis. Uninstrumented today, so a
+// multi-minute stall in any of them mis-attributes to the last docker:* crumb (the
+// docker:rm-leurre seen in data/loop-stalls.jsonl with durationMs << lagMs). Leaf-only:
+// the wrapped fn must NOT nest another beginOp/dockerSync (single-slot model).
+function syncStep(label, meta, fn) {
+  const _crumb = _beginOp(label, meta || null);
+  try { return fn(); } finally { _endOp(_crumb); }
+}
+
 const DOCKER_IMAGE = 'muaddib-sandbox';
 const CONTAINER_TIMEOUT = 120000; // 120 seconds
 const SINGLE_RUN_TIMEOUT = 90000; // 90 seconds per run in multi-run mode (gVisor ~30% I/O overhead)
@@ -595,7 +607,7 @@ async function runSingleSandbox(packageName, options = {}) {
           }
           jsonStr = stdout.substring(jsonStart, jsonEnd + 1);
         }
-        report = JSON.parse(jsonStr);
+        report = syncStep('sandbox:parse-report', { bytes: jsonStr.length }, () => JSON.parse(jsonStr));
         if (local && report) {
           report.package = displayName;
         }
@@ -636,7 +648,7 @@ async function runSingleSandbox(packageName, options = {}) {
       // connections, and process data come from gVisor's kernel-level tracing.
       if (gvisorMode && gvisorContainerId) {
         const gvisorLogDir = process.env.MUADDIB_GVISOR_LOG_DIR || '/tmp/runsc';
-        const gvisorData = parseGvisorLogs(gvisorContainerId, gvisorLogDir);
+        const gvisorData = syncStep('sandbox:gvisor-parse', { container: gvisorContainerId }, () => parseGvisorLogs(gvisorContainerId, gvisorLogDir));
 
         // Merge gVisor findings into report without duplicating
         if (!report.sensitive_files) report.sensitive_files = { read: [], written: [] };
@@ -665,14 +677,14 @@ async function runSingleSandbox(packageName, options = {}) {
         }
 
         // Cleanup gVisor logs to prevent disk fill
-        cleanupGvisorLogs(gvisorContainerId, gvisorLogDir);
+        syncStep('sandbox:gvisor-cleanup', { container: gvisorContainerId }, () => cleanupGvisorLogs(gvisorContainerId, gvisorLogDir));
       }
 
       const { score, findings } = scoreFindings(report);
 
       // Analyze preload log for behavioral findings
       if (report.preload_log) {
-        const preloadResult = analyzePreloadLog(report.preload_log);
+        const preloadResult = syncStep('sandbox:preload-analyze', null, () => analyzePreloadLog(report.preload_log));
         for (const finding of preloadResult.findings) {
           findings.push(finding);
         }

package/stats.json

@@ -1,6 +1,6 @@
 {
   "_comment": "GENERATED by scripts/collect-stats.js — do not edit by hand. Run `npm run docs:stats`.",
-  "version": "2.11.162",
+  "version": "2.11.164",
   "scanners": 21,
   "rulesTotal": 275,
   "rulesCore": 270,