muaddib-scanner 2.11.96 → 2.11.98

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "muaddib-scanner",
-  "version": "2.11.96",
+  "version": "2.11.98",
   "description": "Supply-chain threat detection & response for npm & PyPI/Python",
   "main": "src/index.js",
   "bin": {

package/{self-scan-v2.11.96.json → self-scan-v2.11.98.json}

@@ -1,6 +1,6 @@
 {
   "target": "node_modules",
-  "timestamp": "2026-06-11T13:24:06.648Z",
+  "timestamp": "2026-06-11T15:36:15.399Z",
   "threats": [
     {
       "type": "string_mutation_obfuscation",

package/src/monitor/ingestion.js

@@ -1017,6 +1017,12 @@ async function pollNpm(state, scanQueue, stats) {
 
 const PYPI_USER_AGENT = `${SELF_PACKAGE_NAME} (security-monitor; +https://github.com/DNSZLSK/muaddib)`;
 
+// A normal 15-min poll is a few dozen events; a changelog_since_serial batch
+// caps around ~50K. Anything this large means we are far behind — worth one
+// extra changelog_last_serial call to measure the GLOBAL lag (see the global
+// catch-up protection in pollPyPIChangelog).
+const PYPI_CATCHUP_PROBE_MIN_EVENTS = 10000;
+
 /**
  * Build an XML-RPC methodCall envelope. PyPI accepts only <int> and <string>
  * params for the methods we use (changelog_last_serial, changelog_since_serial),
@@ -1188,6 +1194,38 @@ async function pollPyPIChangelog(state, scanQueue, stats) {
       return 0;
     }
 
+    // GLOBAL catch-up protection (2026-06-11 incident): the per-batch gap
+    // below is bounded by one changelog_since_serial response (~50K events,
+    // observed 33-43K), so it can NEVER exceed PYPI_CATCHUP_MAX (100K) — a
+    // poller resumed from an ancient serial (a test-fixture serial leaked
+    // into prod state) replayed YEARS of history, ~15K ancient packages per
+    // poll, without ever tripping the skip. A full batch is the tell: probe
+    // the registry's current serial and skip to it when the global lag is
+    // beyond the cap. Costs one extra XML-RPC call only on full batches.
+    if (events.length >= PYPI_CATCHUP_PROBE_MIN_EVENTS) {
+      await acquireRegistrySlot();
+      let curBody;
+      try {
+        curBody = await _deps.httpsPost(
+          PYPI_XMLRPC_URL,
+          buildXmlRpcCall('changelog_last_serial', []),
+          { 'User-Agent': PYPI_USER_AGENT },
+          10_000
+        );
+      } finally {
+        releaseRegistrySlot();
+      }
+      const currentSerial = parseXmlRpcInt(curBody);
+      if (currentSerial != null && currentSerial - lastSerial > PYPI_CATCHUP_MAX) {
+        console.warn(`[MONITOR] PyPI changelog globally behind (${currentSerial - lastSerial} serials) — skipping to current ${currentSerial}`);
+        stats.pypiCatchupSkips = (stats.pypiCatchupSkips || 0) + 1;
+        stats.pypiCatchupSkippedEvents = (stats.pypiCatchupSkippedEvents || 0) + (currentSerial - lastSerial);
+        state.pypiLastSerial = currentSerial;
+        savePypiSerial(currentSerial);
+        return 0;
+      }
+    }
+
     // Catch-up protection: if events span more than PYPI_CATCHUP_MAX serials,
     // skip to the latest serial to avoid an avalanche after long downtime.
     const lastEventSerial = events[events.length - 1].serial;

package/src/monitor/queue.js

@@ -19,6 +19,7 @@ const { loadCachedIOCs } = require('../ioc/updater.js');
 const { scanPackageJson } = require('../scanner/package.js');
 const { scanShellScripts } = require('../scanner/shell.js');
 const { buildTrainingRecord } = require('../ml/feature-extractor.js');
+const { appendWorkerMem } = require('./worker-mem.js');
 const { appendRecord: appendTrainingRecord, relabelRecords } = require('../ml/jsonl-writer.js');
 
 // From ./state.js
@@ -426,6 +427,17 @@ function runScanInWorker(extractedDir, timeoutMs, scanContext = null, signal = n
     const _sc = scanContext || {};
     _liveWorkers.set(worker, { name: _sc.name, version: _sc.version, ecosystem: _sc.ecosystem });
 
+    // Off-heap attribution (worker-mem.jsonl, gated MUADDIB_WORKER_MEM=1):
+    // process RSS around each worker's lifetime. tid captured now — after
+    // 'exit' worker.threadId becomes -1.
+    const _wmTid = worker.threadId;
+    const _wmSpawnedAt = Date.now();
+    appendWorkerMem({
+      ev: 'spawn', tid: _wmTid,
+      name: _sc.name, version: _sc.version, ecosystem: _sc.ecosystem,
+      rss: process.memoryUsage().rss
+    });
+
     let settled = false;
     let timer = null;
     const done = (fn) => {
@@ -462,9 +474,19 @@ function runScanInWorker(extractedDir, timeoutMs, scanContext = null, signal = n
 
     worker.on('error', (err) => done(() => reject(err)));
 
-    worker.on('exit', (code) => done(() => {
-      if (code !== 0) reject(new Error(`Worker exited with code ${code}`));
-    }));
+    worker.on('exit', (code) => {
+      // 'exit' fires exactly once per worker (even after terminate/error), so
+      // it is the one reliable place to close the spawn/exit RSS pair.
+      appendWorkerMem({
+        ev: 'exit', tid: _wmTid,
+        name: _sc.name, version: _sc.version, code,
+        durMs: Date.now() - _wmSpawnedAt,
+        rss: process.memoryUsage().rss
+      });
+      done(() => {
+        if (code !== 0) reject(new Error(`Worker exited with code ${code}`));
+      });
+    });
   });
 }
 

package/src/monitor/state.js

@@ -81,7 +81,10 @@ const ALERTS_LOG_DIR = resolveWritableDir(PRIMARY_ALERTS_DIR, FALLBACK_ALERTS_DI
 
 // --- npm seq constants ---
 
-const NPM_SEQ_FILE = path.join(__dirname, '..', '..', 'data', 'npm-seq.json');
+// Env-overridable — same prod-state-pollution guard as PYPI_SERIAL_FILE below
+// (the npm-seq roundtrip test used to unlink the REAL file).
+const NPM_SEQ_FILE = process.env.MUADDIB_NPM_SEQ_FILE
+  || path.join(__dirname, '..', '..', 'data', 'npm-seq.json');
 const CHANGES_STREAM_URL = 'https://replicate.npmjs.com/registry/_changes';
 const CHANGES_LIMIT = 1000;
 const CHANGES_CATCHUP_MAX = 500000; // If behind by more than 500k seqs, skip to "now"
@@ -96,7 +99,13 @@ const CHANGES_CATCHUP_MAX = 500000; // If behind by more than 500k seqs, skip to
 // PYPI_CATCHUP_MAX is the staleness cap: if we are behind by more than this many
 // serials (≈ days of activity at ~30k events/day in 2026), skip to "now" rather
 // than fetch a monster batch. Mirrors CHANGES_CATCHUP_MAX for npm.
-const PYPI_SERIAL_FILE = path.join(__dirname, '..', '..', 'data', 'pypi-serial.json');
+// Env-overridable (2026-06-11 incident): integration tests exercise the real
+// pollPyPIChangelog with stubbed HTTP but the real savePypiSerial — a fixture
+// serial (1002) leaked into prod state, and the next daemon boot replayed the
+// PyPI changelog from 2011 (~15K ancient packages queued per poll). The test
+// harness points this at a tmp file so NO test can touch prod state.
+const PYPI_SERIAL_FILE = process.env.MUADDIB_PYPI_SERIAL_FILE
+  || path.join(__dirname, '..', '..', 'data', 'pypi-serial.json');
 const PYPI_XMLRPC_URL = 'https://pypi.org/pypi';
 const PYPI_CATCHUP_MAX = 100000;
 

package/src/monitor/worker-mem.js

@@ -0,0 +1,72 @@
+'use strict';
+
+/**
+ * Per-worker memory instrumentation (off-heap RSS attribution, 2026-06).
+ *
+ * The EMERGENCY breaker fires on process RSS while the heap sits at ~15% —
+ * the driver is off-heap (malloc arenas + tarball Buffers) and mem-trend.jsonl
+ * only samples the whole process. This module attributes memory to individual
+ * scan workers / packages so the worker_threads → child_process decision can
+ * be made on data:
+ *   H1: RSS stays high AFTER workers die  → arenas never returned to the OS
+ *   H2: RSS peaks only WHILE workers live → concurrent in-flight peak
+ *
+ * Producers:
+ *   - queue.js (parent): ev:'spawn' / ev:'exit' around each scan worker,
+ *     with process-wide RSS (delta attributable per package, noisy but
+ *     aggregable over 24-48h).
+ *   - scan-worker.js (worker): ev:'sample' every sampleIntervalMs() with the
+ *     isolate-local heapUsed/external/arrayBuffers (rss there is process-wide).
+ *
+ * Same hot-path safety rules as spill.js (2026-06-11 prod-freeze lesson):
+ * append-only, stat-gated O(1) rotation, never read the file back, never throw.
+ * OFF unless MUADDIB_WORKER_MEM=1 (staged rollout, same pattern as
+ * MUADDIB_WORKER_MAX_OLD_MB) so tests and CLI runs never touch data/.
+ */
+
+const fs = require('fs');
+const path = require('path');
+
+const DEFAULT_FILE = path.join(__dirname, '..', '..', 'data', 'worker-mem.jsonl');
+const DEFAULT_MAX_MB = 64;        // rotate past 64MB (file + .1 = 128MB worst case, ~2.5 days at concurrency 8)
+const DEFAULT_SAMPLE_MS = 10000;  // per-worker isolate sample cadence
+
+function workerMemEnabled() {
+  return process.env.MUADDIB_WORKER_MEM === '1';
+}
+
+function workerMemFile() {
+  return process.env.MUADDIB_WORKER_MEM_FILE || DEFAULT_FILE;
+}
+
+/** 0 = sampling disabled (instrumentation off, or explicit MUADDIB_WORKER_MEM_SAMPLE_MS=0). */
+function sampleIntervalMs() {
+  if (!workerMemEnabled()) return 0;
+  const v = parseInt(process.env.MUADDIB_WORKER_MEM_SAMPLE_MS, 10);
+  if (Number.isFinite(v) && v >= 0) return v;
+  return DEFAULT_SAMPLE_MS;
+}
+
+/**
+ * Append one instrumentation entry (ts stamped here). Bounded resource
+ * (CLAUDE.md §2): stat-gated truncate-rotate, no read-back on the hot path.
+ * @returns {boolean} true if a line was written
+ */
+function appendWorkerMem(entry) {
+  if (!workerMemEnabled()) return false;
+  try {
+    const file = workerMemFile();
+    const maxMb = parseInt(process.env.MUADDIB_WORKER_MEM_MAX_MB, 10);
+    const maxBytes = (Number.isFinite(maxMb) && maxMb > 0 ? maxMb : DEFAULT_MAX_MB) * 1024 * 1024;
+    try {
+      const st = fs.statSync(file);
+      if (st.size > maxBytes) fs.renameSync(file, file + '.1');
+    } catch { /* no file yet — fine */ }
+    fs.appendFileSync(file, JSON.stringify({ ts: new Date().toISOString(), ...entry }) + '\n', 'utf8');
+    return true;
+  } catch { /* instrumentation must never crash the daemon or a worker */
+    return false;
+  }
+}
+
+module.exports = { appendWorkerMem, sampleIntervalMs, workerMemEnabled, workerMemFile };

package/src/pipeline/scan-worker.js

@@ -12,7 +12,7 @@
  *   parentPort.postMessage({ type: 'error', message: string })
  */
 
-const { parentPort, workerData } = require('worker_threads');
+const { parentPort, workerData, threadId } = require('worker_threads');
 
 if (!parentPort) {
   // Not running as a worker — exit gracefully
@@ -20,17 +20,45 @@ if (!parentPort) {
 }
 
 const { run } = require('../index.js');
+const { appendWorkerMem, sampleIntervalMs } = require('../monitor/worker-mem.js');
 
 (async () => {
+  // Off-heap attribution samples (worker-mem.jsonl): heapUsed/external/
+  // arrayBuffers are isolate-local here, rss is process-wide. The samples MUST
+  // NOT go through parentPort — the parent settles the scan promise on the
+  // first message it receives (queue.js done()), so a sample message would
+  // hang the scan forever. unref() so the timer never keeps the worker alive.
+  const scanContext = workerData.scanContext || {};
+  const everyMs = sampleIntervalMs();
+  let sampler = null;
+  if (everyMs > 0) {
+    const sampleNow = () => {
+      const m = process.memoryUsage();
+      appendWorkerMem({
+        ev: 'sample', tid: threadId,
+        name: scanContext.name, version: scanContext.version,
+        heapUsed: m.heapUsed, external: m.external, arrayBuffers: m.arrayBuffers, rss: m.rss
+      });
+    };
+    // One immediate baseline sample, deterministically: a mostly-synchronous
+    // scan (small package, sync AST walks, microtask-only awaits) can starve
+    // the event loop for its whole lifetime, so the interval alone may never
+    // fire (bit CI on 2026-06-11). The baseline also gives the per-package
+    // delta a clean starting point.
+    sampleNow();
+    sampler = setInterval(sampleNow, everyMs);
+    sampler.unref();
+  }
   try {
     // scanContext (optional) carries monitor-side info that opt-in scanners need
     // (e.g. trusted-dep-diff requires package name + version to query the registry).
     // It is spread INTO the pipeline options, but `_capture: true` always wins so
     // the worker keeps returning the result object — never prints.
-    const scanContext = workerData.scanContext || {};
     const result = await run(workerData.extractedDir, { ...scanContext, _capture: true });
     parentPort.postMessage({ type: 'result', data: result });
   } catch (err) {
     parentPort.postMessage({ type: 'error', message: err.message || String(err) });
+  } finally {
+    if (sampler) clearInterval(sampler);
   }
 })();