muaddib-scanner 2.11.97 → 2.11.98

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "muaddib-scanner",
-  "version": "2.11.97",
+  "version": "2.11.98",
   "description": "Supply-chain threat detection & response for npm & PyPI/Python",
   "main": "src/index.js",
   "bin": {

package/{self-scan-v2.11.97.json → self-scan-v2.11.98.json}

@@ -1,6 +1,6 @@
 {
   "target": "node_modules",
-  "timestamp": "2026-06-11T15:01:04.220Z",
+  "timestamp": "2026-06-11T15:36:15.399Z",
   "threats": [
     {
       "type": "string_mutation_obfuscation",

package/src/monitor/ingestion.js

@@ -1017,6 +1017,12 @@ async function pollNpm(state, scanQueue, stats) {
 
 const PYPI_USER_AGENT = `${SELF_PACKAGE_NAME} (security-monitor; +https://github.com/DNSZLSK/muaddib)`;
 
+// A normal 15-min poll is a few dozen events; a changelog_since_serial batch
+// caps around ~50K. Anything this large means we are far behind — worth one
+// extra changelog_last_serial call to measure the GLOBAL lag (see the global
+// catch-up protection in pollPyPIChangelog).
+const PYPI_CATCHUP_PROBE_MIN_EVENTS = 10000;
+
 /**
  * Build an XML-RPC methodCall envelope. PyPI accepts only <int> and <string>
  * params for the methods we use (changelog_last_serial, changelog_since_serial),
@@ -1188,6 +1194,38 @@ async function pollPyPIChangelog(state, scanQueue, stats) {
       return 0;
     }
 
+    // GLOBAL catch-up protection (2026-06-11 incident): the per-batch gap
+    // below is bounded by one changelog_since_serial response (~50K events,
+    // observed 33-43K), so it can NEVER exceed PYPI_CATCHUP_MAX (100K) — a
+    // poller resumed from an ancient serial (a test-fixture serial leaked
+    // into prod state) replayed YEARS of history, ~15K ancient packages per
+    // poll, without ever tripping the skip. A full batch is the tell: probe
+    // the registry's current serial and skip to it when the global lag is
+    // beyond the cap. Costs one extra XML-RPC call only on full batches.
+    if (events.length >= PYPI_CATCHUP_PROBE_MIN_EVENTS) {
+      await acquireRegistrySlot();
+      let curBody;
+      try {
+        curBody = await _deps.httpsPost(
+          PYPI_XMLRPC_URL,
+          buildXmlRpcCall('changelog_last_serial', []),
+          { 'User-Agent': PYPI_USER_AGENT },
+          10_000
+        );
+      } finally {
+        releaseRegistrySlot();
+      }
+      const currentSerial = parseXmlRpcInt(curBody);
+      if (currentSerial != null && currentSerial - lastSerial > PYPI_CATCHUP_MAX) {
+        console.warn(`[MONITOR] PyPI changelog globally behind (${currentSerial - lastSerial} serials) — skipping to current ${currentSerial}`);
+        stats.pypiCatchupSkips = (stats.pypiCatchupSkips || 0) + 1;
+        stats.pypiCatchupSkippedEvents = (stats.pypiCatchupSkippedEvents || 0) + (currentSerial - lastSerial);
+        state.pypiLastSerial = currentSerial;
+        savePypiSerial(currentSerial);
+        return 0;
+      }
+    }
+
     // Catch-up protection: if events span more than PYPI_CATCHUP_MAX serials,
     // skip to the latest serial to avoid an avalanche after long downtime.
     const lastEventSerial = events[events.length - 1].serial;

package/src/monitor/state.js

@@ -81,7 +81,10 @@ const ALERTS_LOG_DIR = resolveWritableDir(PRIMARY_ALERTS_DIR, FALLBACK_ALERTS_DI
 
 // --- npm seq constants ---
 
-const NPM_SEQ_FILE = path.join(__dirname, '..', '..', 'data', 'npm-seq.json');
+// Env-overridable — same prod-state-pollution guard as PYPI_SERIAL_FILE below
+// (the npm-seq roundtrip test used to unlink the REAL file).
+const NPM_SEQ_FILE = process.env.MUADDIB_NPM_SEQ_FILE
+  || path.join(__dirname, '..', '..', 'data', 'npm-seq.json');
 const CHANGES_STREAM_URL = 'https://replicate.npmjs.com/registry/_changes';
 const CHANGES_LIMIT = 1000;
 const CHANGES_CATCHUP_MAX = 500000; // If behind by more than 500k seqs, skip to "now"
@@ -96,7 +99,13 @@ const CHANGES_CATCHUP_MAX = 500000; // If behind by more than 500k seqs, skip to
 // PYPI_CATCHUP_MAX is the staleness cap: if we are behind by more than this many
 // serials (≈ days of activity at ~30k events/day in 2026), skip to "now" rather
 // than fetch a monster batch. Mirrors CHANGES_CATCHUP_MAX for npm.
-const PYPI_SERIAL_FILE = path.join(__dirname, '..', '..', 'data', 'pypi-serial.json');
+// Env-overridable (2026-06-11 incident): integration tests exercise the real
+// pollPyPIChangelog with stubbed HTTP but the real savePypiSerial — a fixture
+// serial (1002) leaked into prod state, and the next daemon boot replayed the
+// PyPI changelog from 2011 (~15K ancient packages queued per poll). The test
+// harness points this at a tmp file so NO test can touch prod state.
+const PYPI_SERIAL_FILE = process.env.MUADDIB_PYPI_SERIAL_FILE
+  || path.join(__dirname, '..', '..', 'data', 'pypi-serial.json');
 const PYPI_XMLRPC_URL = 'https://pypi.org/pypi';
 const PYPI_CATCHUP_MAX = 100000;
 

package/src/pipeline/scan-worker.js

@@ -32,14 +32,21 @@ const { appendWorkerMem, sampleIntervalMs } = require('../monitor/worker-mem.js'
   const everyMs = sampleIntervalMs();
   let sampler = null;
   if (everyMs > 0) {
-    sampler = setInterval(() => {
+    const sampleNow = () => {
       const m = process.memoryUsage();
       appendWorkerMem({
         ev: 'sample', tid: threadId,
         name: scanContext.name, version: scanContext.version,
         heapUsed: m.heapUsed, external: m.external, arrayBuffers: m.arrayBuffers, rss: m.rss
       });
-    }, everyMs);
+    };
+    // One immediate baseline sample, deterministically: a mostly-synchronous
+    // scan (small package, sync AST walks, microtask-only awaits) can starve
+    // the event loop for its whole lifetime, so the interval alone may never
+    // fire (bit CI on 2026-06-11). The baseline also gives the per-package
+    // delta a clean starting point.
+    sampleNow();
+    sampler = setInterval(sampleNow, everyMs);
     sampler.unref();
   }
   try {