muaddib-scanner 2.11.100 → 2.11.102

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "muaddib-scanner",
-  "version": "2.11.100",
+  "version": "2.11.102",
   "description": "Supply-chain threat detection & response for npm & PyPI/Python",
   "main": "src/index.js",
   "bin": {

package/{self-scan-v2.11.100.json → self-scan-v2.11.102.json}

@@ -1,6 +1,6 @@
 {
   "target": "node_modules",
-  "timestamp": "2026-06-11T17:09:48.912Z",
+  "timestamp": "2026-06-11T19:20:45.240Z",
   "threats": [
     {
       "type": "string_mutation_obfuscation",

package/src/ioc/scraper.js

@@ -7,7 +7,7 @@ const AdmZip = require('adm-zip');
 const IOC_FILE = path.join(__dirname, 'data/iocs.json');
 const COMPACT_IOC_FILE = path.join(__dirname, 'data/iocs-compact.json');
 const HOME_IOC_FILE = path.join(os.homedir(), '.muaddib', 'data', 'iocs.json');
-const { generateCompactIOCs, NEVER_WILDCARD, expandCompactIOCs } = require('./updater.js');
+const { generateCompactIOCs, NEVER_WILDCARD, expandCompactIOCs, writeLeanIOCFile } = require('./updater.js');
 const { Spinner } = require('../utils.js');
 const { NPM_PACKAGE_REGEX } = require('../shared/constants.js');
 const { version: PKG_VERSION } = require('../../package.json');
@@ -1274,6 +1274,11 @@ async function runScraper() {
   fs.writeFileSync(tmpCompactFile, JSON.stringify(compactIOCs));
   fs.renameSync(tmpCompactFile, COMPACT_IOC_FILE);
 
+  // Save the lean projection in lock-step with the full file (~24MB) — what
+  // scan workers load instead of the 223MB full (RSS fix). Built from the
+  // in-memory object, so no extra parse peak. See updater.js:createLeanIOCs.
+  writeLeanIOCFile(existingIOCs);
+
   // Persist to ~/.muaddib/data/ (survives npm update)
   saveSpinner.update('Persisting to home directory...');
   const homeDir = path.dirname(HOME_IOC_FILE);

package/src/ioc/updater.js

@@ -6,6 +6,14 @@ const crypto = require('crypto');
 const HOME_DATA_PATH = path.join(os.homedir(), '.muaddib', 'data');
 const CACHE_IOC_FILE = path.join(HOME_DATA_PATH, 'iocs.json');
 const LOCAL_IOC_FILE = path.join(__dirname, 'data/iocs.json');
+// Lean projection of LOCAL_IOC_FILE — only the fields the matcher/alert read
+// ({name,version,severity,source} + hashes/markers/files/stringIocs). The full
+// file is ~223MB → 447MB string during JSON.parse, reloaded by every one-shot
+// worker that touches IOC matching (heap-snapshot-confirmed ~900MB peak). The
+// lean is ~24MB → ~50MB peak. Workers READ this; only the daemon/scraper write
+// it (a worker must never re-read the 223MB full to regenerate — that is the
+// very peak we are removing). See ensureLeanIOCFile + createLeanIOCs below.
+const LOCAL_LEAN_FILE = path.join(__dirname, 'data/iocs-lean.json');
 const LOCAL_COMPACT_FILE = path.join(__dirname, 'data/iocs-compact.json');
 const { loadYAMLIOCs } = require('./yaml-loader.js');
 
@@ -241,7 +249,7 @@ function mergeIOCs(target, source) {
 // scan/poll) does zero disk I/O.
 const IOCS_DIR = path.join(__dirname, '..', '..', 'iocs');
 const IOC_SOURCE_FILES = [
-  CACHE_IOC_FILE, LOCAL_IOC_FILE, LOCAL_COMPACT_FILE,
+  CACHE_IOC_FILE, LOCAL_IOC_FILE, LOCAL_LEAN_FILE, LOCAL_COMPACT_FILE,
   path.join(IOCS_DIR, 'packages.yaml'), path.join(IOCS_DIR, 'builtin.yaml'),
   path.join(IOCS_DIR, 'hashes.yaml'), path.join(IOCS_DIR, 'string-iocs.yaml')
 ];
@@ -279,8 +287,19 @@ function loadCachedIOCs() {
     stringIocs: Array.isArray(yamlIOCs.stringIocs) ? [...yamlIOCs.stringIocs] : []
   };
 
-  // Priority 2a: Local scraped IOCs (full enriched file)
-  if (fs.existsSync(LOCAL_IOC_FILE)) {
+  // Priority 2a: Local scraped IOCs. Prefer the lean projection (~24MB) — it
+  // carries every field the matcher/alert read. Only fall back to the full
+  // ~223MB file when the lean is absent (backward-compat / before the daemon
+  // has generated it), which costs the ~450MB parse peak. ensureLeanIOCFile()
+  // (called at daemon boot + after each scrape) keeps the lean present & fresh.
+  if (fs.existsSync(LOCAL_LEAN_FILE)) {
+    try {
+      const leanIOCs = JSON.parse(fs.readFileSync(LOCAL_LEAN_FILE, 'utf8'));
+      mergeIOCs(merged, leanIOCs);
+    } catch (e) {
+      console.log('[WARN] Failed to load lean IOC database (iocs-lean.json): ' + e.message);
+    }
+  } else if (fs.existsSync(LOCAL_IOC_FILE)) {
     try {
       const localIOCs = JSON.parse(fs.readFileSync(LOCAL_IOC_FILE, 'utf8'));
       mergeIOCs(merged, localIOCs);
@@ -471,6 +490,67 @@ const NEVER_WILDCARD_PYPI = new Set([
   'scipy', 'tensorflow', 'torch', 'fastapi', 'uvicorn'
 ]);
 
+// Lean projection of a full IOC object: keep only the fields the matcher and
+// the alert message read on package entries ({name,version,severity,source}),
+// drop the enrichment (id/description/references/mitre/published/freshness/
+// sources/confidence — never read after load; profiled). hashes/markers/files/
+// stringIocs are simple values / small (YAML-sourced) and kept verbatim.
+// Pure: no I/O. Used to write LOCAL_LEAN_FILE from an in-memory full object
+// (zero extra parse peak) and by ensureLeanIOCFile.
+function createLeanIOCs(fullIOCs) {
+  const leanPkg = p => ({ name: p.name, version: p.version, severity: p.severity, source: p.source });
+  return {
+    packages: (fullIOCs.packages || []).map(leanPkg),
+    pypi_packages: (fullIOCs.pypi_packages || []).map(leanPkg),
+    hashes: fullIOCs.hashes || [],
+    markers: fullIOCs.markers || [],
+    files: fullIOCs.files || [],
+    stringIocs: fullIOCs.stringIocs || [],
+    updated: fullIOCs.updated,
+    sources: fullIOCs.sources
+  };
+}
+
+// Ensure LOCAL_LEAN_FILE exists and is at least as fresh as LOCAL_IOC_FILE.
+// Reads the 223MB full ONCE (the ~450MB parse peak) — acceptable only in a
+// long-lived process (daemon boot); NEVER call from a one-shot scan worker.
+// Atomic write (.tmp → rename). Returns {generated:boolean, bytes:number}.
+function ensureLeanIOCFile() {
+  try {
+    if (!fs.existsSync(LOCAL_IOC_FILE)) return { generated: false, bytes: 0 };
+    let fresh = false;
+    if (fs.existsSync(LOCAL_LEAN_FILE)) {
+      try { fresh = fs.statSync(LOCAL_LEAN_FILE).mtimeMs >= fs.statSync(LOCAL_IOC_FILE).mtimeMs; } catch { fresh = false; }
+    }
+    if (fresh) return { generated: false, bytes: fs.statSync(LOCAL_LEAN_FILE).size };
+    const full = JSON.parse(fs.readFileSync(LOCAL_IOC_FILE, 'utf8'));
+    const lean = createLeanIOCs(full);
+    const tmp = LOCAL_LEAN_FILE + '.tmp';
+    const data = JSON.stringify(lean);
+    fs.writeFileSync(tmp, data);
+    fs.renameSync(tmp, LOCAL_LEAN_FILE);
+    return { generated: true, bytes: Buffer.byteLength(data) };
+  } catch (e) {
+    console.log('[WARN] ensureLeanIOCFile failed: ' + e.message);
+    return { generated: false, bytes: 0 };
+  }
+}
+
+// Write the lean file from an already-in-memory full object (zero extra parse
+// peak). Called by the scraper right after it writes LOCAL_IOC_FILE so the
+// lean stays in lock-step with the full after every deep scrape.
+function writeLeanIOCFile(fullIOCs) {
+  try {
+    const tmp = LOCAL_LEAN_FILE + '.tmp';
+    fs.writeFileSync(tmp, JSON.stringify(createLeanIOCs(fullIOCs)));
+    fs.renameSync(tmp, LOCAL_LEAN_FILE);
+    return true;
+  } catch (e) {
+    console.log('[WARN] writeLeanIOCFile failed: ' + e.message);
+    return false;
+  }
+}
+
 function generateCompactIOCs(fullIOCs) {
   const wildcards = [];
   const versioned = Object.create(null);
@@ -693,4 +773,4 @@ function verifyIOCHMAC(data, hmac) {
   }
 }
 
-module.exports = { updateIOCs, loadCachedIOCs, invalidateCache, generateCompactIOCs, expandCompactIOCs, mergeIOCs, createOptimizedIOCs, generateIOCHMAC, verifyIOCHMAC, checkIOCStaleness, NEVER_WILDCARD, NEVER_WILDCARD_PYPI };
+module.exports = { updateIOCs, loadCachedIOCs, invalidateCache, generateCompactIOCs, expandCompactIOCs, createLeanIOCs, ensureLeanIOCFile, writeLeanIOCFile, LOCAL_LEAN_FILE, LOCAL_IOC_FILE, mergeIOCs, createOptimizedIOCs, generateIOCHMAC, verifyIOCHMAC, checkIOCStaleness, NEVER_WILDCARD, NEVER_WILDCARD_PYPI };

package/src/monitor/daemon.js

@@ -813,6 +813,18 @@ async function startMonitor(options, stats, dailyAlerts, recentlyScanned, downlo
     console.warn(`[Archive] Failed to start periodic cleanup: ${err.message}`);
   }
 
+  // RSS fix (C2): make sure the lean IOC projection exists & is fresh BEFORE any
+  // scan worker spawns. Workers load the ~24MB lean instead of the ~223MB full
+  // (heap-snapshot-confirmed ~900MB→~50MB per IOC-matching scan). The full read
+  // here is paid ONCE by this long-lived daemon (never by a one-shot worker).
+  try {
+    const { ensureLeanIOCFile } = require('../ioc/updater.js');
+    const r = ensureLeanIOCFile();
+    if (r.generated) console.log(`[MONITOR] IOC lean projection regenerated (${(r.bytes / 1024 / 1024).toFixed(1)}MB) — workers avoid the 223MB full load`);
+  } catch (err) {
+    console.warn(`[MONITOR] IOC lean bootstrap failed (workers fall back to full file): ${err.message}`);
+  }
+
   console.log('\n' + banner([
     "MUAD'DIB - Registry Monitor",
     'Scanning npm + PyPI new packages'

package/src/monitor/heavy-lane.js

@@ -59,13 +59,16 @@ const _lane = { active: 0, queue: [] };
  * package is exactly the kind that blows a worker. Compares weightedJsBytes
  * (plain + ×12 minified — see measureJsWeight in queue.js: raw bytes alone
  * missed the minified explosions, powerlines 517KB → 1151MB heap) and falls
- * back to totalJsBytes for callers that don't weight.
- * @param {{totalJsBytes: number, weightedJsBytes?: number, truncated: boolean}|null} weight
+ * back to totalJsBytes for callers that don't weight. `oversize` (any single
+ * JS file > getMaxFileSize) also forces heavy — content scanners load such a
+ * file whole even though the AST skips it (omnius: a 30MB index.js → 1347MB).
+ * @param {{totalJsBytes: number, weightedJsBytes?: number, oversize?: boolean, truncated: boolean}|null} weight
  * @param {number} [thresholdBytes]
  */
 function isHeavyScan(weight, thresholdBytes = heavyScanBytesThreshold()) {
   if (!weight) return false;
   if (weight.truncated) return true;
+  if (weight.oversize) return true; // a single JS file > getMaxFileSize — content scanners load it whole
   const effective = Number.isFinite(weight.weightedJsBytes) ? weight.weightedJsBytes : (weight.totalJsBytes || 0);
   return effective >= thresholdBytes;
 }

package/src/monitor/queue.js

@@ -323,15 +323,21 @@ const JS_WEIGHT_FILE_PATTERN = /\.(?:[cm]?js|[jt]sx?)$/i;
 // license header pads the probe window.
 const JS_MINIFIED_WEIGHT = 12;
 const JS_MINIFIED_AVG_LINE = 250;
-const JS_MINIFIED_PROBE_BYTES = 4096;
-
-/** Probe the first 4KB of a file (never loads the rest) for minification. */
-function probeIsMinified(filePath) {
+// 64KB, not 4KB: bike4mind sailed under the 4KB probe (a license/banner header
+// padded the window; the minified body started later) → mis-classified light →
+// 890MB heap. Probe a 64KB window from ~2KB in to skip any header and still
+// never load a 30MB file. Cheap (one readSync) at JS_WEIGHT_MAX_FILES files.
+const JS_MINIFIED_PROBE_OFFSET = 2048;
+const JS_MINIFIED_PROBE_BYTES = 64 * 1024;
+
+/** Probe a 64KB window of a file (never loads the rest) for minification. */
+function probeIsMinified(filePath, size) {
   let fd = null;
   try {
     fd = fs.openSync(filePath, 'r');
+    const offset = size > JS_MINIFIED_PROBE_OFFSET + JS_MINIFIED_PROBE_BYTES ? JS_MINIFIED_PROBE_OFFSET : 0;
     const buf = Buffer.alloc(JS_MINIFIED_PROBE_BYTES);
-    const n = fs.readSync(fd, buf, 0, JS_MINIFIED_PROBE_BYTES, 0);
+    const n = fs.readSync(fd, buf, 0, JS_MINIFIED_PROBE_BYTES, offset);
     if (n <= 0) return false;
     const head = buf.toString('utf8', 0, n);
     return (head.length / head.split('\n').length) > JS_MINIFIED_AVG_LINE;
@@ -359,13 +365,20 @@ function probeIsMinified(filePath) {
  * value isHeavyScan compares against the threshold (raw bytes alone missed
  * the minified explosions, see JS_MINIFIED_WEIGHT above).
  *
+ * `oversize` (any single JS file > getMaxFileSize) forces heavy: the AST
+ * executor skips such files, but the content scanners (entropy/hash/
+ * ioc-strings/deobfuscate) still readFileSync the whole thing — omnius
+ * (a 30MB dist/index.js, 39KB of other JS) blew a 'light' worker to 1347MB.
+ * So an oversize JS file is the STRONGEST heavy signal, not something to skip.
+ *
  * @param {string} dir - extracted package directory
- * @returns {{ totalJsBytes: number, minifiedJsBytes: number, weightedJsBytes: number, maxJsFileBytes: number, truncated: boolean }}
+ * @returns {{ totalJsBytes: number, minifiedJsBytes: number, weightedJsBytes: number, maxJsFileBytes: number, oversize: boolean, truncated: boolean }}
  */
 function measureJsWeight(dir) {
   let totalJsBytes = 0;
   let minifiedJsBytes = 0;
   let maxJsFileBytes = 0;
+  let oversize = false;
   let seen = 0;
   let truncated = false;
   const perFileCap = getMaxFileSize();
@@ -385,17 +398,21 @@ function measureJsWeight(dir) {
         const filePath = path.join(current, entry.name);
         let size;
         try { size = fs.statSync(filePath).size; } catch { continue; }
-        if (size > perFileCap) continue; // executor skips these — they never reach the AST
-        totalJsBytes += size;
-        if (probeIsMinified(filePath)) minifiedJsBytes += size;
         if (size > maxJsFileBytes) maxJsFileBytes = size;
+        if (size > perFileCap) {
+          // The AST skips it, but content scanners load it whole → heap blow-up.
+          oversize = true;
+          continue;
+        }
+        totalJsBytes += size;
+        if (probeIsMinified(filePath, size)) minifiedJsBytes += size;
       }
     }
   }
 
   walk(dir, 0);
   const weightedJsBytes = (totalJsBytes - minifiedJsBytes) + JS_MINIFIED_WEIGHT * minifiedJsBytes;
-  return { totalJsBytes, minifiedJsBytes, weightedJsBytes, maxJsFileBytes, truncated };
+  return { totalJsBytes, minifiedJsBytes, weightedJsBytes, maxJsFileBytes, oversize, truncated };
 }
 
 /**
@@ -1437,6 +1454,27 @@ async function scanPackage(name, version, ecosystem, tarballUrl, registryMeta, s
     // count clean — inconclusive, distinct ledger source, distinct log line
     // (the live-validation metric for the limits rollout). No retry: an OOM
     // re-OOMs deterministically.
+    // Reactive heap watermark (C2 volet B): the worker self-terminated before
+    // blowing the process RSS. Same disposition as a resourceLimits OOM —
+    // inconclusive, NOT clean, no retry (a re-scan re-explodes the same way) —
+    // but a distinct ledger source so the watchdog's catch rate is measurable
+    // separately from the V8 hard-cap OOMs.
+    const isHeapWatermark = err && /WORKER_HEAP_WATERMARK/.test(err.message || '');
+    if (isHeapWatermark) {
+      console.error(`[MONITOR] WORKER_HEAP_WATERMARK: ${name}@${version} — scan worker self-terminated over the heap watermark (kept INCONCLUSIVE, not clean)`);
+      stats.workerHeapWatermark = (stats.workerHeapWatermark || 0) + 1;
+      updateScanStats('sandbox_inconclusive');
+      try {
+        appendScanLedger({ name, version, ecosystem, outcome: 'error', source: 'worker_heap_watermark' });
+      } catch { /* ledger is best-effort */ }
+      return { sandboxResult: null, staticClean: false };
+    }
+    // Per-worker resourceLimits breach: the worker died on ITS V8 cap
+    // (ERR_WORKER_OUT_OF_MEMORY) instead of blowing the process RSS. Same
+    // garde-fou as static_timeout: a package that OOMs the scanner must NOT
+    // count clean — inconclusive, distinct ledger source, distinct log line
+    // (the live-validation metric for the limits rollout). No retry: an OOM
+    // re-OOMs deterministically.
     const isWorkerOom = err && (err.code === 'ERR_WORKER_OUT_OF_MEMORY' ||
       /ERR_WORKER_OUT_OF_MEMORY|reached its memory limit/i.test(err.message || ''));
     if (isWorkerOom) {

package/src/pipeline/scan-worker.js

@@ -22,6 +22,22 @@ if (!parentPort) {
 const { run } = require('../index.js');
 const { appendWorkerMem, sampleIntervalMs } = require('../monitor/worker-mem.js');
 
+// Reactive heap watermark (C2 volet B): the static heavy-lane classifier
+// predicts the peak from on-disk bytes and WILL miss cases (omnius: 39KB JS →
+// 1347MB). This is the prediction-free backstop — the worker watches its OWN
+// isolate heap and bails before it contributes to a process-wide RSS spike.
+// CAVEAT: a watchdog timer can only fire when the event loop yields, so it
+// catches PROGRESSIVE (multi-file, async-between-files) growth; a single
+// synchronous 30MB parse never yields and is caught only by the V8 hard cap
+// (MUADDIB_WORKER_MAX_OLD_MB resourceLimits). The two are complementary.
+// Default 2200MB: above the ~1.3GB legitimate scans that finish CLEAN, below
+// the 3072MB resourceLimits cap. 0 disables.
+const HEAP_WATERMARK_MB = (() => {
+  const v = parseInt(process.env.MUADDIB_WORKER_HEAP_WATERMARK_MB, 10);
+  return Number.isFinite(v) && v >= 0 ? v : 2200;
+})();
+const HEAP_WATERMARK_CHECK_MS = 1000;
+
 (async () => {
   // Off-heap attribution samples (worker-mem.jsonl): heapUsed/external/
   // arrayBuffers are isolate-local here, rss is process-wide. The samples MUST
@@ -49,6 +65,34 @@ const { appendWorkerMem, sampleIntervalMs } = require('../monitor/worker-mem.js'
     sampler = setInterval(sampleNow, everyMs);
     sampler.unref();
   }
+
+  // Heap-watermark watchdog. On breach, post a tagged error and exit — the
+  // parent maps the WORKER_HEAP_WATERMARK message onto its existing worker_oom
+  // path (inconclusive, ledgered, NOT counted clean). NOT unref'd: while the
+  // scan is in flight this watchdog must stay live to fire.
+  let watchdog = null;
+  if (HEAP_WATERMARK_MB > 0) {
+    const limitBytes = HEAP_WATERMARK_MB * 1024 * 1024;
+    watchdog = setInterval(() => {
+      if (process.memoryUsage().heapUsed > limitBytes) {
+        clearInterval(watchdog); watchdog = null;
+        if (sampler) clearInterval(sampler);
+        try {
+          parentPort.postMessage({
+            type: 'error',
+            message: `WORKER_HEAP_WATERMARK: isolate heap exceeded ${HEAP_WATERMARK_MB}MB (${scanContext.name}@${scanContext.version})`
+          });
+        } catch { /* parent gone */ }
+        // Exit NON-ZERO so the parent settles even if the message above is lost
+        // in the post/exit race: the worker.on('exit') handler rejects on any
+        // non-zero code, and the catch matches WORKER_HEAP_WATERMARK when the
+        // message did arrive, or the generic scan_error path when it didn't —
+        // never clean, never a hung promise. (exit(0) would let the exit
+        // handler no-op and hang the scan until the 300s outer timeout.)
+        process.exit(1);
+      }
+    }, HEAP_WATERMARK_CHECK_MS);
+  }
   try {
     // scanContext (optional) carries monitor-side info that opt-in scanners need
     // (e.g. trusted-dep-diff requires package name + version to query the registry).
@@ -60,5 +104,6 @@ const { appendWorkerMem, sampleIntervalMs } = require('../monitor/worker-mem.js'
     parentPort.postMessage({ type: 'error', message: err.message || String(err) });
   } finally {
     if (sampler) clearInterval(sampler);
+    if (watchdog) clearInterval(watchdog);
   }
 })();