muaddib-scanner 2.11.121 → 2.11.124

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "muaddib-scanner",
-  "version": "2.11.121",
+  "version": "2.11.124",
   "description": "Supply-chain threat detection & response for npm & PyPI/Python",
   "main": "src/index.js",
   "bin": {

package/{self-scan-v2.11.121.json → self-scan-v2.11.124.json}

@@ -1,6 +1,6 @@
 {
   "target": "node_modules",
-  "timestamp": "2026-06-19T09:42:28.283Z",
+  "timestamp": "2026-06-19T13:30:20.182Z",
   "threats": [
     {
       "type": "string_mutation_obfuscation",

package/src/monitor/queue.js

@@ -12,7 +12,7 @@ const os = require('os');
 const { Worker } = require('worker_threads');
 const { runSandbox, tryAcquireSandboxSlot } = require('../sandbox/index.js');
 const { sendWebhook } = require('../webhook.js');
-const { downloadToFile, extractArchive, sanitizePackageName } = require('../shared/download.js');
+const { downloadToFile, extractArchive, extractArchiveOffThread, sanitizePackageName } = require('../shared/download.js');
 const { MAX_TARBALL_SIZE, getMaxFileSize } = require('../shared/constants.js');
 const { acquireRegistrySlot, releaseRegistrySlot, awaitRateToken: awaitRateTokenForWorker, signal429: signal429ForWorker } = require('../shared/http-limiter.js');
 const { loadCachedIOCs } = require('../ioc/updater.js');
@@ -178,6 +178,25 @@ const STATIC_SCAN_TIMEOUT_MS = 45_000; // 45s for static analysis only
 const LARGE_PACKAGE_SIZE = 10 * 1024 * 1024; // 10MB
 const RECENTLY_SCANNED_MAX = 50_000; // FIFO cap for the dedup Set (P0c — bounded resource)
 
+// OOM fix (2026-06-19): archives larger than this (COMPRESSED, on-disk size) are
+// extracted off the main thread in a worker, so the synchronous extractor
+// (adm-zip extractAllTo / execFileSync tar) can no longer wedge the event loop and
+// starve the RSS breaker / memory governor / EMERGENCY purge (all main-thread
+// timers) → cgroup OOM. Confirmed culprit: data/loop-stalls.jsonl (extract:* up to
+// 148s). Small archives extract inline — a worker spawn costs more than their
+// sub-100ms extraction. Env-tunable via MUADDIB_INLINE_EXTRACT_MB.
+const INLINE_EXTRACT_MAX_BYTES = (parseInt(process.env.MUADDIB_INLINE_EXTRACT_MB, 10) || 4) * 1024 * 1024;
+
+// Extract inline for small archives, off-thread for large ones. compressedSize is
+// the on-disk tarball size (reliable, unlike the registry unpackedSize metadata).
+// Always returns a Promise so the call sites can uniformly `await`.
+function extractGated(archivePath, destDir, compressedSize) {
+  if (compressedSize > INLINE_EXTRACT_MAX_BYTES) {
+    return extractArchiveOffThread(archivePath, destDir);
+  }
+  return Promise.resolve(extractArchive(archivePath, destDir));
+}
+
 // First-publish sandbox: max pending sandbox items before deferring first-publish clean scans
 // Prevents starving T1a sandbox capacity when many first-publish packages arrive at once
 const FIRST_PUBLISH_SANDBOX_MAX_QUEUE = parseInt(process.env.MUADDIB_FIRST_PUBLISH_SANDBOX_MAX_QUEUE, 10) || 10;
@@ -766,7 +785,7 @@ async function scanPackage(name, version, ecosystem, tarballUrl, registryMeta, s
         let bypassQuickScan = false;
         try {
           const _crumb = beginOp('extract:quickscan', { name, version, unpackedSizeMb: Math.round(unpackedSize / 1024 / 1024) });
-          try { extractedDir = extractArchive(tgzPath, tmpDir); } finally { endOp(_crumb); }
+          try { extractedDir = await extractGated(tgzPath, tmpDir, fileSize); } finally { endOp(_crumb); }
 
           const [pkgThreats, shellThreats] = await Promise.all([
             scanPackageJson(extractedDir),
@@ -816,7 +835,7 @@ async function scanPackage(name, version, ecosystem, tarballUrl, registryMeta, s
 
     if (!extractedDir) {
       const _crumb = beginOp('extract:prework', { name, version, unpackedSizeMb: Math.round((meta.unpackedSize || 0) / 1024 / 1024) });
-      try { extractedDir = extractArchive(tgzPath, tmpDir); } finally { endOp(_crumb); }
+      try { extractedDir = await extractGated(tgzPath, tmpDir, fileSize); } finally { endOp(_crumb); }
     }
 
     // ML Phase 2a: Count JS files and detect test presence for enriched features

package/src/scanner/shell.js

@@ -8,7 +8,7 @@ const SHELL_EXCLUDED_DIRS = ['node_modules', '.git', '.muaddib-cache'];
 const MALICIOUS_PATTERNS = [
   { pattern: /curl[^\n]{0,5000}\|[^\n]{0,5000}sh/m, name: 'curl_pipe_shell', severity: 'HIGH' },
   { pattern: /wget[^\n]{0,5000}&&[^\n]{0,5000}chmod[^\n]{0,5000}\+x/m, name: 'wget_chmod_exec', severity: 'HIGH' },
-  { pattern: /bash\s+-i\s+>&\s+\/dev\/tcp/m, name: 'reverse_shell', severity: 'CRITICAL' },
+  { pattern: /(?:ba)?sh\s+-i\s+>&\s*\/dev\/tcp/m, name: 'reverse_shell', severity: 'CRITICAL' },
   { pattern: /nc\s+-e\s+\/bin\/(ba)?sh/m, name: 'netcat_shell', severity: 'CRITICAL' },
   { pattern: /rm\s+-rf\s+(~\/|\$HOME|\/home)/m, name: 'home_deletion', severity: 'CRITICAL' },
   { pattern: /shred.*\$HOME/m, name: 'shred_home', severity: 'CRITICAL' },
@@ -40,13 +40,26 @@ const MALICIOUS_PATTERNS = [
 
 const SHEBANG_RE = /^#!.*\b(?:ba)?sh\b/;
 
-function scanFileContent(file, content, targetPath, threats) {
+// Source files (.js/.ts/...) can embed shell reverse-shell commands inside
+// child_process exec/execSync/spawn string args. shell.js historically scanned only
+// .sh/shebang files, so `execSync("bash -i >& /dev/tcp/...")` in index.js was invisible
+// (missed npx-whoami-demo, 2026-06 — a revshell that scored grs 25 with type_reverse_shell=0).
+// Apply ONLY the unambiguous reverse-shell command patterns to source files — NOT the
+// context-dependent ones (curl|sh, systemctl, rm -rf, base64|sh) which would false-positive
+// on JS string literals / build tooling. FPR-gate (2026-06-19): these 4 matched 0 port-check
+// idioms (</dev/tcp, echo >/dev/tcp, nc -z) and 0 node_modules .js files.
+const SOURCE_SCAN_PATTERN_NAMES = new Set([
+  'reverse_shell', 'netcat_shell', 'fifo_reverse_shell', 'fifo_nc_reverse_shell'
+]);
+const SOURCE_SCAN_EXTENSIONS = ['.js', '.cjs', '.mjs', '.ts', '.jsx', '.tsx'];
+
+function scanFileContent(file, content, targetPath, threats, patterns = MALICIOUS_PATTERNS) {
   // Strip comment lines to avoid false positives on documentation
   const activeContent = content.split(/\r?\n/)
     .filter(line => !line.trimStart().startsWith('#'))
     .join('\n');
 
-  for (const { pattern, name, severity } of MALICIOUS_PATTERNS) {
+  for (const { pattern, name, severity } of patterns) {
     if (pattern.test(activeContent)) {
       threats.push({
         type: name,
@@ -106,6 +119,14 @@ async function scanShellScripts(targetPath) {
     } catch (e) { debugLog('[SHELL] readFile error:', e?.message); }
   }
 
+  // Pass 3: source files (.js/.ts/...) — only the unambiguous reverse-shell command
+  // patterns (revshell commands embedded in child_process exec/spawn string args).
+  const sourcePatterns = MALICIOUS_PATTERNS.filter(p => SOURCE_SCAN_PATTERN_NAMES.has(p.name));
+  const sourceFiles = findFiles(targetPath, { extensions: SOURCE_SCAN_EXTENSIONS, excludedDirs: SHELL_EXCLUDED_DIRS });
+  forEachSafeFile(sourceFiles, (file, content) => {
+    scanFileContent(file, content, targetPath, threats, sourcePatterns);
+  });
+
   return threats;
 }
 

package/src/shared/download.js

@@ -340,6 +340,52 @@ function extractArchive(archivePath, destDir, options = {}) {
   throw new Error(`Unsupported archive format for ${path.basename(archivePath)}`);
 }
 
+// Hard cap for off-thread extraction (OOM fix). Large legit packages can take
+// 10-30s; 120s leaves headroom while still bounding a pathological extraction
+// (the worker is terminated past this so a runaway cannot pin RSS forever).
+const EXTRACT_OFFTHREAD_TIMEOUT_MS = parseInt(process.env.MUADDIB_EXTRACT_OFFTHREAD_TIMEOUT_MS, 10) || 120_000;
+
+/**
+ * Off-main-thread variant of extractArchive: runs the SAME synchronous extractor
+ * in a worker thread (src/shared/extract-worker.js) so the caller's event loop
+ * stays responsive during extraction. See extract-worker.js header for why this
+ * is the OOM fix. Same return contract as extractArchive (resolves to the
+ * extracted package root); rejects on extraction error, worker crash, or timeout.
+ * Callers gate on archive size — small archives extract inline (cheaper than a
+ * worker spawn), large ones offload here.
+ *
+ * @param {string} archivePath
+ * @param {string} destDir   - must already exist
+ * @param {Object} [options]
+ * @param {'targz'|'zip'} [options.format] - override auto-detection
+ * @param {number} [options.timeoutMs]     - hard cap; worker terminated past it
+ * @returns {Promise<string>} extracted package root
+ */
+function extractArchiveOffThread(archivePath, destDir, options = {}) {
+  const { Worker } = require('worker_threads');
+  const timeoutMs = Number.isFinite(options.timeoutMs) ? options.timeoutMs : EXTRACT_OFFTHREAD_TIMEOUT_MS;
+  return new Promise((resolve, reject) => {
+    let settled = false;
+    const worker = new Worker(path.join(__dirname, 'extract-worker.js'), {
+      workerData: { archivePath, destDir, format: options.format || null }
+    });
+    const finish = (fn) => { if (settled) return; settled = true; clearTimeout(timer); fn(); };
+    const timer = setTimeout(() => finish(() => {
+      worker.terminate().finally(() =>
+        reject(new Error(`extractArchiveOffThread timeout after ${timeoutMs}ms: ${path.basename(archivePath)}`)));
+    }), timeoutMs);
+    if (timer && typeof timer.unref === 'function') timer.unref();
+    worker.once('message', (msg) => finish(() => {
+      worker.terminate();
+      if (msg && msg.ok) resolve(msg.dir);
+      else reject(new Error((msg && msg.error) || 'extractArchiveOffThread: worker reported failure'));
+    }));
+    worker.once('error', (err) => finish(() => { worker.terminate(); reject(err); }));
+    worker.once('exit', (code) => finish(() =>
+      reject(new Error(`extractArchiveOffThread: worker exited (${code}) without a result`))));
+  });
+}
+
 /**
  * Backwards-compatible wrapper for the original tar.gz-only extractor.
  * Kept because src/scanner/temporal-ast-diff.js and existing tests still
@@ -370,6 +416,7 @@ module.exports = {
   downloadToFile,
   extractTarGz,
   extractArchive,
+  extractArchiveOffThread,
   detectArchiveFormat,
   sanitizePackageName,
   isAllowedDownloadRedirect,

package/src/shared/extract-worker.js

@@ -0,0 +1,30 @@
+'use strict';
+
+/**
+ * Worker-thread entry for off-main-thread archive extraction (OOM fix 2026-06-19).
+ *
+ * extractArchive() runs a SYNCHRONOUS extractor — adm-zip `extractAllTo` (.zip/
+ * .whl) or `execFileSync('tar', …)` (.tgz). On the main thread that wedges the
+ * event loop for the whole extraction (measured: up to 148s on large packages,
+ * data/loop-stalls.jsonl). With the loop wedged the RSS circuit breaker, the
+ * memory governor's RSS feed, and the EMERGENCY queue purge — all main-thread
+ * `setInterval` timers (daemon.js) — never fire, so RSS climbs to the cgroup
+ * MemoryMax unchecked → kernel SIGKILL. Running the same sync extractor here
+ * blocks only the WORKER loop; the parent's loop stays live so those defenses run.
+ *
+ * Contract: workerData = { archivePath, destDir, format }. Posts exactly one
+ * message — { ok: true, dir } on success, { ok: false, error } on failure — and
+ * never throws to the thread (the parent also handles a worker 'error', but an
+ * explicit message keeps the failure path uniform). All extraction hardening
+ * (zip-slip, zip-bomb uncompressed-size cap) lives in extractArchive and runs here.
+ */
+const { workerData, parentPort } = require('worker_threads');
+const { extractArchive } = require('./download.js');
+
+try {
+  const opts = workerData && workerData.format ? { format: workerData.format } : {};
+  const dir = extractArchive(workerData.archivePath, workerData.destDir, opts);
+  parentPort.postMessage({ ok: true, dir });
+} catch (err) {
+  parentPort.postMessage({ ok: false, error: err && err.message ? err.message : String(err) });
+}