muaddib-scanner 2.11.159 → 2.11.161

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "muaddib-scanner",
-  "version": "2.11.159",
+  "version": "2.11.161",
   "description": "Supply-chain threat detection & response for npm & PyPI/Python",
   "main": "src/index.js",
   "bin": {
@@ -17,6 +17,7 @@
     "docs:stats": "node scripts/collect-stats.js && node scripts/sync-stats.js",
     "docs:stats:tests": "node scripts/collect-stats.js --with-tests && node scripts/sync-stats.js",
     "docs:stats:check": "node scripts/collect-stats.js --check && node scripts/sync-stats.js --check",
+    "version": "npm run docs:stats && git diff --name-only | xargs -r git add",
     "compress-iocs": "node -e \"const fs=require('fs');const zlib=require('zlib');zlib.gzip(fs.readFileSync('src/ioc/data/iocs.json'),(e,b)=>{if(e)throw e;fs.writeFileSync('iocs.json.gz',b);console.log('Compressed: '+b.length+' bytes')})\"",
     "prepublishOnly": "node -e \"if(!process.env.CI){console.error('ERR: Publish via CI workflow (tag v* push). Local publishes are disabled.');process.exit(1)}\""
   },
@@ -53,13 +54,13 @@
     "@inquirer/prompts": "8.5.2",
     "acorn": "8.17.0",
     "acorn-walk": "8.3.5",
-    "adm-zip": "0.5.17",
+    "adm-zip": "0.5.18",
     "js-yaml": "5.1.0",
     "web-tree-sitter": "^0.26.9"
   },
   "devDependencies": {
     "@eslint/js": "10.0.1",
-    "eslint": "10.5.0",
+    "eslint": "10.6.0",
     "eslint-plugin-security": "^4.0.0",
     "globals": "17.7.0"
   }

package/{self-scan-v2.11.159.json → self-scan-v2.11.161.json}

@@ -1,6 +1,6 @@
 {
   "target": "node_modules",
-  "timestamp": "2026-07-07T13:26:22.703Z",
+  "timestamp": "2026-07-09T09:23:51.701Z",
   "threats": [
     {
       "type": "string_mutation_obfuscation",
@@ -468,26 +468,6 @@
       "playbook": "CRITIQUE: Assignation a Module._resolveFilename/_compile/_extensions. Detournement du systeme de modules Node.js. Tous les require() sont interceptes. Supprimer le package immediatement. Auditer tous les modules charges apres installation.",
       "points": 25
     },
-    {
-      "type": "dynamic_require",
-      "severity": "HIGH",
-      "message": "Dynamic require() with computed argument (possible decode obfuscation).",
-      "file": "ajv/scripts/bundle.js",
-      "count": 2,
-      "reductions": [],
-      "originalSeverity": "HIGH",
-      "confidenceTier": "medium",
-      "rule_id": "MUADDIB-AST-006",
-      "rule_name": "Dynamic Require with Concatenation",
-      "confidence": "high",
-      "domain": "malware",
-      "references": [
-        "https://attack.mitre.org/techniques/T1027/"
-      ],
-      "mitre": "T1027",
-      "playbook": "require() avec concatenation detecte. Technique d'obfuscation pour masquer le module charge. Analyser les variables concatenees.",
-      "points": 10
-    },
     {
       "type": "env_access",
       "severity": "MEDIUM",
@@ -2090,9 +2070,9 @@
   ],
   "python": null,
   "summary": {
-    "total": 92,
+    "total": 91,
     "critical": 7,
-    "high": 10,
+    "high": 9,
     "medium": 44,
     "low": 31,
     "riskScore": 35,
@@ -2106,7 +2086,6 @@
       "web-tree-sitter/web-tree-sitter.cjs": 42,
       "web-tree-sitter/web-tree-sitter.js": 42,
       "ajv/lib/ajv.js": 25,
-      "ajv/scripts/bundle.js": 10,
       "debug/src/node.js": 3,
       "eslint/bin/eslint.js": 10,
       "fast-json-stable-stringify/benchmark/index.js": 10,
@@ -2129,7 +2108,6 @@
       "web-tree-sitter/web-tree-sitter.cjs": 165707,
       "web-tree-sitter/web-tree-sitter.js": 153666,
       "ajv/lib/ajv.js": 15837,
-      "ajv/scripts/bundle.js": 1795,
       "debug/src/node.js": 4728,
       "eslint/bin/eslint.js": 6028,
       "fast-json-stable-stringify/benchmark/index.js": 740,
@@ -2202,12 +2180,6 @@
         "points": 10,
         "reason": "Binary file reference (.png/.jpg/.wasm/etc.) + eval() in same file — possible steganographic payload execution."
       },
-      {
-        "rule": "MUADDIB-AST-006",
-        "type": "dynamic_require",
-        "points": 10,
-        "reason": "Dynamic require() with computed argument (possible decode obfuscation)."
-      },
       {
         "rule": "MUADDIB-AST-006",
         "type": "dynamic_require",
@@ -2698,7 +2670,7 @@
     "tierCounts": {
       "verified": 0,
       "high": 0,
-      "medium": 18,
+      "medium": 17,
       "low": 74
     },
     "perceivedFlagged": 0

package/src/ioc/scraper.js

@@ -201,7 +201,12 @@ function parseCSV(csvContent, hasHeader = true) {
 }
 
 const MAX_REDIRECTS = 5;
-const MAX_RESPONSE_SIZE = 200 * 1024 * 1024; // 200MB
+// Env-tunable (idiom: parseInt(env) || default). Default 512MB — the OSV npm all.zip
+// crossed the old 200MB cap on 2026-07 (measured 200.04MB), which silently dropped the
+// entire OSV npm dump AND emptied knownIds (→ OSSF re-fetched all ~31k entries one-by-one).
+// Bump gives headroom for growth; lower it via MUADDIB_MAX_RESPONSE_MB on RAM-constrained
+// hosts (the whole response is buffered in memory — streaming-to-disk is the long-term fix).
+const MAX_RESPONSE_SIZE = (parseInt(process.env.MUADDIB_MAX_RESPONSE_MB, 10) || 512) * 1024 * 1024;
 const MAX_ENTRY_UNCOMPRESSED = 50 * 1024 * 1024;   // 50MB per zip entry
 const MAX_TOTAL_UNCOMPRESSED = 500 * 1024 * 1024;   // 500MB total budget per zip
 

package/src/ml/jsonl-writer.js

@@ -37,6 +37,20 @@ const DEFAULT_TRAINING_FILE = path.join(__dirname, '..', '..', 'data', 'ml-train
 let TRAINING_FILE = DEFAULT_TRAINING_FILE;
 const MAX_JSONL_SIZE = 100 * 1024 * 1024; // 100MB rotation threshold
 
+// Benign down-sampling: 'clean' records are ~88% of the stream and near-identical
+// (the same popular packages re-scanned on every backlog replay / monitor restart).
+// They balloon the file without adding ML signal — negatives are re-derived from
+// GHSA/GT at retrain time, not from these self-labels. Keep 1 in CLEAN_SAMPLE_RATE
+// clean records; every non-clean outcome (suspect/confirmed/unconfirmed/…) is always
+// written. Gating here (the single write chokepoint) rather than in the monitor's
+// recordTrainingSample keeps the separate scan-ledger (coverage/throughput) complete.
+const CLEAN_SAMPLE_RATE = 100; // 1% of 'clean' records are persisted
+let _cleanCounter = 0;
+
+// Rotation retention: keep only the MAX_ROTATED_SNAPSHOTS most recent rotated files
+// on disk; prune the rest at each rotation (see pruneOldSnapshots).
+const MAX_ROTATED_SNAPSHOTS = 7;
+
 // In-memory line counter. null = needs recompute (cold boot, file rewrite, or
 // path swap). Maintained incrementally by appendRecord and invalidated by
 // relabelRecords and setTrainingFile. Prior to this cache, getStats read the
@@ -50,6 +64,7 @@ let _cachedLineCount = null;
 function setTrainingFile(filePath) {
   TRAINING_FILE = filePath;
   _cachedLineCount = null; // different file → recompute on next getStats
+  _cleanCounter = 0;       // reset benign-sampling counter for test isolation
 }
 
 /**
@@ -58,6 +73,7 @@ function setTrainingFile(filePath) {
 function resetTrainingFile() {
   TRAINING_FILE = DEFAULT_TRAINING_FILE;
   _cachedLineCount = null;
+  _cleanCounter = 0;       // reset benign-sampling counter for test isolation
 }
 
 /**
@@ -66,6 +82,12 @@ function resetTrainingFile() {
  */
 function appendRecord(record) {
   try {
+    // Down-sample benign records (see CLEAN_SAMPLE_RATE). Every non-clean outcome
+    // is always persisted; only 1 in CLEAN_SAMPLE_RATE 'clean' records is kept.
+    if (record && record.label === 'clean' && (_cleanCounter++ % CLEAN_SAMPLE_RATE) !== 0) {
+      return;
+    }
+
     const dir = path.dirname(TRAINING_FILE);
     if (!fs.existsSync(dir)) {
       fs.mkdirSync(dir, { recursive: true });
@@ -103,11 +125,49 @@ function maybeRotate() {
     fs.renameSync(TRAINING_FILE, rotatedName);
     _cachedLineCount = 0; // fresh file starts empty
     console.log(`[ML] Rotated training file → ${path.basename(rotatedName)} (${(stat.size / 1024 / 1024).toFixed(1)}MB)`);
+    pruneOldSnapshots();
   } catch (err) {
     console.error(`[ML] Rotation failed: ${err.message}`);
   }
 }
 
+/**
+ * Delete rotated snapshots beyond the MAX_ROTATED_SNAPSHOTS most recent.
+ * Matches ONLY timestamped rotation files (`<base>-<ISO>.jsonl`, e.g.
+ * ml-training-2026-07-08T08-28-49-263Z.jsonl) so the live file and sibling data
+ * files (ml-training-datadog.jsonl, ml-training-relabeled.jsonl,
+ * ml-corpus-consolidated.jsonl, …) are never touched.
+ */
+function pruneOldSnapshots() {
+  try {
+    const dir = path.dirname(TRAINING_FILE);
+    const base = path.basename(TRAINING_FILE, '.jsonl'); // e.g. 'ml-training'
+    const escBase = base.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+    const rotatedRe = new RegExp(`^${escBase}-\\d{4}-\\d{2}-\\d{2}T[0-9-]+Z\\.jsonl$`);
+
+    const rotated = fs.readdirSync(dir)
+      .filter(f => rotatedRe.test(f))
+      .map(f => {
+        const full = path.join(dir, f);
+        try { return { full, mtime: fs.statSync(full).mtimeMs }; }
+        catch { return null; }
+      })
+      .filter(Boolean)
+      .sort((a, b) => b.mtime - a.mtime); // most recent first
+
+    for (const { full } of rotated.slice(MAX_ROTATED_SNAPSHOTS)) {
+      try {
+        fs.unlinkSync(full);
+        console.log(`[ML] Pruned old snapshot → ${path.basename(full)}`);
+      } catch (e) {
+        console.warn(`[ML] Prune failed for ${path.basename(full)}: ${e.message}`);
+      }
+    }
+  } catch (err) {
+    console.error(`[ML] Snapshot prune failed: ${err.message}`);
+  }
+}
+
 /**
  * Read all records from the current JSONL file.
  * Useful for offline analysis and model training.

package/src/monitor/daemon.js

@@ -32,7 +32,7 @@ const { poll, getPollBackoffMs, SOFT_BACKPRESSURE_THRESHOLD } = require('./inges
 const { ensureWorkers, drainWorkers, getTargetConcurrency, setTargetConcurrency, getActiveWorkers, terminateAllWorkers, getInFlightItems, computeInterruptDisposition } = require('./queue.js');
 const { computeTarget, ADJUST_INTERVAL_MS, BASE_CONCURRENCY } = require('./adaptive-concurrency.js');
 const { startHealthcheck } = require('./healthcheck.js');
-const { startLagSampler } = require('./event-loop-monitor.js');
+const { startLagSampler, runInstrumented } = require('./event-loop-monitor.js');
 const { startDeferredWorker, stopDeferredWorker, persistDeferredQueue, restoreDeferredQueue, clearDeferredQueue } = require('./deferred-sandbox.js');
 const { evictFromScanQueueBulk, enqueueScan } = require('./scan-queue.js');
 const { isSpillEnabled, shouldDrain, drainBacklog, getBacklogSize } = require('./spill.js');
@@ -373,23 +373,28 @@ function cleanupOrphanTmpDirs() {
 }
 
 function cleanupOrphanContainers() {
-  try {
-    // List running containers with the sandbox name prefix (npm-audit-*)
-    const output = execFileSync('docker', ['ps', '-q', '--filter', 'name=npm-audit-'], {
-      stdio: ['pipe', 'pipe', 'pipe'],
-      timeout: 10000
-    }).toString().trim();
-    if (!output) return;
-    const ids = output.split(/\s+/).filter(Boolean);
-    for (const id of ids) {
-      try {
-        execFileSync('docker', ['rm', '-f', id], { stdio: 'pipe', timeout: 10000 });
-      } catch {}
+  // Instrumented (2026-07-09): `docker ps` + a sequential `docker rm -f` per orphan
+  // (10s timeout EACH, count unbounded) is a synchronous main-thread op. Name it so a
+  // concurrent lag stall attributes here instead of op:null (event-loop-monitor.js).
+  return runInstrumented('maint:orphan-containers', null, () => {
+    try {
+      // List running containers with the sandbox name prefix (npm-audit-*)
+      const output = execFileSync('docker', ['ps', '-q', '--filter', 'name=npm-audit-'], {
+        stdio: ['pipe', 'pipe', 'pipe'],
+        timeout: 10000
+      }).toString().trim();
+      if (!output) return;
+      const ids = output.split(/\s+/).filter(Boolean);
+      for (const id of ids) {
+        try {
+          execFileSync('docker', ['rm', '-f', id], { stdio: 'pipe', timeout: 10000 });
+        } catch {}
+      }
+      console.log(`[MONITOR] Cleaned up ${ids.length} orphan sandbox container(s)`);
+    } catch {
+      // Docker not available or command failed — skip silently
     }
-    console.log(`[MONITOR] Cleaned up ${ids.length} orphan sandbox container(s)`);
-  } catch {
-    // Docker not available or command failed — skip silently
-  }
+  });
 }
 
 /**
@@ -401,28 +406,34 @@ function cleanupOrphanContainers() {
  */
 function cleanupRunscOrphans(maxAgeMs = 3600_000) {
   const runscDir = process.env.MUADDIB_GVISOR_LOG_DIR || '/tmp/runsc';
-  try {
-    if (!fs.existsSync(runscDir)) return 0;
-    const entries = fs.readdirSync(runscDir);
-    const now = Date.now();
-    let cleaned = 0;
-    for (const entry of entries) {
-      const fullPath = path.join(runscDir, entry);
-      try {
-        const stat = fs.statSync(fullPath);
-        if (now - stat.mtimeMs > maxAgeMs) {
-          fs.rmSync(fullPath, { recursive: true, force: true });
-          cleaned++;
-        }
-      } catch { /* skip unreadable entries */ }
-    }
-    if (cleaned > 0) {
-      console.log(`[MONITOR] Cleaned up ${cleaned} orphan runsc dir(s) in ${runscDir}`);
+  // Instrumented (2026-07-09): readdir + per-entry statSync + recursive rmSync over
+  // /tmp/runsc is a synchronous main-thread op that blocks the poll loop (RSS breaker,
+  // governor feed) for as long as it runs — the prime suspect for the op:null
+  // multi-minute stalls in loop-stalls.jsonl. Name it (also covers the boot call).
+  return runInstrumented('maint:runsc-cleanup', { dir: runscDir }, () => {
+    try {
+      if (!fs.existsSync(runscDir)) return 0;
+      const entries = fs.readdirSync(runscDir);
+      const now = Date.now();
+      let cleaned = 0;
+      for (const entry of entries) {
+        const fullPath = path.join(runscDir, entry);
+        try {
+          const stat = fs.statSync(fullPath);
+          if (now - stat.mtimeMs > maxAgeMs) {
+            fs.rmSync(fullPath, { recursive: true, force: true });
+            cleaned++;
+          }
+        } catch { /* skip unreadable entries */ }
+      }
+      if (cleaned > 0) {
+        console.log(`[MONITOR] Cleaned up ${cleaned} orphan runsc dir(s) in ${runscDir}`);
+      }
+      return cleaned;
+    } catch {
+      return 0;
     }
-    return cleaned;
-  } catch {
-    return 0;
-  }
+  });
 }
 
 /**
@@ -445,9 +456,9 @@ function checkDiskSpace() {
 
     // Top consumers in /tmp/
     try {
-      const tmpDu = execFileSync('du', ['-sh', '--max-depth=1', '/tmp/'], {
+      const tmpDu = runInstrumented('maint:disk-du-tmp', null, () => execFileSync('du', ['-sh', '--max-depth=1', '/tmp/'], {
         encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe'], timeout: 10000
-      });
+      }));
       const lines = tmpDu.trim().split('\n')
         .map(l => { const m = l.match(/^([\d.]+[KMGT]?)\s+(.+)/); return m ? { size: m[1], path: m[2] } : null; })
         .filter(Boolean)
@@ -462,9 +473,9 @@ function checkDiskSpace() {
     // Top consumers in data/
     const dataDir = path.join(__dirname, '..', '..', 'data');
     try {
-      const dataDu = execFileSync('du', ['-sh', '--max-depth=1', dataDir], {
+      const dataDu = runInstrumented('maint:disk-du-data', null, () => execFileSync('du', ['-sh', '--max-depth=1', dataDir], {
         encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe'], timeout: 10000
-      });
+      }));
       const lines = dataDu.trim().split('\n')
         .map(l => { const m = l.match(/^([\d.]+[KMGT]?)\s+(.+)/); return m ? { size: m[1], path: m[2] } : null; })
         .filter(Boolean)
@@ -1328,10 +1339,13 @@ async function startMonitor(options, stats, dailyAlerts, recentlyScanned, downlo
 
     // Hourly stats report + cache purge + runsc cleanup + memory pruning
     if (Date.now() - stats.lastReportTime >= 3600_000) {
-      reportStats(stats);
-      purgeTarballCache();
+      // Hourly maintenance runs synchronously on the poll loop; instrument each step so
+      // a concurrent lag stall names the culprit (was op:null). cleanupRunscOrphans
+      // self-wraps in its own body (that also covers the boot call at startup).
+      runInstrumented('maint:report-stats', null, () => reportStats(stats));
+      runInstrumented('maint:tarball-purge', null, () => purgeTarballCache());
       cleanupRunscOrphans();
-      pruneMemoryCaches(recentlyScanned, downloadsCache, alertedPackageRules);
+      runInstrumented('maint:prune-caches', null, () => pruneMemoryCaches(recentlyScanned, downloadsCache, alertedPackageRules));
     }
 
     // Daily webhook report at 08:00 Paris time

package/src/monitor/event-loop-monitor.js

@@ -84,6 +84,23 @@ function endOp(token) {
   _current = null;
 }
 
+/**
+ * Run a synchronous, potentially-blocking main-thread op UNDER a breadcrumb so a
+ * concurrent lag-sampler stall attributes itself to THIS op instead of reporting
+ * `op:null`. Mirrors the sandbox `dockerSync` wrapper (sandbox/index.js). Passes
+ * fn's return value through; the breadcrumb is cleared in `finally` even when fn
+ * throws (and the error re-thrown). Single-slot model (see file header) — the
+ * wrapped fn must NOT itself call beginOp/endOp/runInstrumented (no nesting).
+ */
+function runInstrumented(label, meta, fn) {
+  const token = beginOp(label, meta);
+  try {
+    return fn();
+  } finally {
+    endOp(token);
+  }
+}
+
 /**
  * The op whose lifetime overlaps [windowStartMs, windowEndMs] — i.e. the op on
  * the thread during the blocked window. Prefers the still-running op, else the
@@ -191,6 +208,7 @@ function _reset(clockFn) {
 module.exports = {
   beginOp,
   endOp,
+  runInstrumented,
   opOverlapping,
   configure,
   observeTick,

package/src/scanner/ast-detectors/handle-call-expression.js

@@ -219,9 +219,18 @@ function handleCallExpression(node, ctx) {
         file: ctx.relFile
       });
     } else if (arg.type === 'CallExpression') {
-      const argCallName = getCallName(arg);
-      // Skip safe patterns: require(path.join(...)), require(path.resolve(...))
-      if (argCallName !== 'path.join' && argCallName !== 'path.resolve') {
+      // Skip safe path-builder patterns: require(path.join(...)), require(path.resolve(...)).
+      // NB: getCallName(arg) returns the bare method name ('join'/'resolve'), NOT 'path.join', so
+      // the previous `argCallName !== 'path.join'` guard never matched — a dead check that let the
+      // standard require(path.join(...)) idiom raise a HIGH dynamic_require FP. Inspect the
+      // MemberExpression object directly instead. Real obfuscation still fires: require(atob(...)),
+      // require(decoder.decode(...)), require(Buffer.from(...).toString()) all have object!=='path'.
+      const callee = arg.callee;
+      const isPathBuilder =
+        callee.type === 'MemberExpression' &&
+        callee.object && callee.object.name === 'path' &&
+        callee.property && (callee.property.name === 'join' || callee.property.name === 'resolve');
+      if (!isPathBuilder) {
         const hasDecode = containsDecodePattern(arg);
         ctx.threats.push({
           type: 'dynamic_require',

package/stats.json

@@ -1,6 +1,6 @@
 {
   "_comment": "GENERATED by scripts/collect-stats.js — do not edit by hand. Run `npm run docs:stats`.",
-  "version": "2.11.159",
+  "version": "2.11.161",
   "scanners": 21,
   "rulesTotal": 275,
   "rulesCore": 270,