muaddib-scanner 2.11.160 → 2.11.162

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "muaddib-scanner",
-  "version": "2.11.160",
+  "version": "2.11.162",
   "description": "Supply-chain threat detection & response for npm & PyPI/Python",
   "main": "src/index.js",
   "bin": {
@@ -17,6 +17,7 @@
     "docs:stats": "node scripts/collect-stats.js && node scripts/sync-stats.js",
     "docs:stats:tests": "node scripts/collect-stats.js --with-tests && node scripts/sync-stats.js",
     "docs:stats:check": "node scripts/collect-stats.js --check && node scripts/sync-stats.js --check",
+    "version": "npm run docs:stats && git diff --name-only | xargs -r git add",
     "compress-iocs": "node -e \"const fs=require('fs');const zlib=require('zlib');zlib.gzip(fs.readFileSync('src/ioc/data/iocs.json'),(e,b)=>{if(e)throw e;fs.writeFileSync('iocs.json.gz',b);console.log('Compressed: '+b.length+' bytes')})\"",
     "prepublishOnly": "node -e \"if(!process.env.CI){console.error('ERR: Publish via CI workflow (tag v* push). Local publishes are disabled.');process.exit(1)}\""
   },
@@ -53,13 +54,13 @@
     "@inquirer/prompts": "8.5.2",
     "acorn": "8.17.0",
     "acorn-walk": "8.3.5",
-    "adm-zip": "0.5.17",
+    "adm-zip": "0.5.18",
     "js-yaml": "5.1.0",
     "web-tree-sitter": "^0.26.9"
   },
   "devDependencies": {
     "@eslint/js": "10.0.1",
-    "eslint": "10.5.0",
+    "eslint": "10.6.0",
     "eslint-plugin-security": "^4.0.0",
     "globals": "17.7.0"
   }

package/{self-scan-v2.11.160.json → self-scan-v2.11.162.json}

@@ -1,6 +1,6 @@
 {
   "target": "node_modules",
-  "timestamp": "2026-07-07T14:05:05.690Z",
+  "timestamp": "2026-07-10T07:45:05.079Z",
   "threats": [
     {
       "type": "string_mutation_obfuscation",

package/src/ioc/scraper.js

@@ -201,7 +201,12 @@ function parseCSV(csvContent, hasHeader = true) {
 }
 
 const MAX_REDIRECTS = 5;
-const MAX_RESPONSE_SIZE = 200 * 1024 * 1024; // 200MB
+// Env-tunable (idiom: parseInt(env) || default). Default 512MB — the OSV npm all.zip
+// crossed the old 200MB cap on 2026-07 (measured 200.04MB), which silently dropped the
+// entire OSV npm dump AND emptied knownIds (→ OSSF re-fetched all ~31k entries one-by-one).
+// Bump gives headroom for growth; lower it via MUADDIB_MAX_RESPONSE_MB on RAM-constrained
+// hosts (the whole response is buffered in memory — streaming-to-disk is the long-term fix).
+const MAX_RESPONSE_SIZE = (parseInt(process.env.MUADDIB_MAX_RESPONSE_MB, 10) || 512) * 1024 * 1024;
 const MAX_ENTRY_UNCOMPRESSED = 50 * 1024 * 1024;   // 50MB per zip entry
 const MAX_TOTAL_UNCOMPRESSED = 500 * 1024 * 1024;   // 500MB total budget per zip
 

package/src/ml/jsonl-writer.js

@@ -37,6 +37,20 @@ const DEFAULT_TRAINING_FILE = path.join(__dirname, '..', '..', 'data', 'ml-train
 let TRAINING_FILE = DEFAULT_TRAINING_FILE;
 const MAX_JSONL_SIZE = 100 * 1024 * 1024; // 100MB rotation threshold
 
+// Benign down-sampling: 'clean' records are ~88% of the stream and near-identical
+// (the same popular packages re-scanned on every backlog replay / monitor restart).
+// They balloon the file without adding ML signal — negatives are re-derived from
+// GHSA/GT at retrain time, not from these self-labels. Keep 1 in CLEAN_SAMPLE_RATE
+// clean records; every non-clean outcome (suspect/confirmed/unconfirmed/…) is always
+// written. Gating here (the single write chokepoint) rather than in the monitor's
+// recordTrainingSample keeps the separate scan-ledger (coverage/throughput) complete.
+const CLEAN_SAMPLE_RATE = 100; // 1% of 'clean' records are persisted
+let _cleanCounter = 0;
+
+// Rotation retention: keep only the MAX_ROTATED_SNAPSHOTS most recent rotated files
+// on disk; prune the rest at each rotation (see pruneOldSnapshots).
+const MAX_ROTATED_SNAPSHOTS = 7;
+
 // In-memory line counter. null = needs recompute (cold boot, file rewrite, or
 // path swap). Maintained incrementally by appendRecord and invalidated by
 // relabelRecords and setTrainingFile. Prior to this cache, getStats read the
@@ -50,6 +64,7 @@ let _cachedLineCount = null;
 function setTrainingFile(filePath) {
   TRAINING_FILE = filePath;
   _cachedLineCount = null; // different file → recompute on next getStats
+  _cleanCounter = 0;       // reset benign-sampling counter for test isolation
 }
 
 /**
@@ -58,6 +73,7 @@ function setTrainingFile(filePath) {
 function resetTrainingFile() {
   TRAINING_FILE = DEFAULT_TRAINING_FILE;
   _cachedLineCount = null;
+  _cleanCounter = 0;       // reset benign-sampling counter for test isolation
 }
 
 /**
@@ -66,6 +82,12 @@ function resetTrainingFile() {
  */
 function appendRecord(record) {
   try {
+    // Down-sample benign records (see CLEAN_SAMPLE_RATE). Every non-clean outcome
+    // is always persisted; only 1 in CLEAN_SAMPLE_RATE 'clean' records is kept.
+    if (record && record.label === 'clean' && (_cleanCounter++ % CLEAN_SAMPLE_RATE) !== 0) {
+      return;
+    }
+
     const dir = path.dirname(TRAINING_FILE);
     if (!fs.existsSync(dir)) {
       fs.mkdirSync(dir, { recursive: true });
@@ -103,11 +125,49 @@ function maybeRotate() {
     fs.renameSync(TRAINING_FILE, rotatedName);
     _cachedLineCount = 0; // fresh file starts empty
     console.log(`[ML] Rotated training file → ${path.basename(rotatedName)} (${(stat.size / 1024 / 1024).toFixed(1)}MB)`);
+    pruneOldSnapshots();
   } catch (err) {
     console.error(`[ML] Rotation failed: ${err.message}`);
   }
 }
 
+/**
+ * Delete rotated snapshots beyond the MAX_ROTATED_SNAPSHOTS most recent.
+ * Matches ONLY timestamped rotation files (`<base>-<ISO>.jsonl`, e.g.
+ * ml-training-2026-07-08T08-28-49-263Z.jsonl) so the live file and sibling data
+ * files (ml-training-datadog.jsonl, ml-training-relabeled.jsonl,
+ * ml-corpus-consolidated.jsonl, …) are never touched.
+ */
+function pruneOldSnapshots() {
+  try {
+    const dir = path.dirname(TRAINING_FILE);
+    const base = path.basename(TRAINING_FILE, '.jsonl'); // e.g. 'ml-training'
+    const escBase = base.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+    const rotatedRe = new RegExp(`^${escBase}-\\d{4}-\\d{2}-\\d{2}T[0-9-]+Z\\.jsonl$`);
+
+    const rotated = fs.readdirSync(dir)
+      .filter(f => rotatedRe.test(f))
+      .map(f => {
+        const full = path.join(dir, f);
+        try { return { full, mtime: fs.statSync(full).mtimeMs }; }
+        catch { return null; }
+      })
+      .filter(Boolean)
+      .sort((a, b) => b.mtime - a.mtime); // most recent first
+
+    for (const { full } of rotated.slice(MAX_ROTATED_SNAPSHOTS)) {
+      try {
+        fs.unlinkSync(full);
+        console.log(`[ML] Pruned old snapshot → ${path.basename(full)}`);
+      } catch (e) {
+        console.warn(`[ML] Prune failed for ${path.basename(full)}: ${e.message}`);
+      }
+    }
+  } catch (err) {
+    console.error(`[ML] Snapshot prune failed: ${err.message}`);
+  }
+}
+
 /**
  * Read all records from the current JSONL file.
  * Useful for offline analysis and model training.

package/src/monitor/daemon.js

@@ -32,7 +32,7 @@ const { poll, getPollBackoffMs, SOFT_BACKPRESSURE_THRESHOLD } = require('./inges
 const { ensureWorkers, drainWorkers, getTargetConcurrency, setTargetConcurrency, getActiveWorkers, terminateAllWorkers, getInFlightItems, computeInterruptDisposition } = require('./queue.js');
 const { computeTarget, ADJUST_INTERVAL_MS, BASE_CONCURRENCY } = require('./adaptive-concurrency.js');
 const { startHealthcheck } = require('./healthcheck.js');
-const { startLagSampler } = require('./event-loop-monitor.js');
+const { startLagSampler, runInstrumented } = require('./event-loop-monitor.js');
 const { startDeferredWorker, stopDeferredWorker, persistDeferredQueue, restoreDeferredQueue, clearDeferredQueue } = require('./deferred-sandbox.js');
 const { evictFromScanQueueBulk, enqueueScan } = require('./scan-queue.js');
 const { isSpillEnabled, shouldDrain, drainBacklog, getBacklogSize } = require('./spill.js');
@@ -373,23 +373,28 @@ function cleanupOrphanTmpDirs() {
 }
 
 function cleanupOrphanContainers() {
-  try {
-    // List running containers with the sandbox name prefix (npm-audit-*)
-    const output = execFileSync('docker', ['ps', '-q', '--filter', 'name=npm-audit-'], {
-      stdio: ['pipe', 'pipe', 'pipe'],
-      timeout: 10000
-    }).toString().trim();
-    if (!output) return;
-    const ids = output.split(/\s+/).filter(Boolean);
-    for (const id of ids) {
-      try {
-        execFileSync('docker', ['rm', '-f', id], { stdio: 'pipe', timeout: 10000 });
-      } catch {}
+  // Instrumented (2026-07-09): `docker ps` + a sequential `docker rm -f` per orphan
+  // (10s timeout EACH, count unbounded) is a synchronous main-thread op. Name it so a
+  // concurrent lag stall attributes here instead of op:null (event-loop-monitor.js).
+  return runInstrumented('maint:orphan-containers', null, () => {
+    try {
+      // List running containers with the sandbox name prefix (npm-audit-*)
+      const output = execFileSync('docker', ['ps', '-q', '--filter', 'name=npm-audit-'], {
+        stdio: ['pipe', 'pipe', 'pipe'],
+        timeout: 10000
+      }).toString().trim();
+      if (!output) return;
+      const ids = output.split(/\s+/).filter(Boolean);
+      for (const id of ids) {
+        try {
+          execFileSync('docker', ['rm', '-f', id], { stdio: 'pipe', timeout: 10000 });
+        } catch {}
+      }
+      console.log(`[MONITOR] Cleaned up ${ids.length} orphan sandbox container(s)`);
+    } catch {
+      // Docker not available or command failed — skip silently
     }
-    console.log(`[MONITOR] Cleaned up ${ids.length} orphan sandbox container(s)`);
-  } catch {
-    // Docker not available or command failed — skip silently
-  }
+  });
 }
 
 /**
@@ -401,28 +406,34 @@ function cleanupOrphanContainers() {
  */
 function cleanupRunscOrphans(maxAgeMs = 3600_000) {
   const runscDir = process.env.MUADDIB_GVISOR_LOG_DIR || '/tmp/runsc';
-  try {
-    if (!fs.existsSync(runscDir)) return 0;
-    const entries = fs.readdirSync(runscDir);
-    const now = Date.now();
-    let cleaned = 0;
-    for (const entry of entries) {
-      const fullPath = path.join(runscDir, entry);
-      try {
-        const stat = fs.statSync(fullPath);
-        if (now - stat.mtimeMs > maxAgeMs) {
-          fs.rmSync(fullPath, { recursive: true, force: true });
-          cleaned++;
-        }
-      } catch { /* skip unreadable entries */ }
-    }
-    if (cleaned > 0) {
-      console.log(`[MONITOR] Cleaned up ${cleaned} orphan runsc dir(s) in ${runscDir}`);
+  // Instrumented (2026-07-09): readdir + per-entry statSync + recursive rmSync over
+  // /tmp/runsc is a synchronous main-thread op that blocks the poll loop (RSS breaker,
+  // governor feed) for as long as it runs — the prime suspect for the op:null
+  // multi-minute stalls in loop-stalls.jsonl. Name it (also covers the boot call).
+  return runInstrumented('maint:runsc-cleanup', { dir: runscDir }, () => {
+    try {
+      if (!fs.existsSync(runscDir)) return 0;
+      const entries = fs.readdirSync(runscDir);
+      const now = Date.now();
+      let cleaned = 0;
+      for (const entry of entries) {
+        const fullPath = path.join(runscDir, entry);
+        try {
+          const stat = fs.statSync(fullPath);
+          if (now - stat.mtimeMs > maxAgeMs) {
+            fs.rmSync(fullPath, { recursive: true, force: true });
+            cleaned++;
+          }
+        } catch { /* skip unreadable entries */ }
+      }
+      if (cleaned > 0) {
+        console.log(`[MONITOR] Cleaned up ${cleaned} orphan runsc dir(s) in ${runscDir}`);
+      }
+      return cleaned;
+    } catch {
+      return 0;
     }
-    return cleaned;
-  } catch {
-    return 0;
-  }
+  });
 }
 
 /**
@@ -445,9 +456,9 @@ function checkDiskSpace() {
 
     // Top consumers in /tmp/
     try {
-      const tmpDu = execFileSync('du', ['-sh', '--max-depth=1', '/tmp/'], {
+      const tmpDu = runInstrumented('maint:disk-du-tmp', null, () => execFileSync('du', ['-sh', '--max-depth=1', '/tmp/'], {
         encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe'], timeout: 10000
-      });
+      }));
       const lines = tmpDu.trim().split('\n')
         .map(l => { const m = l.match(/^([\d.]+[KMGT]?)\s+(.+)/); return m ? { size: m[1], path: m[2] } : null; })
         .filter(Boolean)
@@ -462,9 +473,9 @@ function checkDiskSpace() {
     // Top consumers in data/
     const dataDir = path.join(__dirname, '..', '..', 'data');
     try {
-      const dataDu = execFileSync('du', ['-sh', '--max-depth=1', dataDir], {
+      const dataDu = runInstrumented('maint:disk-du-data', null, () => execFileSync('du', ['-sh', '--max-depth=1', dataDir], {
         encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe'], timeout: 10000
-      });
+      }));
       const lines = dataDu.trim().split('\n')
         .map(l => { const m = l.match(/^([\d.]+[KMGT]?)\s+(.+)/); return m ? { size: m[1], path: m[2] } : null; })
         .filter(Boolean)
@@ -1328,10 +1339,13 @@ async function startMonitor(options, stats, dailyAlerts, recentlyScanned, downlo
 
     // Hourly stats report + cache purge + runsc cleanup + memory pruning
     if (Date.now() - stats.lastReportTime >= 3600_000) {
-      reportStats(stats);
-      purgeTarballCache();
+      // Hourly maintenance runs synchronously on the poll loop; instrument each step so
+      // a concurrent lag stall names the culprit (was op:null). cleanupRunscOrphans
+      // self-wraps in its own body (that also covers the boot call at startup).
+      runInstrumented('maint:report-stats', null, () => reportStats(stats));
+      runInstrumented('maint:tarball-purge', null, () => purgeTarballCache());
       cleanupRunscOrphans();
-      pruneMemoryCaches(recentlyScanned, downloadsCache, alertedPackageRules);
+      runInstrumented('maint:prune-caches', null, () => pruneMemoryCaches(recentlyScanned, downloadsCache, alertedPackageRules));
     }
 
     // Daily webhook report at 08:00 Paris time

package/src/monitor/event-loop-monitor.js

@@ -84,6 +84,23 @@ function endOp(token) {
   _current = null;
 }
 
+/**
+ * Run a synchronous, potentially-blocking main-thread op UNDER a breadcrumb so a
+ * concurrent lag-sampler stall attributes itself to THIS op instead of reporting
+ * `op:null`. Mirrors the sandbox `dockerSync` wrapper (sandbox/index.js). Passes
+ * fn's return value through; the breadcrumb is cleared in `finally` even when fn
+ * throws (and the error re-thrown). Single-slot model (see file header) — the
+ * wrapped fn must NOT itself call beginOp/endOp/runInstrumented (no nesting).
+ */
+function runInstrumented(label, meta, fn) {
+  const token = beginOp(label, meta);
+  try {
+    return fn();
+  } finally {
+    endOp(token);
+  }
+}
+
 /**
  * The op whose lifetime overlaps [windowStartMs, windowEndMs] — i.e. the op on
  * the thread during the blocked window. Prefers the still-running op, else the
@@ -191,6 +208,7 @@ function _reset(clockFn) {
 module.exports = {
   beginOp,
   endOp,
+  runInstrumented,
   opOverlapping,
   configure,
   observeTick,

package/stats.json

@@ -1,6 +1,6 @@
 {
   "_comment": "GENERATED by scripts/collect-stats.js — do not edit by hand. Run `npm run docs:stats`.",
-  "version": "2.11.160",
+  "version": "2.11.162",
   "scanners": 21,
   "rulesTotal": 275,
   "rulesCore": 270,

package/zizmor.yml

@@ -0,0 +1,18 @@
+# zizmor configuration — https://docs.zizmor.sh/configuration/
+#
+# Suppress a finding ONLY when it is a confirmed false positive, and keep the
+# rationale on the same line. Schema:
+#
+#   rules:
+#     <audit-id>:                 # e.g. template-injection, artipacked, cache-poisoning
+#       ignore:
+#         - <workflow>.yml         # ignore the whole file
+#         - <workflow>.yml:LINE    # ignore one line
+#         - <workflow>.yml:LINE:COL# ignore one exact finding
+#
+# Policy for this repo: prefer FIXING findings over ignoring them. As of the first
+# run every finding is real (template-injection in publish.yml, artipacked on the
+# checkout steps, cache-poisoning, adhoc-packages), so nothing is suppressed here.
+# Add an entry below only after confirming a genuine false positive.
+
+rules: {}