muaddib-scanner 2.10.43 → 2.10.45

package/README.md

@@ -30,7 +30,7 @@
 
 npm and PyPI supply-chain attacks are exploding. Shai-Hulud compromised 25K+ repos in 2025. Existing tools detect threats but don't help you respond.
 
-MUAD'DIB combines **14 parallel scanners** (195 detection rules), a **deobfuscation engine**, **inter-module dataflow analysis**, **compound scoring**, **ML classifiers** (XGBoost), and Docker sandbox to detect known threats and suspicious behavioral patterns in npm and PyPI packages.
+MUAD'DIB combines **14 parallel scanners** (200 detection rules), a **deobfuscation engine**, **inter-module dataflow analysis**, **compound scoring**, **ML classifiers** (XGBoost), and gVisor/Docker sandbox to detect known threats and suspicious behavioral patterns in npm and PyPI packages.
 
 ---
 
@@ -195,7 +195,7 @@ muaddib replay                     # Ground truth validation (46/49 TPR)
 | GitHub Actions | Shai-Hulud backdoor detection |
 | Hash Scanner | Known malicious file hashes |
 
-### 195 detection rules
+### 200 detection rules
 
 All rules are mapped to MITRE ATT&CK techniques. See [SECURITY.md](SECURITY.md#detection-rules-v21021) for the complete rules reference.
 
@@ -271,7 +271,7 @@ With pre-commit framework:
 ```yaml
 repos:
   - repo: https://github.com/DNSZLSK/muad-dib
-    rev: v2.10.31
+    rev: v2.10.43
     hooks:
       - id: muaddib-scan
 ```
@@ -288,7 +288,7 @@ repos:
 | **FPR** (Benign random) | **7.5%** (15/200) | 200 random npm packages, stratified sampling |
 | **ADR** (Adversarial + Holdout) | **94.0%** (101/107) | 67 adversarial + 40 holdout (107 available on disk), global threshold=20 |
 
-**2868 tests** across 62 files. **195 rules** (190 RULES + 5 PARANOID).
+**3034 tests** across 65 files. **200 rules** (195 RULES + 5 PARANOID).
 
 > **Methodology caveats:**
 > - TPR measured on 49 Node.js attack samples (3 browser-only excluded from 51 total)
@@ -329,7 +329,7 @@ npm test
 
 ### Testing
 
-- **2868 tests** across 62 modular test files
+- **3034 tests** across 65 modular test files
 - **56 fuzz tests** - Malformed inputs, ReDoS, unicode, binary
 - **Datadog 17K benchmark** - 14,587 confirmed malware samples (in-scope)
 - **Ground truth validation** - 51 real-world attacks (93.9% TPR)
@@ -351,7 +351,7 @@ npm test
 - [Evaluation Methodology](docs/EVALUATION_METHODOLOGY.md) - Experimental protocol, holdout scores
 - [Threat Model](docs/threat-model.md) - What MUAD'DIB detects and doesn't detect
 - [Adversarial Evaluation](ADVERSARIAL.md) - Red team samples and ADR results
-- [Security Policy](SECURITY.md) - Detection rules reference (195 rules)
+- [Security Policy](SECURITY.md) - Detection rules reference (200 rules)
 - [Security Audit](docs/SECURITY_AUDIT.md) - Bypass validation report
 - [FP Analysis](docs/EVALUATION.md) - Historical false positive analysis
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "muaddib-scanner",
-  "version": "2.10.43",
+  "version": "2.10.45",
   "description": "Supply-chain threat detection & response for npm & PyPI/Python",
   "main": "src/index.js",
   "bin": {

package/src/monitor/daemon.js

@@ -4,7 +4,7 @@ const path = require('path');
 const os = require('os');
 const { isDockerAvailable, SANDBOX_CONCURRENCY_MAX } = require('../sandbox/index.js');
 const { setVerboseMode, isSandboxEnabled, isCanaryEnabled, isLlmDetectiveEnabled, getLlmDetectiveMode } = require('./classify.js');
-const { loadState, saveState, loadDailyStats, saveDailyStats, purgeTarballCache, getParisHour } = require('./state.js');
+const { loadState, saveState, loadDailyStats, saveDailyStats, purgeTarballCache, getParisHour, atomicWriteFileSync } = require('./state.js');
 const { isTemporalEnabled, isTemporalAstEnabled, isTemporalPublishEnabled, isTemporalMaintainerEnabled } = require('./temporal.js');
 const { pendingGrouped, flushScopeGroup, sendDailyReport, DAILY_REPORT_HOUR } = require('./webhook.js');
 const { poll } = require('./ingestion.js');
@@ -14,11 +14,88 @@ const { startHealthcheck } = require('./healthcheck.js');
 const POLL_INTERVAL = 60_000;
 const PROCESS_LOOP_INTERVAL = 2_000;    // Queue check interval when empty
 const QUEUE_WARNING_THRESHOLD = 5_000;  // Warn if queue depth exceeds this
+const QUEUE_PERSIST_INTERVAL = 60_000;  // Persist queue to disk every 60s
+const QUEUE_STATE_FILE = path.join(__dirname, '..', '..', 'data', 'queue-state.json');
+const QUEUE_STATE_MAX_AGE_MS = 24 * 60 * 60 * 1000; // 24h expiry
+const MAX_QUEUE_PERSIST_SIZE = 100_000; // Don't persist if queue > 100K items
 
 function sleep(ms) {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
 
+/**
+ * Persist scanQueue to disk so it survives restarts.
+ * Uses atomicWriteFileSync (write-to-tmp + rename) for crash safety.
+ * Skips if queue is empty or exceeds MAX_QUEUE_PERSIST_SIZE.
+ */
+function persistQueue(scanQueue, state) {
+  try {
+    if (scanQueue.length === 0) {
+      // Empty queue — remove stale file if it exists
+      try { fs.unlinkSync(QUEUE_STATE_FILE); } catch {}
+      return;
+    }
+    if (scanQueue.length > MAX_QUEUE_PERSIST_SIZE) {
+      console.log(`[MONITOR] WARNING: queue too large to persist (${scanQueue.length} > ${MAX_QUEUE_PERSIST_SIZE})`);
+      return;
+    }
+    const payload = JSON.stringify({
+      savedAt: new Date().toISOString(),
+      lastSeq: state.npmLastSeq || null,
+      count: scanQueue.length,
+      items: scanQueue
+    });
+    atomicWriteFileSync(QUEUE_STATE_FILE, payload);
+  } catch (err) {
+    console.error('[MONITOR] Failed to persist queue:', err.message);
+  }
+}
+
+/**
+ * Restore scanQueue from disk on boot. Items are appended to the (empty) scanQueue.
+ * File is deleted after successful restore to prevent double-restore.
+ * Skips if file is missing, corrupt, or older than 24h.
+ */
+function restoreQueue(scanQueue) {
+  try {
+    if (!fs.existsSync(QUEUE_STATE_FILE)) return 0;
+    const raw = fs.readFileSync(QUEUE_STATE_FILE, 'utf8');
+    const data = JSON.parse(raw);
+
+    // Validate structure
+    if (!data || !Array.isArray(data.items) || !data.savedAt) {
+      console.log('[MONITOR] Queue state file invalid — ignoring');
+      try { fs.unlinkSync(QUEUE_STATE_FILE); } catch {}
+      return 0;
+    }
+
+    // Check age — discard if > 24h
+    const ageMs = Date.now() - new Date(data.savedAt).getTime();
+    if (ageMs > QUEUE_STATE_MAX_AGE_MS) {
+      console.log(`[MONITOR] Queue state expired (${Math.round(ageMs / 3600000)}h old) — ignoring`);
+      try { fs.unlinkSync(QUEUE_STATE_FILE); } catch {}
+      return 0;
+    }
+
+    // Restore items
+    const count = data.items.length;
+    if (count === 0) {
+      try { fs.unlinkSync(QUEUE_STATE_FILE); } catch {}
+      return 0;
+    }
+    scanQueue.push(...data.items);
+    console.log(`[MONITOR] Restored ${count} packages from queue state (saved at ${data.savedAt})`);
+
+    // Delete after successful restore
+    try { fs.unlinkSync(QUEUE_STATE_FILE); } catch {}
+    return count;
+  } catch (err) {
+    console.log(`[MONITOR] WARNING: could not restore queue state: ${err.message}`);
+    try { fs.unlinkSync(QUEUE_STATE_FILE); } catch {}
+    return 0;
+  }
+}
+
 function cleanupOrphanTmpDirs() {
   const tmpBase = path.join(os.tmpdir(), 'muaddib-monitor');
   try {
@@ -176,7 +253,14 @@ async function startMonitor(options, stats, dailyAlerts, recentlyScanned, downlo
   console.log(`[MONITOR] Polling every ${POLL_INTERVAL / 1000}s (decoupled from processing). Ctrl+C to stop.\n`);
 
   let running = true;
-  let pollIntervalHandle = null;  // Decoupled poll timer — set after initial poll
+  let pollIntervalHandle = null;   // Decoupled poll timer — set after initial poll
+  let queuePersistHandle = null;   // Queue persistence timer
+
+  // Restore queue from previous run (if file exists and is < 24h old)
+  const restoredCount = restoreQueue(scanQueue);
+  if (restoredCount > 0) {
+    console.log(`[MONITOR] ${restoredCount} packages pre-loaded from previous session`);
+  }
 
   // Graceful shutdown handler (shared by SIGINT and SIGTERM)
   // Daily report is NEVER sent on shutdown — it only fires at 08:00 Paris time.
@@ -188,6 +272,12 @@ async function startMonitor(options, stats, dailyAlerts, recentlyScanned, downlo
       clearInterval(pollIntervalHandle);
       pollIntervalHandle = null;
     }
+    if (queuePersistHandle) {
+      clearInterval(queuePersistHandle);
+      queuePersistHandle = null;
+    }
+    // Persist remaining queue items so they survive the restart
+    persistQueue(scanQueue, state);
     healthcheck.stop();
     // Flush all pending scope groups before exit
     for (const [scope, group] of pendingGrouped) {
@@ -232,6 +322,15 @@ async function startMonitor(options, stats, dailyAlerts, recentlyScanned, downlo
     }
   }, POLL_INTERVAL);
 
+  // ─── Queue persistence ───
+  // Snapshot queue to disk every 60s so items survive restarts/crashes.
+  // Without this, the decoupled poll advances the CouchDB seq but queued
+  // items are lost on restart — they won't be re-polled.
+  queuePersistHandle = setInterval(() => {
+    if (!running) return;
+    persistQueue(scanQueue, state);
+  }, QUEUE_PERSIST_INTERVAL);
+
   // ─── Continuous processing loop ───
   // Consumes scanQueue independently of polling. Workers inside processQueue
   // check scanQueue.length > 0 after each item, so items added by a concurrent
@@ -264,7 +363,13 @@ module.exports = {
   reportStats,
   isDailyReportDue,
   sleep,
+  persistQueue,
+  restoreQueue,
   POLL_INTERVAL,
   PROCESS_LOOP_INTERVAL,
-  QUEUE_WARNING_THRESHOLD
+  QUEUE_WARNING_THRESHOLD,
+  QUEUE_PERSIST_INTERVAL,
+  QUEUE_STATE_FILE,
+  QUEUE_STATE_MAX_AGE_MS,
+  MAX_QUEUE_PERSIST_SIZE
 };

package/src/monitor/queue.js

@@ -105,8 +105,8 @@ const { archiveSuspectTarball } = require('./tarball-archive.js');
 
 // --- Constants ---
 
-const SCAN_CONCURRENCY = Math.max(1, parseInt(process.env.MUADDIB_SCAN_CONCURRENCY, 10) || 5);
-const SCAN_TIMEOUT_MS = 180_000; // 3 minutes per package
+const SCAN_CONCURRENCY = Math.max(1, parseInt(process.env.MUADDIB_SCAN_CONCURRENCY, 10) || 8);
+const SCAN_TIMEOUT_MS = 300_000; // 5 minutes per package (3 sandbox runs × 90s + static scan headroom)
 const STATIC_SCAN_TIMEOUT_MS = 45_000; // 45s for static analysis only
 const LARGE_PACKAGE_SIZE = 10 * 1024 * 1024; // 10MB
 

package/src/sandbox/index.js

@@ -20,7 +20,7 @@ const { parseGvisorLogs, cleanupGvisorLogs } = require('./gvisor-parser.js');
 
 const DOCKER_IMAGE = 'muaddib-sandbox';
 const CONTAINER_TIMEOUT = 120000; // 120 seconds
-const SINGLE_RUN_TIMEOUT = 60000; // 60 seconds per run in multi-run mode
+const SINGLE_RUN_TIMEOUT = 90000; // 90 seconds per run in multi-run mode (gVisor ~30% I/O overhead)
 
 // ── Sandbox concurrency limiter ──
 // Prevents Docker container saturation under load (16 workers × 3 runs = 48 containers).