muaddib-scanner 2.10.46 → 2.10.48

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "muaddib-scanner",
-  "version": "2.10.46",
+  "version": "2.10.48",
   "description": "Supply-chain threat detection & response for npm & PyPI/Python",
   "main": "src/index.js",
   "bin": {

package/src/monitor/daemon.js

@@ -58,6 +58,15 @@ function persistQueue(scanQueue, state) {
  * Skips if file is missing, corrupt, or older than 24h.
  */
 function restoreQueue(scanQueue) {
+  // Cleanup orphan .tmp from previous crash / disk-full (ENOSPC)
+  const tmpFile = QUEUE_STATE_FILE + '.tmp';
+  try {
+    if (fs.existsSync(tmpFile)) {
+      console.log(`[MONITOR] Cleaning up orphan ${path.basename(tmpFile)}`);
+      fs.unlinkSync(tmpFile);
+    }
+  } catch { /* best-effort */ }
+
   try {
     if (!fs.existsSync(QUEUE_STATE_FILE)) return 0;
     const raw = fs.readFileSync(QUEUE_STATE_FILE, 'utf8');
@@ -134,6 +143,94 @@ function cleanupOrphanContainers() {
   }
 }
 
+/**
+ * Clean up orphan gVisor runtime directories in /tmp/runsc.
+ * runsc creates per-container state dirs that are NOT cleaned up when gVisor or
+ * Docker crashes. In production this reached 61GB and filled the disk (ENOSPC),
+ * cascading into 0-byte .tmp files and total persistence failure.
+ * Removes directories older than maxAgeMs (default: 1h).
+ */
+function cleanupRunscOrphans(maxAgeMs = 3600_000) {
+  const runscDir = process.env.MUADDIB_GVISOR_LOG_DIR || '/tmp/runsc';
+  try {
+    if (!fs.existsSync(runscDir)) return 0;
+    const entries = fs.readdirSync(runscDir);
+    const now = Date.now();
+    let cleaned = 0;
+    for (const entry of entries) {
+      const fullPath = path.join(runscDir, entry);
+      try {
+        const stat = fs.statSync(fullPath);
+        if (now - stat.mtimeMs > maxAgeMs) {
+          fs.rmSync(fullPath, { recursive: true, force: true });
+          cleaned++;
+        }
+      } catch { /* skip unreadable entries */ }
+    }
+    if (cleaned > 0) {
+      console.log(`[MONITOR] Cleaned up ${cleaned} orphan runsc dir(s) in ${runscDir}`);
+    }
+    return cleaned;
+  } catch {
+    return 0;
+  }
+}
+
+/**
+ * Check disk usage at boot. Warns if root partition > 90% full and logs
+ * the largest consumers in /tmp/ and data/ to aid diagnosis.
+ * Uses df + du — Linux-only, silently skips on other platforms.
+ */
+function checkDiskSpace() {
+  try {
+    // df --output=pcent / → "Use%\n 42%\n"
+    const dfOutput = execFileSync('df', ['--output=pcent', '/'], {
+      encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe'], timeout: 5000
+    });
+    const match = dfOutput.match(/(\d+)%/);
+    if (!match) return;
+    const usagePercent = parseInt(match[1], 10);
+    if (usagePercent < 90) return;
+
+    console.warn(`[MONITOR] WARNING: disk usage at ${usagePercent}% — persistence may fail (ENOSPC)`);
+
+    // Top consumers in /tmp/
+    try {
+      const tmpDu = execFileSync('du', ['-sh', '--max-depth=1', '/tmp/'], {
+        encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe'], timeout: 10000
+      });
+      const lines = tmpDu.trim().split('\n')
+        .map(l => { const m = l.match(/^([\d.]+[KMGT]?)\s+(.+)/); return m ? { size: m[1], path: m[2] } : null; })
+        .filter(Boolean)
+        .sort((a, b) => b.size.localeCompare(a.size))
+        .slice(0, 5);
+      if (lines.length > 0) {
+        console.warn('[MONITOR]   Top /tmp/ consumers:');
+        for (const l of lines) console.warn(`[MONITOR]     ${l.size}\t${l.path}`);
+      }
+    } catch { /* du failed */ }
+
+    // Top consumers in data/
+    const dataDir = path.join(__dirname, '..', '..', 'data');
+    try {
+      const dataDu = execFileSync('du', ['-sh', '--max-depth=1', dataDir], {
+        encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe'], timeout: 10000
+      });
+      const lines = dataDu.trim().split('\n')
+        .map(l => { const m = l.match(/^([\d.]+[KMGT]?)\s+(.+)/); return m ? { size: m[1], path: m[2] } : null; })
+        .filter(Boolean)
+        .sort((a, b) => b.size.localeCompare(a.size))
+        .slice(0, 5);
+      if (lines.length > 0) {
+        console.warn('[MONITOR]   Top data/ consumers:');
+        for (const l of lines) console.warn(`[MONITOR]     ${l.size}\t${l.path}`);
+      }
+    } catch { /* du failed */ }
+  } catch {
+    // df not available (non-Linux) — skip silently
+  }
+}
+
 function reportStats(stats) {
   const avg = stats.scanned > 0 ? (stats.totalTimeMs / stats.scanned / 1000).toFixed(1) : '0.0';
   const { t1, t1a, t1b, t2, t3 } = stats.suspectByTier;
@@ -153,7 +250,7 @@ function reportStats(stats) {
   if (stats.sandboxDeferred || stats.deferredProcessed) {
     const { getDeferredQueueStats } = require('./deferred-sandbox.js');
     const dq = getDeferredQueueStats();
-    console.log(`[MONITOR]   Deferred sandbox: ${stats.sandboxDeferred || 0} enqueued, ${stats.deferredProcessed || 0} processed, ${stats.deferredExpired || 0} expired, ${dq.size} pending`);
+    console.log(`[MONITOR]   Deferred sandbox: ${stats.sandboxDeferred || 0} enqueued, ${stats.deferredProcessed || 0} processed, ${stats.deferredExpired || 0} expired, ${stats.deferredSkipped || 0} skipped, ${dq.size} pending`);
   }
   stats.lastReportTime = Date.now();
 }
@@ -171,10 +268,14 @@ async function startMonitor(options, stats, dailyAlerts, recentlyScanned, downlo
     setVerboseMode(true);
   }
 
+  // Disk space check — early warning before ENOSPC cascading failure
+  checkDiskSpace();
   // Cleanup temp dirs from previous runs (SIGTERM/crash may leave orphans)
   cleanupOrphanTmpDirs();
   // Kill orphan sandbox containers from previous crash (npm-audit-* prefix)
   cleanupOrphanContainers();
+  // Clean up stale gVisor runtime dirs (runsc leak — caused 61GB disk fill in prod)
+  cleanupRunscOrphans();
   // Layer 3: Purge expired cached tarballs on startup
   purgeTarballCache();
 
@@ -364,10 +465,11 @@ async function startMonitor(options, stats, dailyAlerts, recentlyScanned, downlo
       await processQueue(scanQueue, stats, dailyAlerts, recentlyScanned, downloadsCache, sandboxAvailableRef.value);
     }
 
-    // Hourly stats report + cache purge
+    // Hourly stats report + cache purge + runsc cleanup
     if (Date.now() - stats.lastReportTime >= 3600_000) {
       reportStats(stats);
       purgeTarballCache();
+      cleanupRunscOrphans();
     }
 
     // Daily webhook report at 08:00 Paris time
@@ -384,6 +486,8 @@ module.exports = {
   startMonitor,
   cleanupOrphanTmpDirs,
   cleanupOrphanContainers,
+  cleanupRunscOrphans,
+  checkDiskSpace,
   reportStats,
   isDailyReportDue,
   sleep,

package/src/monitor/deferred-sandbox.js

@@ -6,11 +6,15 @@
  * Items are sorted by riskScore DESC (highest-risk first) to defend
  * against queue-poisoning attacks.
  *
- * The worker reserves 1 sandbox slot for T1a (never uses the last slot).
+ * The worker owns a dedicated sandbox slot (_deferredSlotBusy) that is
+ * completely independent from the shared semaphore used by T1a/T1b/T2.
+ * This guarantees the deferred worker can always process, regardless of
+ * how many main-path sandboxes are running. The VPS supports N+1
+ * concurrent gVisor containers (3 main + 1 deferred).
  */
 const fs = require('fs');
 const path = require('path');
-const { runSandbox, getSandboxSemaphore, SANDBOX_CONCURRENCY_MAX } = require('../sandbox/index.js');
+const { runSandbox } = require('../sandbox/index.js');
 const { isCanaryEnabled } = require('./classify.js');
 const { getWebhookUrl, alertedPackageRules, persistAlert, buildAlertData } = require('./webhook.js');
 const { sendWebhook } = require('../webhook.js');
@@ -28,6 +32,7 @@ const _deferredQueue = [];
 const _deferredSeen = new Set(); // name@version dedup
 let _workerHandle = null;
 let _stats = null; // reference to shared stats object
+let _deferredSlotBusy = false;   // Dedicated slot: true while deferred sandbox is running
 
 // ── Queue management ──
 
@@ -126,10 +131,10 @@ async function processDeferredItem(stats) {
 
   if (_deferredQueue.length === 0) return null;
 
-  // 2. Yield check: reserve 1 slot for T1a
-  const sem = getSandboxSemaphore();
-  if (sem.active >= SANDBOX_CONCURRENCY_MAX - 1) {
-    return null; // All slots busy or only 1 free — keep it for T1a
+  // 2. Dedicated slot check — completely independent from main semaphore
+  if (_deferredSlotBusy) {
+    if (stats) stats.deferredSkipped = (stats.deferredSkipped || 0) + 1;
+    return null;
   }
 
   // 3. Pick highest-score item
@@ -139,11 +144,12 @@ async function processDeferredItem(stats) {
 
   console.log(`[DEFERRED] PROCESSING: ${key} (tier=${item.tier === 2 ? 'T2' : 'T1b'}, score=${item.riskScore}, retries=${item.retries})`);
 
-  // 4. Run sandbox
+  // 4. Run sandbox on dedicated slot (bypasses shared semaphore)
+  _deferredSlotBusy = true;
   let sandboxResult;
   try {
     const canary = isCanaryEnabled();
-    sandboxResult = await runSandbox(item.name, { canary });
+    sandboxResult = await runSandbox(item.name, { canary, skipSemaphore: true });
     console.log(`[DEFERRED] SANDBOX COMPLETE: ${key} -> score=${sandboxResult.score}, severity=${sandboxResult.severity}`);
   } catch (err) {
     console.error(`[DEFERRED] SANDBOX ERROR: ${key} — ${err.message}`);
@@ -158,6 +164,8 @@ async function processDeferredItem(stats) {
       console.log(`[DEFERRED] RE-ENQUEUED: ${key} for retry (attempt ${item.retries + 1}/${DEFERRED_MAX_RETRIES})`);
     }
     return null;
+  } finally {
+    _deferredSlotBusy = false;
   }
 
   // 5. Follow-up webhook if sandbox found something
@@ -311,6 +319,16 @@ function persistDeferredQueue() {
 }
 
 function restoreDeferredQueue() {
+  // Cleanup orphan .tmp from previous crash / disk-full (ENOSPC)
+  const tmpFile = DEFERRED_STATE_FILE + '.tmp';
+  try {
+    if (fs.existsSync(tmpFile)) {
+      const stat = fs.statSync(tmpFile);
+      console.log(`[DEFERRED] Cleaning up orphan ${path.basename(tmpFile)} (${stat.size} bytes)`);
+      fs.unlinkSync(tmpFile);
+    }
+  } catch { /* best-effort */ }
+
   try {
     if (!fs.existsSync(DEFERRED_STATE_FILE)) return 0;
     const raw = fs.readFileSync(DEFERRED_STATE_FILE, 'utf8');
@@ -365,9 +383,14 @@ function _resetDeferredQueue() {
   _deferredQueue.length = 0;
   _deferredSeen.clear();
   _stats = null;
+  _deferredSlotBusy = false;
   stopDeferredWorker();
 }
 
+function isDeferredSlotBusy() {
+  return _deferredSlotBusy;
+}
+
 module.exports = {
   enqueueDeferred,
   getDeferredQueue,
@@ -379,6 +402,7 @@ module.exports = {
   restoreDeferredQueue,
   buildDeferredFollowUpEmbed,
   pruneExpired,
+  isDeferredSlotBusy,
   _resetDeferredQueue,
   DEFERRED_QUEUE_MAX,
   DEFERRED_TTL_MS,

package/src/monitor/state.js

@@ -116,6 +116,10 @@ function atomicWriteFileSync(filePath, data) {
       console.warn(`[MONITOR] Cannot create directory ${dir} (${err.code}) — skipping write to ${path.basename(filePath)}`);
       return;
     }
+    if (err.code === 'ENOSPC') {
+      console.warn(`[MONITOR] WARNING: disk full (ENOSPC) — cannot create directory ${dir}. Free space immediately.`);
+      return;
+    }
     throw err;
   }
   const tmpFile = filePath + '.tmp';
@@ -125,7 +129,11 @@ function atomicWriteFileSync(filePath, data) {
   } catch (err) {
     if (err.code === 'EROFS' || err.code === 'EACCES' || err.code === 'EPERM') {
       console.warn(`[MONITOR] Cannot write ${path.basename(filePath)} (${err.code}) — skipping`);
-      // Clean up .tmp if it was partially written
+      try { fs.unlinkSync(tmpFile); } catch (_) { /* ignore */ }
+      return;
+    }
+    if (err.code === 'ENOSPC') {
+      console.warn(`[MONITOR] WARNING: disk full (ENOSPC) — cannot write ${path.basename(filePath)}. Free space in /tmp and data/ immediately.`);
       try { fs.unlinkSync(tmpFile); } catch (_) { /* ignore */ }
       return;
     }
@@ -682,6 +690,7 @@ function loadDailyStats(stats, dailyAlerts) {
       stats.mlFiltered = data.mlFiltered || 0;
       stats.llmAnalyzed = data.llmAnalyzed || 0;
       stats.llmSuppressed = data.llmSuppressed || 0;
+      stats.changesStreamPackages = data.changesStreamPackages || 0;
       if (Array.isArray(data.dailyAlerts)) {
         dailyAlerts.length = 0;
         dailyAlerts.push(...data.dailyAlerts);
@@ -708,6 +717,7 @@ function saveDailyStats(stats, dailyAlerts) {
       mlFiltered: stats.mlFiltered,
       llmAnalyzed: stats.llmAnalyzed || 0,
       llmSuppressed: stats.llmSuppressed || 0,
+      changesStreamPackages: stats.changesStreamPackages || 0,
       dailyAlerts: dailyAlerts.slice()
     };
     atomicWriteFileSync(DAILY_STATS_FILE, JSON.stringify(data, null, 2));

package/src/sandbox/index.js

@@ -23,7 +23,8 @@ const CONTAINER_TIMEOUT = 120000; // 120 seconds
 const SINGLE_RUN_TIMEOUT = 90000; // 90 seconds per run in multi-run mode (gVisor ~30% I/O overhead)
 
 // ── Sandbox concurrency limiter ──
-// Prevents Docker container saturation under load (16 workers × 3 runs = 48 containers).
+// Prevents Docker container saturation under load (main-path T1a/T1b/T2).
+// The deferred worker manages its own dedicated slot outside this semaphore.
 // Pattern: same semaphore as src/shared/http-limiter.js.
 const SANDBOX_CONCURRENCY_MAX = Math.max(1, parseInt(process.env.MUADDIB_SANDBOX_CONCURRENCY, 10) || 3);
 
@@ -565,16 +566,21 @@ async function runSandbox(packageName, options = {}) {
 
   const mode = strict ? 'strict' : 'permissive';
 
-  // Acquire sandbox slot — blocks if SANDBOX_CONCURRENCY_MAX containers already running
-  const queueLen = _sandboxSemaphore.queue.length;
-  if (queueLen > 0) {
-    console.log(`[SANDBOX] Waiting for sandbox slot (${_sandboxSemaphore.active}/${SANDBOX_CONCURRENCY_MAX} active, ${queueLen} queued)...`);
+  // Acquire sandbox slot — blocks if SANDBOX_CONCURRENCY_MAX containers already running.
+  // skipSemaphore: deferred worker manages its own dedicated slot outside this semaphore.
+  const skipSem = options.skipSemaphore === true;
+  if (!skipSem) {
+    const queueLen = _sandboxSemaphore.queue.length;
+    if (queueLen > 0) {
+      console.log(`[SANDBOX] Waiting for sandbox slot (${_sandboxSemaphore.active}/${SANDBOX_CONCURRENCY_MAX} active, ${queueLen} queued)...`);
+    }
+    await acquireSandboxSlot();
   }
-  await acquireSandboxSlot();
 
   try {
     const runtimeLabel = useGvisor ? 'gvisor' : 'docker';
-    console.log(`[SANDBOX] Analyzing "${displayName}" in isolated container (mode: ${mode}, runtime: ${runtimeLabel}${canaryEnabled ? ', canary: on' : ''}${local ? ', local' : ''}, runs: ${TIME_OFFSETS.length}, slots: ${_sandboxSemaphore.active}/${SANDBOX_CONCURRENCY_MAX})...`);
+    const slotInfo = skipSem ? 'deferred-slot' : `${_sandboxSemaphore.active}/${SANDBOX_CONCURRENCY_MAX}`;
+    console.log(`[SANDBOX] Analyzing "${displayName}" in isolated container (mode: ${mode}, runtime: ${runtimeLabel}${canaryEnabled ? ', canary: on' : ''}${local ? ', local' : ''}, runs: ${TIME_OFFSETS.length}, slots: ${slotInfo})...`);
 
     const allRuns = [];
     let bestResult = cleanResult;
@@ -639,7 +645,7 @@ async function runSandbox(packageName, options = {}) {
     displayResults(bestResult);
     return bestResult;
   } finally {
-    releaseSandboxSlot();
+    if (!skipSem) releaseSandboxSlot();
   }
 }