muaddib-scanner 2.10.45 → 2.10.47

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "muaddib-scanner",
-  "version": "2.10.45",
+  "version": "2.10.47",
   "description": "Supply-chain threat detection & response for npm & PyPI/Python",
   "main": "src/index.js",
   "bin": {

package/src/monitor/daemon.js

@@ -10,6 +10,7 @@ const { pendingGrouped, flushScopeGroup, sendDailyReport, DAILY_REPORT_HOUR } =
 const { poll } = require('./ingestion.js');
 const { processQueue, SCAN_CONCURRENCY } = require('./queue.js');
 const { startHealthcheck } = require('./healthcheck.js');
+const { startDeferredWorker, stopDeferredWorker, persistDeferredQueue, restoreDeferredQueue } = require('./deferred-sandbox.js');
 
 const POLL_INTERVAL = 60_000;
 const PROCESS_LOOP_INTERVAL = 2_000;    // Queue check interval when empty
@@ -57,6 +58,15 @@ function persistQueue(scanQueue, state) {
  * Skips if file is missing, corrupt, or older than 24h.
  */
 function restoreQueue(scanQueue) {
+  // Cleanup orphan .tmp from previous crash / disk-full (ENOSPC)
+  const tmpFile = QUEUE_STATE_FILE + '.tmp';
+  try {
+    if (fs.existsSync(tmpFile)) {
+      console.log(`[MONITOR] Cleaning up orphan ${path.basename(tmpFile)}`);
+      fs.unlinkSync(tmpFile);
+    }
+  } catch { /* best-effort */ }
+
   try {
     if (!fs.existsSync(QUEUE_STATE_FILE)) return 0;
     const raw = fs.readFileSync(QUEUE_STATE_FILE, 'utf8');
@@ -133,6 +143,94 @@ function cleanupOrphanContainers() {
   }
 }
 
+/**
+ * Clean up orphan gVisor runtime directories in /tmp/runsc.
+ * runsc creates per-container state dirs that are NOT cleaned up when gVisor or
+ * Docker crashes. In production this reached 61GB and filled the disk (ENOSPC),
+ * cascading into 0-byte .tmp files and total persistence failure.
+ * Removes directories older than maxAgeMs (default: 1h).
+ */
+function cleanupRunscOrphans(maxAgeMs = 3600_000) {
+  const runscDir = process.env.MUADDIB_GVISOR_LOG_DIR || '/tmp/runsc';
+  try {
+    if (!fs.existsSync(runscDir)) return 0;
+    const entries = fs.readdirSync(runscDir);
+    const now = Date.now();
+    let cleaned = 0;
+    for (const entry of entries) {
+      const fullPath = path.join(runscDir, entry);
+      try {
+        const stat = fs.statSync(fullPath);
+        if (now - stat.mtimeMs > maxAgeMs) {
+          fs.rmSync(fullPath, { recursive: true, force: true });
+          cleaned++;
+        }
+      } catch { /* skip unreadable entries */ }
+    }
+    if (cleaned > 0) {
+      console.log(`[MONITOR] Cleaned up ${cleaned} orphan runsc dir(s) in ${runscDir}`);
+    }
+    return cleaned;
+  } catch {
+    return 0;
+  }
+}
+
+/**
+ * Check disk usage at boot. Warns if root partition > 90% full and logs
+ * the largest consumers in /tmp/ and data/ to aid diagnosis.
+ * Uses df + du — Linux-only, silently skips on other platforms.
+ */
+function checkDiskSpace() {
+  try {
+    // df --output=pcent / → "Use%\n 42%\n"
+    const dfOutput = execFileSync('df', ['--output=pcent', '/'], {
+      encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe'], timeout: 5000
+    });
+    const match = dfOutput.match(/(\d+)%/);
+    if (!match) return;
+    const usagePercent = parseInt(match[1], 10);
+    if (usagePercent < 90) return;
+
+    console.warn(`[MONITOR] WARNING: disk usage at ${usagePercent}% — persistence may fail (ENOSPC)`);
+
+    // Top consumers in /tmp/
+    try {
+      const tmpDu = execFileSync('du', ['-sh', '--max-depth=1', '/tmp/'], {
+        encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe'], timeout: 10000
+      });
+      const lines = tmpDu.trim().split('\n')
+        .map(l => { const m = l.match(/^([\d.]+[KMGT]?)\s+(.+)/); return m ? { size: m[1], path: m[2] } : null; })
+        .filter(Boolean)
+        .sort((a, b) => b.size.localeCompare(a.size))
+        .slice(0, 5);
+      if (lines.length > 0) {
+        console.warn('[MONITOR]   Top /tmp/ consumers:');
+        for (const l of lines) console.warn(`[MONITOR]     ${l.size}\t${l.path}`);
+      }
+    } catch { /* du failed */ }
+
+    // Top consumers in data/
+    const dataDir = path.join(__dirname, '..', '..', 'data');
+    try {
+      const dataDu = execFileSync('du', ['-sh', '--max-depth=1', dataDir], {
+        encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe'], timeout: 10000
+      });
+      const lines = dataDu.trim().split('\n')
+        .map(l => { const m = l.match(/^([\d.]+[KMGT]?)\s+(.+)/); return m ? { size: m[1], path: m[2] } : null; })
+        .filter(Boolean)
+        .sort((a, b) => b.size.localeCompare(a.size))
+        .slice(0, 5);
+      if (lines.length > 0) {
+        console.warn('[MONITOR]   Top data/ consumers:');
+        for (const l of lines) console.warn(`[MONITOR]     ${l.size}\t${l.path}`);
+      }
+    } catch { /* du failed */ }
+  } catch {
+    // df not available (non-Linux) — skip silently
+  }
+}
+
 function reportStats(stats) {
   const avg = stats.scanned > 0 ? (stats.totalTimeMs / stats.scanned / 1000).toFixed(1) : '0.0';
   const { t1, t1a, t1b, t2, t3 } = stats.suspectByTier;
@@ -149,6 +247,11 @@ function reportStats(stats) {
   if (stats.tarballCacheHits) {
     console.log(`[MONITOR]   Tarball cache hits: ${stats.tarballCacheHits}`);
   }
+  if (stats.sandboxDeferred || stats.deferredProcessed) {
+    const { getDeferredQueueStats } = require('./deferred-sandbox.js');
+    const dq = getDeferredQueueStats();
+    console.log(`[MONITOR]   Deferred sandbox: ${stats.sandboxDeferred || 0} enqueued, ${stats.deferredProcessed || 0} processed, ${stats.deferredExpired || 0} expired, ${stats.deferredSkipped || 0} skipped, ${dq.size} pending`);
+  }
   stats.lastReportTime = Date.now();
 }
 
@@ -165,10 +268,14 @@ async function startMonitor(options, stats, dailyAlerts, recentlyScanned, downlo
     setVerboseMode(true);
   }
 
+  // Disk space check — early warning before ENOSPC cascading failure
+  checkDiskSpace();
   // Cleanup temp dirs from previous runs (SIGTERM/crash may leave orphans)
   cleanupOrphanTmpDirs();
   // Kill orphan sandbox containers from previous crash (npm-audit-* prefix)
   cleanupOrphanContainers();
+  // Clean up stale gVisor runtime dirs (runsc leak — caused 61GB disk fill in prod)
+  cleanupRunscOrphans();
   // Layer 3: Purge expired cached tarballs on startup
   purgeTarballCache();
 
@@ -262,6 +369,12 @@ async function startMonitor(options, stats, dailyAlerts, recentlyScanned, downlo
     console.log(`[MONITOR] ${restoredCount} packages pre-loaded from previous session`);
   }
 
+  // Restore deferred sandbox queue from previous run
+  const deferredRestored = restoreDeferredQueue();
+  if (deferredRestored > 0) {
+    console.log(`[MONITOR] ${deferredRestored} deferred sandbox items restored from previous session`);
+  }
+
   // Graceful shutdown handler (shared by SIGINT and SIGTERM)
   // Daily report is NEVER sent on shutdown — it only fires at 08:00 Paris time.
   // Counters are persisted to disk so they survive the restart.
@@ -278,6 +391,9 @@ async function startMonitor(options, stats, dailyAlerts, recentlyScanned, downlo
     }
     // Persist remaining queue items so they survive the restart
     persistQueue(scanQueue, state);
+    // Stop deferred sandbox worker and persist its queue
+    stopDeferredWorker();
+    persistDeferredQueue();
     healthcheck.stop();
     // Flush all pending scope groups before exit
     for (const [scope, group] of pendingGrouped) {
@@ -329,8 +445,17 @@ async function startMonitor(options, stats, dailyAlerts, recentlyScanned, downlo
   queuePersistHandle = setInterval(() => {
     if (!running) return;
     persistQueue(scanQueue, state);
+    persistDeferredQueue(); // Piggyback: persist deferred sandbox queue on same interval
   }, QUEUE_PERSIST_INTERVAL);
 
+  // ─── Deferred sandbox worker ───
+  // Retries T1b/T2 packages that were skipped when sandbox slots were full.
+  // Runs every 30s, processes at most 1 item per tick, yields to T1a.
+  if (isSandboxEnabled() && sandboxAvailableRef.value) {
+    startDeferredWorker(stats);
+    console.log('[MONITOR] Deferred sandbox worker started (30s interval, T1a-safe)');
+  }
+
   // ─── Continuous processing loop ───
   // Consumes scanQueue independently of polling. Workers inside processQueue
   // check scanQueue.length > 0 after each item, so items added by a concurrent
@@ -340,10 +465,11 @@ async function startMonitor(options, stats, dailyAlerts, recentlyScanned, downlo
       await processQueue(scanQueue, stats, dailyAlerts, recentlyScanned, downloadsCache, sandboxAvailableRef.value);
     }
 
-    // Hourly stats report + cache purge
+    // Hourly stats report + cache purge + runsc cleanup
     if (Date.now() - stats.lastReportTime >= 3600_000) {
       reportStats(stats);
       purgeTarballCache();
+      cleanupRunscOrphans();
     }
 
     // Daily webhook report at 08:00 Paris time
@@ -360,6 +486,8 @@ module.exports = {
   startMonitor,
   cleanupOrphanTmpDirs,
   cleanupOrphanContainers,
+  cleanupRunscOrphans,
+  checkDiskSpace,
   reportStats,
   isDailyReportDue,
   sleep,

package/src/monitor/deferred-sandbox.js

@@ -0,0 +1,417 @@
+/**
+ * Deferred Sandbox Queue
+ *
+ * When T1b/T2 packages are skipped due to sandbox slot pressure,
+ * they are enqueued here and retried when slots free up.
+ * Items are sorted by riskScore DESC (highest-risk first) to defend
+ * against queue-poisoning attacks.
+ *
+ * The worker reserves 1 sandbox slot for T1a (never uses the last slot).
+ */
+const fs = require('fs');
+const path = require('path');
+const { runSandbox, getSandboxSemaphore, SANDBOX_CONCURRENCY_MAX } = require('../sandbox/index.js');
+const { isCanaryEnabled } = require('./classify.js');
+const { getWebhookUrl, alertedPackageRules, persistAlert, buildAlertData } = require('./webhook.js');
+const { sendWebhook } = require('../webhook.js');
+const { atomicWriteFileSync } = require('./state.js');
+
+// ── Constants ──
+const DEFERRED_QUEUE_MAX = 500;
+const DEFERRED_TTL_MS = 24 * 60 * 60 * 1000; // 24h
+const DEFERRED_MAX_RETRIES = 2;
+const DEFERRED_WORKER_INTERVAL_MS = 30_000; // 30s
+const DEFERRED_STATE_FILE = path.join(__dirname, '..', '..', 'data', 'deferred-queue.json');
+const SKIP_LOG_INTERVAL = 10;        // Log every N skipped ticks (throttle)
+const ANTI_STARVATION_TICKS = 20;    // Force processing after N consecutive skips (~10min)
+
+// ── Mutable state ──
+const _deferredQueue = [];
+const _deferredSeen = new Set(); // name@version dedup
+let _workerHandle = null;
+let _stats = null; // reference to shared stats object
+let _consecutiveSkips = 0;       // Tracks consecutive yield-skips for anti-starvation
+
+// ── Queue management ──
+
+/**
+ * Enqueue a T1b/T2 package for deferred sandbox analysis.
+ * Items are sorted by riskScore DESC (highest risk first).
+ * When the queue is full, the lowest-score item is evicted if the new item scores higher.
+ *
+ * @param {object} item - Package to defer
+ * @returns {boolean} true if enqueued, false if rejected
+ */
+function enqueueDeferred(item) {
+  // Guard: only T1b and T2 are allowed
+  if (item.tier !== '1b' && item.tier !== 2) {
+    console.error(`[DEFERRED] REJECTED: ${item.name}@${item.version} — tier ${item.tier} not eligible`);
+    return false;
+  }
+
+  const key = `${item.name}@${item.version}`;
+
+  // Dedup
+  if (_deferredSeen.has(key)) {
+    console.log(`[DEFERRED] DEDUP: ${key} already in deferred queue`);
+    return false;
+  }
+
+  // Queue full — evict lowest or reject
+  if (_deferredQueue.length >= DEFERRED_QUEUE_MAX) {
+    const lowest = _deferredQueue[_deferredQueue.length - 1];
+    if (item.riskScore > lowest.riskScore) {
+      const evictKey = `${lowest.name}@${lowest.version}`;
+      _deferredQueue.pop();
+      _deferredSeen.delete(evictKey);
+      console.log(`[DEFERRED] EVICTED: ${evictKey} (score=${lowest.riskScore}) to make room for ${key} (score=${item.riskScore})`);
+    } else {
+      console.log(`[DEFERRED] QUEUE FULL: ${key} (score=${item.riskScore}) rejected — all ${DEFERRED_QUEUE_MAX} items have higher scores`);
+      return false;
+    }
+  }
+
+  _deferredQueue.push(item);
+  _deferredSeen.add(key);
+  // Sort by riskScore DESC (highest first)
+  _deferredQueue.sort((a, b) => b.riskScore - a.riskScore);
+  console.log(`[DEFERRED] ENQUEUED: ${key} (tier=${item.tier === 2 ? 'T2' : 'T1b'}, score=${item.riskScore}, queue=${_deferredQueue.length})`);
+  return true;
+}
+
+function getDeferredQueue() {
+  return _deferredQueue;
+}
+
+function getDeferredQueueStats() {
+  const tierBreakdown = { t1b: 0, t2: 0 };
+  for (const item of _deferredQueue) {
+    if (item.tier === '1b') tierBreakdown.t1b++;
+    else if (item.tier === 2) tierBreakdown.t2++;
+  }
+  return {
+    size: _deferredQueue.length,
+    oldest: _deferredQueue.length > 0
+      ? _deferredQueue[_deferredQueue.length - 1].enqueuedAt
+      : null,
+    tierBreakdown
+  };
+}
+
+// ── TTL pruning ──
+
+function pruneExpired(stats) {
+  const now = Date.now();
+  let pruned = 0;
+  for (let i = _deferredQueue.length - 1; i >= 0; i--) {
+    if (now - _deferredQueue[i].enqueuedAt > DEFERRED_TTL_MS) {
+      const item = _deferredQueue[i];
+      const key = `${item.name}@${item.version}`;
+      _deferredQueue.splice(i, 1);
+      _deferredSeen.delete(key);
+      if (stats) stats.deferredExpired = (stats.deferredExpired || 0) + 1;
+      console.log(`[DEFERRED] EXPIRED: ${key} (age=${((now - item.enqueuedAt) / 3600000).toFixed(1)}h)`);
+      pruned++;
+    }
+  }
+  return pruned;
+}
+
+// ── Worker ──
+
+/**
+ * Process one deferred item. Exported for testing.
+ * @returns {object|null} sandboxResult or null if nothing processed
+ */
+async function processDeferredItem(stats) {
+  // 1. Prune expired items
+  pruneExpired(stats);
+
+  if (_deferredQueue.length === 0) return null;
+
+  // 2. Yield check: reserve 1 slot for T1a (unless anti-starvation kicks in)
+  const sem = getSandboxSemaphore();
+  const slotsFullForDeferred = sem.active >= SANDBOX_CONCURRENCY_MAX - 1;
+  const forceAntiStarvation = _consecutiveSkips >= ANTI_STARVATION_TICKS && sem.active < SANDBOX_CONCURRENCY_MAX;
+
+  if (slotsFullForDeferred && !forceAntiStarvation) {
+    _consecutiveSkips++;
+    if (stats) stats.deferredSkipped = (stats.deferredSkipped || 0) + 1;
+    if (_consecutiveSkips % SKIP_LOG_INTERVAL === 0) {
+      console.log(`[DEFERRED] YIELD: ${_consecutiveSkips} consecutive skips (slots=${sem.active}/${SANDBOX_CONCURRENCY_MAX}, pending=${_deferredQueue.length})`);
+    }
+    return null;
+  }
+
+  if (forceAntiStarvation) {
+    console.log(`[DEFERRED] ANTI-STARVATION: forcing processing after ${_consecutiveSkips} skips (slots=${sem.active}/${SANDBOX_CONCURRENCY_MAX}, pending=${_deferredQueue.length})`);
+  }
+  _consecutiveSkips = 0;
+
+  // 3. Pick highest-score item
+  const item = _deferredQueue.shift();
+  const key = `${item.name}@${item.version}`;
+  _deferredSeen.delete(key);
+
+  console.log(`[DEFERRED] PROCESSING: ${key} (tier=${item.tier === 2 ? 'T2' : 'T1b'}, score=${item.riskScore}, retries=${item.retries})`);
+
+  // 4. Run sandbox
+  let sandboxResult;
+  try {
+    const canary = isCanaryEnabled();
+    sandboxResult = await runSandbox(item.name, { canary });
+    console.log(`[DEFERRED] SANDBOX COMPLETE: ${key} -> score=${sandboxResult.score}, severity=${sandboxResult.severity}`);
+  } catch (err) {
+    console.error(`[DEFERRED] SANDBOX ERROR: ${key} — ${err.message}`);
+    item.retries = (item.retries || 0) + 1;
+    if (item.retries >= DEFERRED_MAX_RETRIES) {
+      console.log(`[DEFERRED] DROPPED: ${key} after ${item.retries} failed attempts`);
+    } else {
+      // Re-enqueue for retry
+      _deferredQueue.push(item);
+      _deferredSeen.add(key);
+      _deferredQueue.sort((a, b) => b.riskScore - a.riskScore);
+      console.log(`[DEFERRED] RE-ENQUEUED: ${key} for retry (attempt ${item.retries + 1}/${DEFERRED_MAX_RETRIES})`);
+    }
+    return null;
+  }
+
+  // 5. Follow-up webhook if sandbox found something
+  if (stats) stats.deferredProcessed = (stats.deferredProcessed || 0) + 1;
+
+  if (sandboxResult && sandboxResult.score > 0) {
+    const deferredDedupKey = 'deferred_sandbox';
+    const previousRules = alertedPackageRules.get(item.name);
+    const alreadySentFollowUp = previousRules && previousRules.has(deferredDedupKey);
+
+    if (!alreadySentFollowUp) {
+      const url = getWebhookUrl();
+      if (url) {
+        try {
+          const embed = buildDeferredFollowUpEmbed(
+            item.name, item.version, item.ecosystem,
+            sandboxResult,
+            item.riskScore
+          );
+          await sendWebhook(url, embed, { rawPayload: true });
+          console.log(`[DEFERRED] FOLLOW-UP WEBHOOK: ${key} (sandbox score=${sandboxResult.score})`);
+
+          // Track in dedup map
+          if (previousRules) {
+            previousRules.add(deferredDedupKey);
+          } else {
+            alertedPackageRules.set(item.name, new Set([deferredDedupKey]));
+          }
+        } catch (webhookErr) {
+          console.error(`[DEFERRED] FOLLOW-UP WEBHOOK FAILED: ${key} — ${webhookErr.message}`);
+        }
+      }
+
+      // Persist updated alert with sandbox data
+      try {
+        const alertData = buildAlertData(
+          item.name, item.version, item.ecosystem,
+          item.staticResult, sandboxResult
+        );
+        persistAlert(item.name, item.version, item.ecosystem, alertData);
+        console.log(`[DEFERRED] ALERT PERSISTED: ${key} (with sandbox data)`);
+      } catch (persistErr) {
+        console.error(`[DEFERRED] ALERT PERSIST FAILED: ${key} — ${persistErr.message}`);
+      }
+    } else {
+      console.log(`[DEFERRED] DEDUP: follow-up already sent for ${item.name}`);
+    }
+  } else {
+    console.log(`[DEFERRED] CLEAN: ${key} (sandbox score=0, static score=${item.riskScore})`);
+  }
+
+  return sandboxResult;
+}
+
+/**
+ * Build Discord embed for deferred sandbox follow-up.
+ */
+function buildDeferredFollowUpEmbed(name, version, ecosystem, sandboxResult, staticScore) {
+  const npmLink = ecosystem === 'npm'
+    ? `https://www.npmjs.com/package/${encodeURIComponent(name)}`
+    : `https://pypi.org/project/${encodeURIComponent(name)}/`;
+
+  const color = sandboxResult.score >= 80 ? 0xe74c3c    // red: critical
+    : sandboxResult.score >= 30 ? 0xe67e22              // orange: high
+    : 0xf1c40f;                                          // yellow: moderate
+
+  const fields = [
+    { name: 'Package', value: `[${name}@${version}](${npmLink})`, inline: true },
+    { name: 'Ecosystem', value: ecosystem.toUpperCase(), inline: true },
+    { name: 'Sandbox Score', value: `**${sandboxResult.score}/100** (${sandboxResult.severity})`, inline: true },
+    { name: 'Static Score', value: String(staticScore), inline: true },
+    { name: 'Status', value: 'Deferred sandbox completed after initial static-only alert', inline: false }
+  ];
+
+  // Top sandbox findings (max 5)
+  if (sandboxResult.findings && sandboxResult.findings.length > 0) {
+    const findingLines = sandboxResult.findings.slice(0, 5)
+      .map(f => `- [${f.severity || 'UNKNOWN'}] ${f.type}: ${(f.detail || '').slice(0, 100)}`)
+      .join('\n');
+    fields.push({ name: 'Sandbox Findings', value: findingLines.slice(0, 1024), inline: false });
+  }
+
+  return {
+    embeds: [{
+      title: `SANDBOX FOLLOW-UP \u2014 ${name}@${version}`,
+      color,
+      fields,
+      footer: {
+        text: `MUAD'DIB Deferred Sandbox | ${new Date().toISOString().replace('T', ' ').replace(/\.\d+Z$/, ' UTC')}`
+      },
+      timestamp: new Date().toISOString()
+    }]
+  };
+}
+
+// ── Worker lifecycle ──
+
+function startDeferredWorker(stats) {
+  _stats = stats;
+  if (_workerHandle) return _workerHandle;
+  console.log(`[DEFERRED] Worker started (interval=${DEFERRED_WORKER_INTERVAL_MS / 1000}s, max=${DEFERRED_QUEUE_MAX}, ttl=${DEFERRED_TTL_MS / 3600000}h)`);
+  _workerHandle = setInterval(async () => {
+    try {
+      await processDeferredItem(_stats);
+    } catch (err) {
+      console.error(`[DEFERRED] Worker tick error: ${err.message}`);
+    }
+  }, DEFERRED_WORKER_INTERVAL_MS);
+  return _workerHandle;
+}
+
+function stopDeferredWorker() {
+  if (_workerHandle) {
+    clearInterval(_workerHandle);
+    _workerHandle = null;
+    console.log('[DEFERRED] Worker stopped');
+  }
+}
+
+// ── Persistence ──
+
+function persistDeferredQueue() {
+  try {
+    if (_deferredQueue.length === 0) {
+      // Remove stale file
+      try { fs.unlinkSync(DEFERRED_STATE_FILE); } catch { /* ignore missing */ }
+      return;
+    }
+    // Strip staticResult to reduce file size (can be large)
+    // Keep only essential fields for persistence
+    const items = _deferredQueue.map(item => ({
+      name: item.name,
+      version: item.version,
+      ecosystem: item.ecosystem,
+      tier: item.tier,
+      riskScore: item.riskScore,
+      tarballUrl: item.tarballUrl,
+      enqueuedAt: item.enqueuedAt,
+      retries: item.retries || 0
+      // staticResult and npmRegistryMeta are NOT persisted (too large, stale after restart)
+    }));
+    const payload = JSON.stringify({
+      savedAt: new Date().toISOString(),
+      count: items.length,
+      items
+    });
+    atomicWriteFileSync(DEFERRED_STATE_FILE, payload);
+  } catch (err) {
+    console.error(`[DEFERRED] Failed to persist queue: ${err.message}`);
+  }
+}
+
+function restoreDeferredQueue() {
+  // Cleanup orphan .tmp from previous crash / disk-full (ENOSPC)
+  const tmpFile = DEFERRED_STATE_FILE + '.tmp';
+  try {
+    if (fs.existsSync(tmpFile)) {
+      const stat = fs.statSync(tmpFile);
+      console.log(`[DEFERRED] Cleaning up orphan ${path.basename(tmpFile)} (${stat.size} bytes)`);
+      fs.unlinkSync(tmpFile);
+    }
+  } catch { /* best-effort */ }
+
+  try {
+    if (!fs.existsSync(DEFERRED_STATE_FILE)) return 0;
+    const raw = fs.readFileSync(DEFERRED_STATE_FILE, 'utf8');
+    const data = JSON.parse(raw);
+
+    if (!data || !Array.isArray(data.items) || !data.savedAt) {
+      console.log('[DEFERRED] State file invalid \u2014 ignoring');
+      try { fs.unlinkSync(DEFERRED_STATE_FILE); } catch { /* ignore missing */ }
+      return 0;
+    }
+
+    // Check file age
+    const ageMs = Date.now() - new Date(data.savedAt).getTime();
+    if (ageMs > DEFERRED_TTL_MS) {
+      console.log(`[DEFERRED] State file expired (${Math.round(ageMs / 3600000)}h old) \u2014 ignoring`);
+      try { fs.unlinkSync(DEFERRED_STATE_FILE); } catch { /* ignore missing */ }
+      return 0;
+    }
+
+    // Restore items, pruning individually expired ones
+    const now = Date.now();
+    let restored = 0;
+    for (const item of data.items) {
+      if (now - item.enqueuedAt > DEFERRED_TTL_MS) continue; // expired
+      const key = `${item.name}@${item.version}`;
+      if (_deferredSeen.has(key)) continue; // dedup
+      _deferredQueue.push(item);
+      _deferredSeen.add(key);
+      restored++;
+    }
+
+    // Sort after bulk insert
+    _deferredQueue.sort((a, b) => b.riskScore - a.riskScore);
+
+    if (restored > 0) {
+      console.log(`[DEFERRED] Restored ${restored} items from disk (saved at ${data.savedAt})`);
+    }
+
+    // Delete after successful restore
+    try { fs.unlinkSync(DEFERRED_STATE_FILE); } catch { /* ignore missing */ }
+    return restored;
+  } catch (err) {
+    console.log(`[DEFERRED] WARNING: could not restore state: ${err.message}`);
+    try { fs.unlinkSync(DEFERRED_STATE_FILE); } catch { /* ignore missing */ }
+    return 0;
+  }
+}
+
+// ── Reset (for testing) ──
+
+function _resetDeferredQueue() {
+  _deferredQueue.length = 0;
+  _deferredSeen.clear();
+  _stats = null;
+  _consecutiveSkips = 0;
+  stopDeferredWorker();
+}
+
+module.exports = {
+  enqueueDeferred,
+  getDeferredQueue,
+  getDeferredQueueStats,
+  startDeferredWorker,
+  stopDeferredWorker,
+  processDeferredItem,
+  persistDeferredQueue,
+  restoreDeferredQueue,
+  buildDeferredFollowUpEmbed,
+  pruneExpired,
+  _resetDeferredQueue,
+  DEFERRED_QUEUE_MAX,
+  DEFERRED_TTL_MS,
+  DEFERRED_MAX_RETRIES,
+  DEFERRED_WORKER_INTERVAL_MS,
+  DEFERRED_STATE_FILE,
+  SKIP_LOG_INTERVAL,
+  ANTI_STARVATION_TICKS
+};

package/src/monitor/queue.js

@@ -103,6 +103,9 @@ const { getNpmLatestTarball, getPyPITarballUrl, getWeeklyDownloads, checkTrusted
 // From ./tarball-archive.js
 const { archiveSuspectTarball } = require('./tarball-archive.js');
 
+// From ./deferred-sandbox.js
+const { enqueueDeferred } = require('./deferred-sandbox.js');
+
 // --- Constants ---
 
 const SCAN_CONCURRENCY = Math.max(1, parseInt(process.env.MUADDIB_SCAN_CONCURRENCY, 10) || 8);
@@ -686,10 +689,30 @@ async function scanPackage(name, version, ecosystem, tarballUrl, registryMeta, s
           } catch (err) {
             console.error(`[MONITOR] SANDBOX error for ${name}@${version}: ${err.message}`);
           }
+        } else if (tier === '1b' && sandboxAvailable) {
+          console.log(`[MONITOR] SANDBOX DEFERRED (T1b, score=${riskScore} < 25, queue ${scanQueue.length} >= 20): ${name}@${version}`);
+          enqueueDeferred({
+            name, version, ecosystem, tier, riskScore, tarballUrl,
+            enqueuedAt: Date.now(),
+            staticResult: result,
+            npmRegistryMeta,
+            retries: 0
+          });
+          stats.sandboxDeferred = (stats.sandboxDeferred || 0) + 1;
         } else if (tier === '1b') {
-          console.log(`[MONITOR] SANDBOX SKIPPED (T1b, score=${riskScore} < 25, queue ${scanQueue.length} >= 20): ${name}@${version}`);
+          console.log(`[MONITOR] SANDBOX SKIPPED (T1b, no Docker): ${name}@${version}`);
+        } else if (tier === 2 && sandboxAvailable) {
+          console.log(`[MONITOR] SANDBOX DEFERRED (T2, queue ${scanQueue.length} >= 50): ${name}@${version}`);
+          enqueueDeferred({
+            name, version, ecosystem, tier, riskScore, tarballUrl,
+            enqueuedAt: Date.now(),
+            staticResult: result,
+            npmRegistryMeta,
+            retries: 0
+          });
+          stats.sandboxDeferred = (stats.sandboxDeferred || 0) + 1;
         } else if (tier === 2) {
-          console.log(`[MONITOR] SANDBOX SKIPPED (T2, queue ${scanQueue.length} >= 50): ${name}@${version}`);
+          console.log(`[MONITOR] SANDBOX SKIPPED (T2, no Docker): ${name}@${version}`);
         }
 
         stats.scanned++;

package/src/monitor/state.js

@@ -116,6 +116,10 @@ function atomicWriteFileSync(filePath, data) {
       console.warn(`[MONITOR] Cannot create directory ${dir} (${err.code}) — skipping write to ${path.basename(filePath)}`);
       return;
     }
+    if (err.code === 'ENOSPC') {
+      console.warn(`[MONITOR] WARNING: disk full (ENOSPC) — cannot create directory ${dir}. Free space immediately.`);
+      return;
+    }
     throw err;
   }
   const tmpFile = filePath + '.tmp';
@@ -125,7 +129,11 @@ function atomicWriteFileSync(filePath, data) {
   } catch (err) {
     if (err.code === 'EROFS' || err.code === 'EACCES' || err.code === 'EPERM') {
       console.warn(`[MONITOR] Cannot write ${path.basename(filePath)} (${err.code}) — skipping`);
-      // Clean up .tmp if it was partially written
+      try { fs.unlinkSync(tmpFile); } catch (_) { /* ignore */ }
+      return;
+    }
+    if (err.code === 'ENOSPC') {
+      console.warn(`[MONITOR] WARNING: disk full (ENOSPC) — cannot write ${path.basename(filePath)}. Free space in /tmp and data/ immediately.`);
       try { fs.unlinkSync(tmpFile); } catch (_) { /* ignore */ }
       return;
     }

package/src/monitor/webhook.js

@@ -894,6 +894,9 @@ function buildDailyReportEmbed(stats, dailyAlerts) {
         { name: 'ML', value: mlText, inline: true },
         { name: 'LLM Detective', value: llmText, inline: true },
         { name: 'Top Suspects', value: top3Text, inline: false },
+        ...((stats.sandboxDeferred || stats.deferredProcessed || stats.deferredExpired)
+          ? [{ name: 'Deferred Sandbox', value: `Enqueued: ${stats.sandboxDeferred || 0} | Processed: ${stats.deferredProcessed || 0} | Expired: ${stats.deferredExpired || 0}`, inline: false }]
+          : []),
         { name: 'System', value: healthText, inline: false }
       ],
       footer: {
@@ -939,6 +942,9 @@ async function sendDailyReport(stats, dailyAlerts, recentlyScanned, downloadsCac
     mlFiltered: stats.mlFiltered || 0,
     llmAnalyzed: stats.llmAnalyzed || 0,
     llmSuppressed: stats.llmSuppressed || 0,
+    sandboxDeferred: stats.sandboxDeferred || 0,
+    deferredProcessed: stats.deferredProcessed || 0,
+    deferredExpired: stats.deferredExpired || 0,
     changesStreamPackages: stats.changesStreamPackages || 0,
     topSuspects: dailyAlerts.slice().sort((a, b) => b.findingsCount - a.findingsCount).slice(0, 10)
   });
@@ -976,6 +982,9 @@ async function sendDailyReport(stats, dailyAlerts, recentlyScanned, downloadsCac
   stats.mlFiltered = 0;
   stats.llmAnalyzed = 0;
   stats.llmSuppressed = 0;
+  stats.sandboxDeferred = 0;
+  stats.deferredProcessed = 0;
+  stats.deferredExpired = 0;
   // Reset LLM detective internal stats
   try { require('../ml/llm-detective.js').resetStats(); } catch {}
   stats.changesStreamPackages = 0;