muaddib-scanner 2.10.45 → 2.10.46

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "muaddib-scanner",
-  "version": "2.10.45",
+  "version": "2.10.46",
   "description": "Supply-chain threat detection & response for npm & PyPI/Python",
   "main": "src/index.js",
   "bin": {

package/src/monitor/daemon.js

@@ -10,6 +10,7 @@ const { pendingGrouped, flushScopeGroup, sendDailyReport, DAILY_REPORT_HOUR } =
 const { poll } = require('./ingestion.js');
 const { processQueue, SCAN_CONCURRENCY } = require('./queue.js');
 const { startHealthcheck } = require('./healthcheck.js');
+const { startDeferredWorker, stopDeferredWorker, persistDeferredQueue, restoreDeferredQueue } = require('./deferred-sandbox.js');
 
 const POLL_INTERVAL = 60_000;
 const PROCESS_LOOP_INTERVAL = 2_000;    // Queue check interval when empty
@@ -149,6 +150,11 @@ function reportStats(stats) {
   if (stats.tarballCacheHits) {
     console.log(`[MONITOR]   Tarball cache hits: ${stats.tarballCacheHits}`);
   }
+  if (stats.sandboxDeferred || stats.deferredProcessed) {
+    const { getDeferredQueueStats } = require('./deferred-sandbox.js');
+    const dq = getDeferredQueueStats();
+    console.log(`[MONITOR]   Deferred sandbox: ${stats.sandboxDeferred || 0} enqueued, ${stats.deferredProcessed || 0} processed, ${stats.deferredExpired || 0} expired, ${dq.size} pending`);
+  }
   stats.lastReportTime = Date.now();
 }
 
@@ -262,6 +268,12 @@ async function startMonitor(options, stats, dailyAlerts, recentlyScanned, downlo
     console.log(`[MONITOR] ${restoredCount} packages pre-loaded from previous session`);
   }
 
+  // Restore deferred sandbox queue from previous run
+  const deferredRestored = restoreDeferredQueue();
+  if (deferredRestored > 0) {
+    console.log(`[MONITOR] ${deferredRestored} deferred sandbox items restored from previous session`);
+  }
+
   // Graceful shutdown handler (shared by SIGINT and SIGTERM)
   // Daily report is NEVER sent on shutdown — it only fires at 08:00 Paris time.
   // Counters are persisted to disk so they survive the restart.
@@ -278,6 +290,9 @@ async function startMonitor(options, stats, dailyAlerts, recentlyScanned, downlo
     }
     // Persist remaining queue items so they survive the restart
     persistQueue(scanQueue, state);
+    // Stop deferred sandbox worker and persist its queue
+    stopDeferredWorker();
+    persistDeferredQueue();
     healthcheck.stop();
     // Flush all pending scope groups before exit
     for (const [scope, group] of pendingGrouped) {
@@ -329,8 +344,17 @@ async function startMonitor(options, stats, dailyAlerts, recentlyScanned, downlo
   queuePersistHandle = setInterval(() => {
     if (!running) return;
     persistQueue(scanQueue, state);
+    persistDeferredQueue(); // Piggyback: persist deferred sandbox queue on same interval
   }, QUEUE_PERSIST_INTERVAL);
 
+  // ─── Deferred sandbox worker ───
+  // Retries T1b/T2 packages that were skipped when sandbox slots were full.
+  // Runs every 30s, processes at most 1 item per tick, yields to T1a.
+  if (isSandboxEnabled() && sandboxAvailableRef.value) {
+    startDeferredWorker(stats);
+    console.log('[MONITOR] Deferred sandbox worker started (30s interval, T1a-safe)');
+  }
+
   // ─── Continuous processing loop ───
   // Consumes scanQueue independently of polling. Workers inside processQueue
   // check scanQueue.length > 0 after each item, so items added by a concurrent

package/src/monitor/deferred-sandbox.js

@@ -0,0 +1,388 @@
+/**
+ * Deferred Sandbox Queue
+ *
+ * When T1b/T2 packages are skipped due to sandbox slot pressure,
+ * they are enqueued here and retried when slots free up.
+ * Items are sorted by riskScore DESC (highest-risk first) to defend
+ * against queue-poisoning attacks.
+ *
+ * The worker reserves 1 sandbox slot for T1a (never uses the last slot).
+ */
+const fs = require('fs');
+const path = require('path');
+const { runSandbox, getSandboxSemaphore, SANDBOX_CONCURRENCY_MAX } = require('../sandbox/index.js');
+const { isCanaryEnabled } = require('./classify.js');
+const { getWebhookUrl, alertedPackageRules, persistAlert, buildAlertData } = require('./webhook.js');
+const { sendWebhook } = require('../webhook.js');
+const { atomicWriteFileSync } = require('./state.js');
+
+// ── Constants ──
+const DEFERRED_QUEUE_MAX = 500;
+const DEFERRED_TTL_MS = 24 * 60 * 60 * 1000; // 24h
+const DEFERRED_MAX_RETRIES = 2;
+const DEFERRED_WORKER_INTERVAL_MS = 30_000; // 30s
+const DEFERRED_STATE_FILE = path.join(__dirname, '..', '..', 'data', 'deferred-queue.json');
+
+// ── Mutable state ──
+const _deferredQueue = [];
+const _deferredSeen = new Set(); // name@version dedup
+let _workerHandle = null;
+let _stats = null; // reference to shared stats object
+
+// ── Queue management ──
+
+/**
+ * Enqueue a T1b/T2 package for deferred sandbox analysis.
+ * Items are sorted by riskScore DESC (highest risk first).
+ * When the queue is full, the lowest-score item is evicted if the new item scores higher.
+ *
+ * @param {object} item - Package to defer
+ * @returns {boolean} true if enqueued, false if rejected
+ */
+function enqueueDeferred(item) {
+  // Guard: only T1b and T2 are allowed
+  if (item.tier !== '1b' && item.tier !== 2) {
+    console.error(`[DEFERRED] REJECTED: ${item.name}@${item.version} — tier ${item.tier} not eligible`);
+    return false;
+  }
+
+  const key = `${item.name}@${item.version}`;
+
+  // Dedup
+  if (_deferredSeen.has(key)) {
+    console.log(`[DEFERRED] DEDUP: ${key} already in deferred queue`);
+    return false;
+  }
+
+  // Queue full — evict lowest or reject
+  if (_deferredQueue.length >= DEFERRED_QUEUE_MAX) {
+    const lowest = _deferredQueue[_deferredQueue.length - 1];
+    if (item.riskScore > lowest.riskScore) {
+      const evictKey = `${lowest.name}@${lowest.version}`;
+      _deferredQueue.pop();
+      _deferredSeen.delete(evictKey);
+      console.log(`[DEFERRED] EVICTED: ${evictKey} (score=${lowest.riskScore}) to make room for ${key} (score=${item.riskScore})`);
+    } else {
+      console.log(`[DEFERRED] QUEUE FULL: ${key} (score=${item.riskScore}) rejected — all ${DEFERRED_QUEUE_MAX} items have higher scores`);
+      return false;
+    }
+  }
+
+  _deferredQueue.push(item);
+  _deferredSeen.add(key);
+  // Sort by riskScore DESC (highest first)
+  _deferredQueue.sort((a, b) => b.riskScore - a.riskScore);
+  console.log(`[DEFERRED] ENQUEUED: ${key} (tier=${item.tier === 2 ? 'T2' : 'T1b'}, score=${item.riskScore}, queue=${_deferredQueue.length})`);
+  return true;
+}
+
+function getDeferredQueue() {
+  return _deferredQueue;
+}
+
+function getDeferredQueueStats() {
+  const tierBreakdown = { t1b: 0, t2: 0 };
+  for (const item of _deferredQueue) {
+    if (item.tier === '1b') tierBreakdown.t1b++;
+    else if (item.tier === 2) tierBreakdown.t2++;
+  }
+  return {
+    size: _deferredQueue.length,
+    oldest: _deferredQueue.length > 0
+      ? _deferredQueue[_deferredQueue.length - 1].enqueuedAt
+      : null,
+    tierBreakdown
+  };
+}
+
+// ── TTL pruning ──
+
+function pruneExpired(stats) {
+  const now = Date.now();
+  let pruned = 0;
+  for (let i = _deferredQueue.length - 1; i >= 0; i--) {
+    if (now - _deferredQueue[i].enqueuedAt > DEFERRED_TTL_MS) {
+      const item = _deferredQueue[i];
+      const key = `${item.name}@${item.version}`;
+      _deferredQueue.splice(i, 1);
+      _deferredSeen.delete(key);
+      if (stats) stats.deferredExpired = (stats.deferredExpired || 0) + 1;
+      console.log(`[DEFERRED] EXPIRED: ${key} (age=${((now - item.enqueuedAt) / 3600000).toFixed(1)}h)`);
+      pruned++;
+    }
+  }
+  return pruned;
+}
+
+// ── Worker ──
+
+/**
+ * Process one deferred item. Exported for testing.
+ * @returns {object|null} sandboxResult or null if nothing processed
+ */
+async function processDeferredItem(stats) {
+  // 1. Prune expired items
+  pruneExpired(stats);
+
+  if (_deferredQueue.length === 0) return null;
+
+  // 2. Yield check: reserve 1 slot for T1a
+  const sem = getSandboxSemaphore();
+  if (sem.active >= SANDBOX_CONCURRENCY_MAX - 1) {
+    return null; // All slots busy or only 1 free — keep it for T1a
+  }
+
+  // 3. Pick highest-score item
+  const item = _deferredQueue.shift();
+  const key = `${item.name}@${item.version}`;
+  _deferredSeen.delete(key);
+
+  console.log(`[DEFERRED] PROCESSING: ${key} (tier=${item.tier === 2 ? 'T2' : 'T1b'}, score=${item.riskScore}, retries=${item.retries})`);
+
+  // 4. Run sandbox
+  let sandboxResult;
+  try {
+    const canary = isCanaryEnabled();
+    sandboxResult = await runSandbox(item.name, { canary });
+    console.log(`[DEFERRED] SANDBOX COMPLETE: ${key} -> score=${sandboxResult.score}, severity=${sandboxResult.severity}`);
+  } catch (err) {
+    console.error(`[DEFERRED] SANDBOX ERROR: ${key} — ${err.message}`);
+    item.retries = (item.retries || 0) + 1;
+    if (item.retries >= DEFERRED_MAX_RETRIES) {
+      console.log(`[DEFERRED] DROPPED: ${key} after ${item.retries} failed attempts`);
+    } else {
+      // Re-enqueue for retry
+      _deferredQueue.push(item);
+      _deferredSeen.add(key);
+      _deferredQueue.sort((a, b) => b.riskScore - a.riskScore);
+      console.log(`[DEFERRED] RE-ENQUEUED: ${key} for retry (attempt ${item.retries + 1}/${DEFERRED_MAX_RETRIES})`);
+    }
+    return null;
+  }
+
+  // 5. Follow-up webhook if sandbox found something
+  if (stats) stats.deferredProcessed = (stats.deferredProcessed || 0) + 1;
+
+  if (sandboxResult && sandboxResult.score > 0) {
+    const deferredDedupKey = 'deferred_sandbox';
+    const previousRules = alertedPackageRules.get(item.name);
+    const alreadySentFollowUp = previousRules && previousRules.has(deferredDedupKey);
+
+    if (!alreadySentFollowUp) {
+      const url = getWebhookUrl();
+      if (url) {
+        try {
+          const embed = buildDeferredFollowUpEmbed(
+            item.name, item.version, item.ecosystem,
+            sandboxResult,
+            item.riskScore
+          );
+          await sendWebhook(url, embed, { rawPayload: true });
+          console.log(`[DEFERRED] FOLLOW-UP WEBHOOK: ${key} (sandbox score=${sandboxResult.score})`);
+
+          // Track in dedup map
+          if (previousRules) {
+            previousRules.add(deferredDedupKey);
+          } else {
+            alertedPackageRules.set(item.name, new Set([deferredDedupKey]));
+          }
+        } catch (webhookErr) {
+          console.error(`[DEFERRED] FOLLOW-UP WEBHOOK FAILED: ${key} — ${webhookErr.message}`);
+        }
+      }
+
+      // Persist updated alert with sandbox data
+      try {
+        const alertData = buildAlertData(
+          item.name, item.version, item.ecosystem,
+          item.staticResult, sandboxResult
+        );
+        persistAlert(item.name, item.version, item.ecosystem, alertData);
+        console.log(`[DEFERRED] ALERT PERSISTED: ${key} (with sandbox data)`);
+      } catch (persistErr) {
+        console.error(`[DEFERRED] ALERT PERSIST FAILED: ${key} — ${persistErr.message}`);
+      }
+    } else {
+      console.log(`[DEFERRED] DEDUP: follow-up already sent for ${item.name}`);
+    }
+  } else {
+    console.log(`[DEFERRED] CLEAN: ${key} (sandbox score=0, static score=${item.riskScore})`);
+  }
+
+  return sandboxResult;
+}
+
+/**
+ * Build Discord embed for deferred sandbox follow-up.
+ */
+function buildDeferredFollowUpEmbed(name, version, ecosystem, sandboxResult, staticScore) {
+  const npmLink = ecosystem === 'npm'
+    ? `https://www.npmjs.com/package/${encodeURIComponent(name)}`
+    : `https://pypi.org/project/${encodeURIComponent(name)}/`;
+
+  const color = sandboxResult.score >= 80 ? 0xe74c3c    // red: critical
+    : sandboxResult.score >= 30 ? 0xe67e22              // orange: high
+    : 0xf1c40f;                                          // yellow: moderate
+
+  const fields = [
+    { name: 'Package', value: `[${name}@${version}](${npmLink})`, inline: true },
+    { name: 'Ecosystem', value: ecosystem.toUpperCase(), inline: true },
+    { name: 'Sandbox Score', value: `**${sandboxResult.score}/100** (${sandboxResult.severity})`, inline: true },
+    { name: 'Static Score', value: String(staticScore), inline: true },
+    { name: 'Status', value: 'Deferred sandbox completed after initial static-only alert', inline: false }
+  ];
+
+  // Top sandbox findings (max 5)
+  if (sandboxResult.findings && sandboxResult.findings.length > 0) {
+    const findingLines = sandboxResult.findings.slice(0, 5)
+      .map(f => `- [${f.severity || 'UNKNOWN'}] ${f.type}: ${(f.detail || '').slice(0, 100)}`)
+      .join('\n');
+    fields.push({ name: 'Sandbox Findings', value: findingLines.slice(0, 1024), inline: false });
+  }
+
+  return {
+    embeds: [{
+      title: `SANDBOX FOLLOW-UP \u2014 ${name}@${version}`,
+      color,
+      fields,
+      footer: {
+        text: `MUAD'DIB Deferred Sandbox | ${new Date().toISOString().replace('T', ' ').replace(/\.\d+Z$/, ' UTC')}`
+      },
+      timestamp: new Date().toISOString()
+    }]
+  };
+}
+
+// ── Worker lifecycle ──
+
+function startDeferredWorker(stats) {
+  _stats = stats;
+  if (_workerHandle) return _workerHandle;
+  console.log(`[DEFERRED] Worker started (interval=${DEFERRED_WORKER_INTERVAL_MS / 1000}s, max=${DEFERRED_QUEUE_MAX}, ttl=${DEFERRED_TTL_MS / 3600000}h)`);
+  _workerHandle = setInterval(async () => {
+    try {
+      await processDeferredItem(_stats);
+    } catch (err) {
+      console.error(`[DEFERRED] Worker tick error: ${err.message}`);
+    }
+  }, DEFERRED_WORKER_INTERVAL_MS);
+  return _workerHandle;
+}
+
+function stopDeferredWorker() {
+  if (_workerHandle) {
+    clearInterval(_workerHandle);
+    _workerHandle = null;
+    console.log('[DEFERRED] Worker stopped');
+  }
+}
+
+// ── Persistence ──
+
+function persistDeferredQueue() {
+  try {
+    if (_deferredQueue.length === 0) {
+      // Remove stale file
+      try { fs.unlinkSync(DEFERRED_STATE_FILE); } catch { /* ignore missing */ }
+      return;
+    }
+    // Strip staticResult to reduce file size (can be large)
+    // Keep only essential fields for persistence
+    const items = _deferredQueue.map(item => ({
+      name: item.name,
+      version: item.version,
+      ecosystem: item.ecosystem,
+      tier: item.tier,
+      riskScore: item.riskScore,
+      tarballUrl: item.tarballUrl,
+      enqueuedAt: item.enqueuedAt,
+      retries: item.retries || 0
+      // staticResult and npmRegistryMeta are NOT persisted (too large, stale after restart)
+    }));
+    const payload = JSON.stringify({
+      savedAt: new Date().toISOString(),
+      count: items.length,
+      items
+    });
+    atomicWriteFileSync(DEFERRED_STATE_FILE, payload);
+  } catch (err) {
+    console.error(`[DEFERRED] Failed to persist queue: ${err.message}`);
+  }
+}
+
+function restoreDeferredQueue() {
+  try {
+    if (!fs.existsSync(DEFERRED_STATE_FILE)) return 0;
+    const raw = fs.readFileSync(DEFERRED_STATE_FILE, 'utf8');
+    const data = JSON.parse(raw);
+
+    if (!data || !Array.isArray(data.items) || !data.savedAt) {
+      console.log('[DEFERRED] State file invalid \u2014 ignoring');
+      try { fs.unlinkSync(DEFERRED_STATE_FILE); } catch { /* ignore missing */ }
+      return 0;
+    }
+
+    // Check file age
+    const ageMs = Date.now() - new Date(data.savedAt).getTime();
+    if (ageMs > DEFERRED_TTL_MS) {
+      console.log(`[DEFERRED] State file expired (${Math.round(ageMs / 3600000)}h old) \u2014 ignoring`);
+      try { fs.unlinkSync(DEFERRED_STATE_FILE); } catch { /* ignore missing */ }
+      return 0;
+    }
+
+    // Restore items, pruning individually expired ones
+    const now = Date.now();
+    let restored = 0;
+    for (const item of data.items) {
+      if (now - item.enqueuedAt > DEFERRED_TTL_MS) continue; // expired
+      const key = `${item.name}@${item.version}`;
+      if (_deferredSeen.has(key)) continue; // dedup
+      _deferredQueue.push(item);
+      _deferredSeen.add(key);
+      restored++;
+    }
+
+    // Sort after bulk insert
+    _deferredQueue.sort((a, b) => b.riskScore - a.riskScore);
+
+    if (restored > 0) {
+      console.log(`[DEFERRED] Restored ${restored} items from disk (saved at ${data.savedAt})`);
+    }
+
+    // Delete after successful restore
+    try { fs.unlinkSync(DEFERRED_STATE_FILE); } catch { /* ignore missing */ }
+    return restored;
+  } catch (err) {
+    console.log(`[DEFERRED] WARNING: could not restore state: ${err.message}`);
+    try { fs.unlinkSync(DEFERRED_STATE_FILE); } catch { /* ignore missing */ }
+    return 0;
+  }
+}
+
+// ── Reset (for testing) ──
+
+function _resetDeferredQueue() {
+  _deferredQueue.length = 0;
+  _deferredSeen.clear();
+  _stats = null;
+  stopDeferredWorker();
+}
+
+module.exports = {
+  enqueueDeferred,
+  getDeferredQueue,
+  getDeferredQueueStats,
+  startDeferredWorker,
+  stopDeferredWorker,
+  processDeferredItem,
+  persistDeferredQueue,
+  restoreDeferredQueue,
+  buildDeferredFollowUpEmbed,
+  pruneExpired,
+  _resetDeferredQueue,
+  DEFERRED_QUEUE_MAX,
+  DEFERRED_TTL_MS,
+  DEFERRED_MAX_RETRIES,
+  DEFERRED_WORKER_INTERVAL_MS,
+  DEFERRED_STATE_FILE
+};

package/src/monitor/queue.js

@@ -103,6 +103,9 @@ const { getNpmLatestTarball, getPyPITarballUrl, getWeeklyDownloads, checkTrusted
 // From ./tarball-archive.js
 const { archiveSuspectTarball } = require('./tarball-archive.js');
 
+// From ./deferred-sandbox.js
+const { enqueueDeferred } = require('./deferred-sandbox.js');
+
 // --- Constants ---
 
 const SCAN_CONCURRENCY = Math.max(1, parseInt(process.env.MUADDIB_SCAN_CONCURRENCY, 10) || 8);
@@ -686,10 +689,30 @@ async function scanPackage(name, version, ecosystem, tarballUrl, registryMeta, s
           } catch (err) {
             console.error(`[MONITOR] SANDBOX error for ${name}@${version}: ${err.message}`);
           }
+        } else if (tier === '1b' && sandboxAvailable) {
+          console.log(`[MONITOR] SANDBOX DEFERRED (T1b, score=${riskScore} < 25, queue ${scanQueue.length} >= 20): ${name}@${version}`);
+          enqueueDeferred({
+            name, version, ecosystem, tier, riskScore, tarballUrl,
+            enqueuedAt: Date.now(),
+            staticResult: result,
+            npmRegistryMeta,
+            retries: 0
+          });
+          stats.sandboxDeferred = (stats.sandboxDeferred || 0) + 1;
         } else if (tier === '1b') {
-          console.log(`[MONITOR] SANDBOX SKIPPED (T1b, score=${riskScore} < 25, queue ${scanQueue.length} >= 20): ${name}@${version}`);
+          console.log(`[MONITOR] SANDBOX SKIPPED (T1b, no Docker): ${name}@${version}`);
+        } else if (tier === 2 && sandboxAvailable) {
+          console.log(`[MONITOR] SANDBOX DEFERRED (T2, queue ${scanQueue.length} >= 50): ${name}@${version}`);
+          enqueueDeferred({
+            name, version, ecosystem, tier, riskScore, tarballUrl,
+            enqueuedAt: Date.now(),
+            staticResult: result,
+            npmRegistryMeta,
+            retries: 0
+          });
+          stats.sandboxDeferred = (stats.sandboxDeferred || 0) + 1;
         } else if (tier === 2) {
-          console.log(`[MONITOR] SANDBOX SKIPPED (T2, queue ${scanQueue.length} >= 50): ${name}@${version}`);
+          console.log(`[MONITOR] SANDBOX SKIPPED (T2, no Docker): ${name}@${version}`);
         }
 
         stats.scanned++;

package/src/monitor/webhook.js

@@ -894,6 +894,9 @@ function buildDailyReportEmbed(stats, dailyAlerts) {
         { name: 'ML', value: mlText, inline: true },
         { name: 'LLM Detective', value: llmText, inline: true },
         { name: 'Top Suspects', value: top3Text, inline: false },
+        ...((stats.sandboxDeferred || stats.deferredProcessed || stats.deferredExpired)
+          ? [{ name: 'Deferred Sandbox', value: `Enqueued: ${stats.sandboxDeferred || 0} | Processed: ${stats.deferredProcessed || 0} | Expired: ${stats.deferredExpired || 0}`, inline: false }]
+          : []),
         { name: 'System', value: healthText, inline: false }
       ],
       footer: {
@@ -939,6 +942,9 @@ async function sendDailyReport(stats, dailyAlerts, recentlyScanned, downloadsCac
     mlFiltered: stats.mlFiltered || 0,
     llmAnalyzed: stats.llmAnalyzed || 0,
     llmSuppressed: stats.llmSuppressed || 0,
+    sandboxDeferred: stats.sandboxDeferred || 0,
+    deferredProcessed: stats.deferredProcessed || 0,
+    deferredExpired: stats.deferredExpired || 0,
     changesStreamPackages: stats.changesStreamPackages || 0,
     topSuspects: dailyAlerts.slice().sort((a, b) => b.findingsCount - a.findingsCount).slice(0, 10)
   });
@@ -976,6 +982,9 @@ async function sendDailyReport(stats, dailyAlerts, recentlyScanned, downloadsCac
   stats.mlFiltered = 0;
   stats.llmAnalyzed = 0;
   stats.llmSuppressed = 0;
+  stats.sandboxDeferred = 0;
+  stats.deferredProcessed = 0;
+  stats.deferredExpired = 0;
   // Reset LLM detective internal stats
   try { require('../ml/llm-detective.js').resetStats(); } catch {}
   stats.changesStreamPackages = 0;