muaddib-scanner 2.10.47 → 2.10.49

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "muaddib-scanner",
-  "version": "2.10.47",
+  "version": "2.10.49",
   "description": "Supply-chain threat detection & response for npm & PyPI/Python",
   "main": "src/index.js",
   "bin": {

package/src/monitor/daemon.js

@@ -411,6 +411,14 @@ async function startMonitor(options, stats, dailyAlerts, recentlyScanned, downlo
   process.on('SIGINT', () => gracefulShutdown('SIGINT'));
   process.on('SIGTERM', () => gracefulShutdown('SIGTERM'));
 
+  // ─── Deferred sandbox worker ───
+  // Started BEFORE the first processQueue so it can process T1b/T2 packages
+  // that get deferred during the initial batch (which blocks for 30min-2h).
+  if (isSandboxEnabled() && sandboxAvailableRef.value) {
+    startDeferredWorker(stats);
+    console.log('[MONITOR] Deferred sandbox worker started (30s interval, dedicated slot)');
+  }
+
   // Initial poll + scan (sequential for first run)
   await poll(state, scanQueue, stats);
   saveState(state, stats);
@@ -448,14 +456,6 @@ async function startMonitor(options, stats, dailyAlerts, recentlyScanned, downlo
     persistDeferredQueue(); // Piggyback: persist deferred sandbox queue on same interval
   }, QUEUE_PERSIST_INTERVAL);
 
-  // ─── Deferred sandbox worker ───
-  // Retries T1b/T2 packages that were skipped when sandbox slots were full.
-  // Runs every 30s, processes at most 1 item per tick, yields to T1a.
-  if (isSandboxEnabled() && sandboxAvailableRef.value) {
-    startDeferredWorker(stats);
-    console.log('[MONITOR] Deferred sandbox worker started (30s interval, T1a-safe)');
-  }
-
   // ─── Continuous processing loop ───
   // Consumes scanQueue independently of polling. Workers inside processQueue
   // check scanQueue.length > 0 after each item, so items added by a concurrent

package/src/monitor/deferred-sandbox.js

@@ -6,11 +6,15 @@
  * Items are sorted by riskScore DESC (highest-risk first) to defend
  * against queue-poisoning attacks.
  *
- * The worker reserves 1 sandbox slot for T1a (never uses the last slot).
+ * The worker owns a dedicated sandbox slot (_deferredSlotBusy) that is
+ * completely independent from the shared semaphore used by T1a/T1b/T2.
+ * This guarantees the deferred worker can always process, regardless of
+ * how many main-path sandboxes are running. The VPS supports N+1
+ * concurrent gVisor containers (3 main + 1 deferred).
  */
 const fs = require('fs');
 const path = require('path');
-const { runSandbox, getSandboxSemaphore, SANDBOX_CONCURRENCY_MAX } = require('../sandbox/index.js');
+const { runSandbox } = require('../sandbox/index.js');
 const { isCanaryEnabled } = require('./classify.js');
 const { getWebhookUrl, alertedPackageRules, persistAlert, buildAlertData } = require('./webhook.js');
 const { sendWebhook } = require('../webhook.js');
@@ -22,15 +26,13 @@ const DEFERRED_TTL_MS = 24 * 60 * 60 * 1000; // 24h
 const DEFERRED_MAX_RETRIES = 2;
 const DEFERRED_WORKER_INTERVAL_MS = 30_000; // 30s
 const DEFERRED_STATE_FILE = path.join(__dirname, '..', '..', 'data', 'deferred-queue.json');
-const SKIP_LOG_INTERVAL = 10;        // Log every N skipped ticks (throttle)
-const ANTI_STARVATION_TICKS = 20;    // Force processing after N consecutive skips (~10min)
 
 // ── Mutable state ──
 const _deferredQueue = [];
 const _deferredSeen = new Set(); // name@version dedup
 let _workerHandle = null;
 let _stats = null; // reference to shared stats object
-let _consecutiveSkips = 0;       // Tracks consecutive yield-skips for anti-starvation
+let _deferredSlotBusy = false;   // Dedicated slot: true while deferred sandbox is running
 
 // ── Queue management ──
 
@@ -129,25 +131,12 @@ async function processDeferredItem(stats) {
 
   if (_deferredQueue.length === 0) return null;
 
-  // 2. Yield check: reserve 1 slot for T1a (unless anti-starvation kicks in)
-  const sem = getSandboxSemaphore();
-  const slotsFullForDeferred = sem.active >= SANDBOX_CONCURRENCY_MAX - 1;
-  const forceAntiStarvation = _consecutiveSkips >= ANTI_STARVATION_TICKS && sem.active < SANDBOX_CONCURRENCY_MAX;
-
-  if (slotsFullForDeferred && !forceAntiStarvation) {
-    _consecutiveSkips++;
+  // 2. Dedicated slot check — completely independent from main semaphore
+  if (_deferredSlotBusy) {
     if (stats) stats.deferredSkipped = (stats.deferredSkipped || 0) + 1;
-    if (_consecutiveSkips % SKIP_LOG_INTERVAL === 0) {
-      console.log(`[DEFERRED] YIELD: ${_consecutiveSkips} consecutive skips (slots=${sem.active}/${SANDBOX_CONCURRENCY_MAX}, pending=${_deferredQueue.length})`);
-    }
     return null;
   }
 
-  if (forceAntiStarvation) {
-    console.log(`[DEFERRED] ANTI-STARVATION: forcing processing after ${_consecutiveSkips} skips (slots=${sem.active}/${SANDBOX_CONCURRENCY_MAX}, pending=${_deferredQueue.length})`);
-  }
-  _consecutiveSkips = 0;
-
   // 3. Pick highest-score item
   const item = _deferredQueue.shift();
   const key = `${item.name}@${item.version}`;
@@ -155,11 +144,12 @@ async function processDeferredItem(stats) {
 
   console.log(`[DEFERRED] PROCESSING: ${key} (tier=${item.tier === 2 ? 'T2' : 'T1b'}, score=${item.riskScore}, retries=${item.retries})`);
 
-  // 4. Run sandbox
+  // 4. Run sandbox on dedicated slot (bypasses shared semaphore)
+  _deferredSlotBusy = true;
   let sandboxResult;
   try {
     const canary = isCanaryEnabled();
-    sandboxResult = await runSandbox(item.name, { canary });
+    sandboxResult = await runSandbox(item.name, { canary, skipSemaphore: true });
     console.log(`[DEFERRED] SANDBOX COMPLETE: ${key} -> score=${sandboxResult.score}, severity=${sandboxResult.severity}`);
   } catch (err) {
     console.error(`[DEFERRED] SANDBOX ERROR: ${key} — ${err.message}`);
@@ -174,6 +164,8 @@ async function processDeferredItem(stats) {
       console.log(`[DEFERRED] RE-ENQUEUED: ${key} for retry (attempt ${item.retries + 1}/${DEFERRED_MAX_RETRIES})`);
     }
     return null;
+  } finally {
+    _deferredSlotBusy = false;
   }
 
   // 5. Follow-up webhook if sandbox found something
@@ -391,10 +383,14 @@ function _resetDeferredQueue() {
   _deferredQueue.length = 0;
   _deferredSeen.clear();
   _stats = null;
-  _consecutiveSkips = 0;
+  _deferredSlotBusy = false;
   stopDeferredWorker();
 }
 
+function isDeferredSlotBusy() {
+  return _deferredSlotBusy;
+}
+
 module.exports = {
   enqueueDeferred,
   getDeferredQueue,
@@ -406,12 +402,11 @@ module.exports = {
   restoreDeferredQueue,
   buildDeferredFollowUpEmbed,
   pruneExpired,
+  isDeferredSlotBusy,
   _resetDeferredQueue,
   DEFERRED_QUEUE_MAX,
   DEFERRED_TTL_MS,
   DEFERRED_MAX_RETRIES,
   DEFERRED_WORKER_INTERVAL_MS,
-  DEFERRED_STATE_FILE,
-  SKIP_LOG_INTERVAL,
-  ANTI_STARVATION_TICKS
+  DEFERRED_STATE_FILE
 };

package/src/monitor/state.js

@@ -690,6 +690,7 @@ function loadDailyStats(stats, dailyAlerts) {
       stats.mlFiltered = data.mlFiltered || 0;
       stats.llmAnalyzed = data.llmAnalyzed || 0;
       stats.llmSuppressed = data.llmSuppressed || 0;
+      stats.changesStreamPackages = data.changesStreamPackages || 0;
       if (Array.isArray(data.dailyAlerts)) {
         dailyAlerts.length = 0;
         dailyAlerts.push(...data.dailyAlerts);
@@ -716,6 +717,7 @@ function saveDailyStats(stats, dailyAlerts) {
       mlFiltered: stats.mlFiltered,
       llmAnalyzed: stats.llmAnalyzed || 0,
       llmSuppressed: stats.llmSuppressed || 0,
+      changesStreamPackages: stats.changesStreamPackages || 0,
       dailyAlerts: dailyAlerts.slice()
     };
     atomicWriteFileSync(DAILY_STATS_FILE, JSON.stringify(data, null, 2));

package/src/sandbox/index.js

@@ -23,7 +23,8 @@ const CONTAINER_TIMEOUT = 120000; // 120 seconds
 const SINGLE_RUN_TIMEOUT = 90000; // 90 seconds per run in multi-run mode (gVisor ~30% I/O overhead)
 
 // ── Sandbox concurrency limiter ──
-// Prevents Docker container saturation under load (16 workers × 3 runs = 48 containers).
+// Prevents Docker container saturation under load (main-path T1a/T1b/T2).
+// The deferred worker manages its own dedicated slot outside this semaphore.
 // Pattern: same semaphore as src/shared/http-limiter.js.
 const SANDBOX_CONCURRENCY_MAX = Math.max(1, parseInt(process.env.MUADDIB_SANDBOX_CONCURRENCY, 10) || 3);
 
@@ -565,16 +566,21 @@ async function runSandbox(packageName, options = {}) {
 
   const mode = strict ? 'strict' : 'permissive';
 
-  // Acquire sandbox slot — blocks if SANDBOX_CONCURRENCY_MAX containers already running
-  const queueLen = _sandboxSemaphore.queue.length;
-  if (queueLen > 0) {
-    console.log(`[SANDBOX] Waiting for sandbox slot (${_sandboxSemaphore.active}/${SANDBOX_CONCURRENCY_MAX} active, ${queueLen} queued)...`);
+  // Acquire sandbox slot — blocks if SANDBOX_CONCURRENCY_MAX containers already running.
+  // skipSemaphore: deferred worker manages its own dedicated slot outside this semaphore.
+  const skipSem = options.skipSemaphore === true;
+  if (!skipSem) {
+    const queueLen = _sandboxSemaphore.queue.length;
+    if (queueLen > 0) {
+      console.log(`[SANDBOX] Waiting for sandbox slot (${_sandboxSemaphore.active}/${SANDBOX_CONCURRENCY_MAX} active, ${queueLen} queued)...`);
+    }
+    await acquireSandboxSlot();
   }
-  await acquireSandboxSlot();
 
   try {
     const runtimeLabel = useGvisor ? 'gvisor' : 'docker';
-    console.log(`[SANDBOX] Analyzing "${displayName}" in isolated container (mode: ${mode}, runtime: ${runtimeLabel}${canaryEnabled ? ', canary: on' : ''}${local ? ', local' : ''}, runs: ${TIME_OFFSETS.length}, slots: ${_sandboxSemaphore.active}/${SANDBOX_CONCURRENCY_MAX})...`);
+    const slotInfo = skipSem ? 'deferred-slot' : `${_sandboxSemaphore.active}/${SANDBOX_CONCURRENCY_MAX}`;
+    console.log(`[SANDBOX] Analyzing "${displayName}" in isolated container (mode: ${mode}, runtime: ${runtimeLabel}${canaryEnabled ? ', canary: on' : ''}${local ? ', local' : ''}, runs: ${TIME_OFFSETS.length}, slots: ${slotInfo})...`);
 
     const allRuns = [];
     let bestResult = cleanResult;
@@ -639,7 +645,7 @@ async function runSandbox(packageName, options = {}) {
     displayResults(bestResult);
     return bestResult;
   } finally {
-    releaseSandboxSlot();
+    if (!skipSem) releaseSandboxSlot();
   }
 }