muaddib-scanner 2.10.78 → 2.10.79

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "muaddib-scanner",
-  "version": "2.10.78",
+  "version": "2.10.79",
   "description": "Supply-chain threat detection & response for npm & PyPI/Python",
   "main": "src/index.js",
   "bin": {

package/src/ioc/scraper.js

@@ -434,6 +434,7 @@ function createFreshness(source, confidence) {
  */
 function extractVersions(affected) {
   const versions = new Set();
+  let hasUnboundedRange = false;
 
   if (affected.versions && affected.versions.length > 0) {
     for (const v of affected.versions) {
@@ -445,7 +446,16 @@ function extractVersions(affected) {
     for (const range of affected.ranges) {
       if (range.events) {
         for (const event of range.events) {
-          if (event.introduced && event.introduced !== '0') {
+          if (event.introduced === '0') {
+            // "introduced": "0" with no "fixed" = all versions malicious (wildcard).
+            // This is the standard OSV format used by Amazon Inspector bulk imports
+            // (tea.xyz campaign, 150K+ packages). Without this, these entries are
+            // silently dropped and the IOC database loses ~185K packages.
+            const hasFixed = range.events.some(e => e.fixed);
+            if (!hasFixed) {
+              hasUnboundedRange = true;
+            }
+          } else if (event.introduced) {
             versions.add(event.introduced);
           }
         }
@@ -453,6 +463,11 @@ function extractVersions(affected) {
     }
   }
 
+  // Wildcard: unbounded range (introduced=0, no fixed) and no explicit versions
+  if (versions.size === 0 && hasUnboundedRange) {
+    return ['*'];
+  }
+
   if (versions.size === 0) {
     _noVersionSkipCount++;
     return [];

package/src/monitor/adaptive-concurrency.js

@@ -0,0 +1,106 @@
+'use strict';
+
+/**
+ * Adaptive concurrency controller for the scan worker pool.
+ *
+ * Adjusts target concurrency every ADJUST_INTERVAL_MS based on three signals:
+ *   1. Queue depth — scale up when backlog grows, down when idle
+ *   2. Memory pressure — always reduce under heap pressure
+ *   3. Timeout rate — reduce when system is saturated (I/O contention)
+ *
+ * Scale-up is aggressive (+4) because backlog = lost coverage.
+ * Scale-down is gradual (-2) to avoid thrashing.
+ * Memory pressure overrides everything (OOM kills lose the in-memory queue).
+ */
+
+const MIN_CONCURRENCY = 4;
+const BASE_CONCURRENCY = Math.max(MIN_CONCURRENCY, parseInt(process.env.MUADDIB_SCAN_CONCURRENCY, 10) || 8);
+const MAX_CONCURRENCY = Math.max(BASE_CONCURRENCY, parseInt(process.env.MUADDIB_MAX_CONCURRENCY, 10) || 32);
+const ADJUST_INTERVAL_MS = 30_000;
+
+// Queue depth thresholds
+const QUEUE_BACKLOG_THRESHOLD = 1000;
+const QUEUE_IDLE_THRESHOLD = 100;
+
+// System pressure thresholds
+const MEMORY_PRESSURE_THRESHOLD = 0.75;
+const TIMEOUT_RATE_THRESHOLD = 0.15;
+const TIMEOUT_RATE_MIN_SAMPLES = 20;
+
+// Track previous stats snapshot for delta computation
+let _prevScanned = 0;
+let _prevTimeouts = 0;
+
+/**
+ * Compute new target concurrency from system signals.
+ * Uses stats deltas (not cumulative) for timeout rate — avoids stale data.
+ *
+ * @param {number} current - Current target concurrency
+ * @param {number} queueDepth - scanQueue.length
+ * @param {Object} stats - Monitor stats object (scanned, errorsByType.static_timeout)
+ * @returns {{ target: number, reason: string }}
+ */
+function computeTarget(current, queueDepth, stats) {
+  const mem = process.memoryUsage();
+  const memPressure = mem.heapUsed / mem.heapTotal;
+
+  // Priority 1: Memory pressure — always reduce, overrides everything
+  if (memPressure > MEMORY_PRESSURE_THRESHOLD) {
+    const target = clamp(current - 4);
+    _prevScanned = stats.scanned || 0;
+    _prevTimeouts = (stats.errorsByType && stats.errorsByType.static_timeout) || 0;
+    return { target, reason: `memory_pressure (${(memPressure * 100).toFixed(0)}%)` };
+  }
+
+  // Compute timeout rate from stats deltas (sliding window between adjustments)
+  const scannedNow = stats.scanned || 0;
+  const timeoutsNow = (stats.errorsByType && stats.errorsByType.static_timeout) || 0;
+  const scannedDelta = scannedNow - _prevScanned;
+  const timeoutDelta = timeoutsNow - _prevTimeouts;
+  _prevScanned = scannedNow;
+  _prevTimeouts = timeoutsNow;
+
+  const timeoutRate = scannedDelta >= TIMEOUT_RATE_MIN_SAMPLES ? timeoutDelta / scannedDelta : 0;
+
+  // Priority 2: High timeout rate — system saturated, adding workers makes it worse
+  if (timeoutRate > TIMEOUT_RATE_THRESHOLD) {
+    const target = clamp(current - 2);
+    return { target, reason: `high_timeout_rate (${(timeoutRate * 100).toFixed(0)}%, ${timeoutDelta}/${scannedDelta})` };
+  }
+
+  // Priority 3: Queue depth — scale up for backlog, down toward base when idle
+  if (queueDepth > QUEUE_BACKLOG_THRESHOLD) {
+    const target = clamp(current + 4);
+    return { target, reason: `backlog (queue=${queueDepth})` };
+  }
+
+  if (queueDepth < QUEUE_IDLE_THRESHOLD) {
+    // Converge toward BASE, not MIN — normal traffic needs BASE capacity
+    const target = Math.max(BASE_CONCURRENCY, clamp(current - 2));
+    return { target, reason: `idle (queue=${queueDepth})` };
+  }
+
+  return { target: current, reason: 'stable' };
+}
+
+function clamp(n) {
+  return Math.max(MIN_CONCURRENCY, Math.min(MAX_CONCURRENCY, n));
+}
+
+/**
+ * Reset delta tracking (e.g. after daily stats reset).
+ */
+function resetDeltas() {
+  _prevScanned = 0;
+  _prevTimeouts = 0;
+}
+
+module.exports = {
+  MIN_CONCURRENCY,
+  BASE_CONCURRENCY,
+  MAX_CONCURRENCY,
+  ADJUST_INTERVAL_MS,
+  computeTarget,
+  resetDeltas,
+  clamp
+};

package/src/monitor/daemon.js

@@ -4,11 +4,12 @@ const path = require('path');
 const os = require('os');
 const { isDockerAvailable, SANDBOX_CONCURRENCY_MAX } = require('../sandbox/index.js');
 const { setVerboseMode, isSandboxEnabled, isCanaryEnabled, isLlmDetectiveEnabled, getLlmDetectiveMode, DOWNLOADS_CACHE_TTL } = require('./classify.js');
-const { loadState, saveState, loadDailyStats, saveDailyStats, purgeTarballCache, getParisHour, atomicWriteFileSync } = require('./state.js');
+const { loadState, saveState, loadDailyStats, saveDailyStats, purgeTarballCache, getParisHour, atomicWriteFileSync, saveNpmSeq } = require('./state.js');
 const { isTemporalEnabled, isTemporalAstEnabled, isTemporalPublishEnabled, isTemporalMaintainerEnabled } = require('./temporal.js');
 const { pendingGrouped, flushScopeGroup, sendDailyReport, DAILY_REPORT_HOUR, alertedPackageRules } = require('./webhook.js');
 const { poll } = require('./ingestion.js');
-const { processQueue, SCAN_CONCURRENCY } = require('./queue.js');
+const { processQueue, ensureWorkers, drainWorkers, getTargetConcurrency, setTargetConcurrency, getActiveWorkers, SCAN_CONCURRENCY } = require('./queue.js');
+const { computeTarget, ADJUST_INTERVAL_MS, BASE_CONCURRENCY, resetDeltas } = require('./adaptive-concurrency.js');
 const { startHealthcheck } = require('./healthcheck.js');
 const { startDeferredWorker, stopDeferredWorker, persistDeferredQueue, restoreDeferredQueue } = require('./deferred-sandbox.js');
 
@@ -18,8 +19,11 @@ const QUEUE_WARNING_THRESHOLD = 5_000;  // Warn if queue depth exceeds this
 const QUEUE_PERSIST_INTERVAL = 60_000;  // Persist queue to disk every 60s
 const QUEUE_STATE_FILE = path.join(__dirname, '..', '..', 'data', 'queue-state.json');
 const QUEUE_STATE_MAX_AGE_MS = 24 * 60 * 60 * 1000; // 24h expiry
-const MAX_QUEUE_PERSIST_SIZE = 100_000; // Don't persist if queue > 100K items
-const MAX_SCAN_QUEUE = 10_000;          // Backpressure: skip polling when queue exceeds this
+const MAX_QUEUE_PERSIST_SIZE = 200_000; // Don't persist if queue > 200K items (OOM guard)
+const MAX_RESTORE_QUEUE_SIZE = 100_000; // Cap restored queue at 100K items
+// MAX_SCAN_QUEUE removed: backpressure no longer skips polling.
+// Queue grows unbounded in memory (entries are ~300B, 100K = 30MB on 12GB VPS).
+// Adaptive concurrency adjusts processing speed to match ingestion rate.
 
 function sleep(ms) {
   return new Promise((resolve) => setTimeout(resolve, ms));
@@ -88,11 +92,11 @@ function restoreQueue(scanQueue) {
       return 0;
     }
 
-    // Restore items (cap at MAX_SCAN_QUEUE to prevent OOM from stale persisted queues)
+    // Restore items (cap at MAX_RESTORE_QUEUE_SIZE to prevent OOM from stale persisted queues)
     let items = data.items;
-    if (items.length > MAX_SCAN_QUEUE) {
-      console.log(`[MONITOR] Truncating restored queue from ${items.length} to ${MAX_SCAN_QUEUE} items`);
-      items = items.slice(0, MAX_SCAN_QUEUE);
+    if (items.length > MAX_RESTORE_QUEUE_SIZE) {
+      console.log(`[MONITOR] Truncating restored queue from ${items.length} to ${MAX_RESTORE_QUEUE_SIZE} items`);
+      items = items.slice(0, MAX_RESTORE_QUEUE_SIZE);
     }
     const count = items.length;
     if (count === 0) {
@@ -404,13 +408,14 @@ async function startMonitor(options, stats, dailyAlerts, recentlyScanned, downlo
   loadDailyStats(stats, dailyAlerts); // Restore counters from previous run (survives restarts)
   console.log(`[MONITOR] State loaded — npm last: ${state.npmLastPackage || 'none'}, pypi last: ${state.pypiLastPackage || 'none'}, npm seq: ${state.npmLastSeq || 'none'}`);
   console.log('[MONITOR] npm changes stream enabled (replicate.npmjs.com) with RSS fallback');
-  console.log(`[MONITOR] Scan concurrency: ${SCAN_CONCURRENCY} (MUADDIB_SCAN_CONCURRENCY to override)`);
+  console.log(`[MONITOR] Scan concurrency: adaptive ${BASE_CONCURRENCY}→${getTargetConcurrency()} (base MUADDIB_SCAN_CONCURRENCY=${BASE_CONCURRENCY}, max MUADDIB_MAX_CONCURRENCY)`);
   console.log(`[MONITOR] Sandbox concurrency: ${SANDBOX_CONCURRENCY_MAX} (MUADDIB_SANDBOX_CONCURRENCY to override)`);
   console.log(`[MONITOR] Polling every ${POLL_INTERVAL / 1000}s (decoupled from processing). Ctrl+C to stop.\n`);
 
   let running = true;
   let pollIntervalHandle = null;   // Decoupled poll timer — set after initial poll
   let queuePersistHandle = null;   // Queue persistence timer
+  let concurrencyAdjustHandle = null; // Adaptive concurrency timer
 
   // Restore queue from previous run (if file exists and is < 24h old)
   const restoredCount = restoreQueue(scanQueue);
@@ -438,6 +443,13 @@ async function startMonitor(options, stats, dailyAlerts, recentlyScanned, downlo
       clearInterval(queuePersistHandle);
       queuePersistHandle = null;
     }
+    if (concurrencyAdjustHandle) {
+      clearInterval(concurrencyAdjustHandle);
+      concurrencyAdjustHandle = null;
+    }
+    // Wait for in-flight scans to complete (soft drain)
+    console.log(`[MONITOR] Draining ${getActiveWorkers()} active worker(s)...`);
+    await drainWorkers();
     // Persist remaining queue items so they survive the restart
     persistQueue(scanQueue, state);
     // Stop deferred sandbox worker and persist its queue
@@ -470,26 +482,28 @@ async function startMonitor(options, stats, dailyAlerts, recentlyScanned, downlo
 
   // Initial poll + scan (sequential for first run)
   await poll(state, scanQueue, stats);
+  // Atomicity fix: persist queue AND seq together after each poll.
+  // Previously, seq was saved inside pollNpmChanges() but queue persisted
+  // every 60s ��� crash between the two lost queued items permanently.
+  persistQueue(scanQueue, state);
+  saveNpmSeq(state.npmLastSeq);
   saveState(state, stats);
   await processQueue(scanQueue, stats, dailyAlerts, recentlyScanned, downloadsCache, sandboxAvailableRef.value);
 
   // ─── Decoupled polling ───
   // Poll runs on its own interval, independent of processing.
   // This ensures new packages are ingested even while a large batch is being scanned.
-  // Without this, a 2h processing batch blocks all polling — packages published and
-  // removed during that window are never seen (e.g. axios/plain-crypto-js 2026-03-30).
+  // Backpressure removed: polling ALWAYS runs. Queue grows unbounded in memory
+  // (entries ~300B, 100K = 30MB). Adaptive concurrency adjusts scan throughput.
   let pollInProgress = false;
   pollIntervalHandle = setInterval(async () => {
     if (!running || pollInProgress) return;
-    // Backpressure: skip poll when queue is too deep.
-    // CouchDB seq is NOT advanced — next poll resumes from the same point. No packages lost.
-    if (scanQueue.length >= MAX_SCAN_QUEUE) {
-      console.log(`[MONITOR] BACKPRESSURE: skipping poll (queue ${scanQueue.length} >= ${MAX_SCAN_QUEUE})`);
-      return;
-    }
     pollInProgress = true;
     try {
       await poll(state, scanQueue, stats);
+      // Atomicity: persist queue + seq together after each poll
+      persistQueue(scanQueue, state);
+      saveNpmSeq(state.npmLastSeq);
       saveState(state, stats);
       if (scanQueue.length > QUEUE_WARNING_THRESHOLD) {
         console.log(`[MONITOR] WARNING: scan queue depth ${scanQueue.length} — processing may be lagging behind ingestion`);
@@ -502,27 +516,41 @@ async function startMonitor(options, stats, dailyAlerts, recentlyScanned, downlo
   }, POLL_INTERVAL);
 
   // ─── Queue persistence ───
-  // Snapshot queue to disk every 60s so items survive restarts/crashes.
-  // Without this, the decoupled poll advances the CouchDB seq but queued
-  // items are lost on restart — they won't be re-polled.
+  // Periodic snapshot as safety net (in addition to post-poll persist).
   queuePersistHandle = setInterval(() => {
     if (!running) return;
     persistQueue(scanQueue, state);
     persistDeferredQueue(); // Piggyback: persist deferred sandbox queue on same interval
   }, QUEUE_PERSIST_INTERVAL);
 
+  // ─── Adaptive concurrency ───
+  // Adjusts scan worker count every 30s based on queue depth, memory, timeout rate.
+  // Scale-up is aggressive (+4) during backlog, scale-down is gradual (-2) when idle.
+  concurrencyAdjustHandle = setInterval(() => {
+    if (!running) return;
+    const current = getTargetConcurrency();
+    const { target, reason } = computeTarget(current, scanQueue.length, stats);
+    if (target !== current) {
+      console.log(`[MONITOR] ADAPTIVE: concurrency ${current} → ${target} (${reason}, active=${getActiveWorkers()})`);
+      setTargetConcurrency(target);
+      // Immediately spawn new workers if scaling up (don't wait for next loop tick)
+      if (target > current) {
+        ensureWorkers(scanQueue, stats, dailyAlerts, recentlyScanned, downloadsCache, sandboxAvailableRef.value);
+      }
+    }
+  }, ADJUST_INTERVAL_MS);
+
   // ─── Continuous processing loop ───
-  // Consumes scanQueue independently of polling. Workers inside processQueue
-  // check scanQueue.length > 0 after each item, so items added by a concurrent
-  // poll are picked up immediately by running workers.
+  // Non-blocking: ensureWorkers spawns fire-and-forget background workers.
+  // This loop tops up workers every 2s AND runs housekeeping (memory, daily report)
+  // without being blocked by long-running scans.
   const MEMORY_LOG_INTERVAL = 300_000; // 5 minutes
   const MEMORY_PRESSURE_THRESHOLD = 0.85; // 85% heap usage triggers emergency prune
   let lastMemoryLogTime = Date.now();
 
   while (running) {
-    if (scanQueue.length > 0) {
-      await processQueue(scanQueue, stats, dailyAlerts, recentlyScanned, downloadsCache, sandboxAvailableRef.value);
-    }
+    // Top up workers (non-blocking — spawns missing workers as background promises)
+    ensureWorkers(scanQueue, stats, dailyAlerts, recentlyScanned, downloadsCache, sandboxAvailableRef.value);
 
     // ─── Memory watchdog (every 5 min) ───
     if (Date.now() - lastMemoryLogTime >= MEMORY_LOG_INTERVAL) {
@@ -596,7 +624,7 @@ module.exports = {
   QUEUE_STATE_FILE,
   QUEUE_STATE_MAX_AGE_MS,
   MAX_QUEUE_PERSIST_SIZE,
-  MAX_SCAN_QUEUE,
+  MAX_RESTORE_QUEUE_SIZE,
   pruneMemoryCaches,
   MAX_RECENTLY_SCANNED,
   MAX_ALERTED_PACKAGES

package/src/monitor/ingestion.js

@@ -455,10 +455,10 @@ async function pollNpmChanges(state, scanQueue, stats) {
       queued++;
     }
 
-    // Persist new seq
+    // Update seq in memory only — disk persistence is handled by daemon.js
+    // after both queue and seq are saved atomically (prevents data loss on crash).
     if (data.last_seq != null) {
       state.npmLastSeq = data.last_seq;
-      saveNpmSeq(data.last_seq);
     }
 
     if (queued > 0) {
@@ -644,12 +644,12 @@ async function pollPyPI(state, scanQueue) {
  * @param {Object} stats - Mutable stats object
  */
 async function poll(state, scanQueue, stats) {
-  // Backpressure: skip ingestion when queue is saturated.
-  // CouchDB seq and PyPI lastPackage are NOT advanced — next poll resumes from same point.
-  const MAX_SCAN_QUEUE = 10_000;
-  if (scanQueue.length >= MAX_SCAN_QUEUE) {
-    console.log(`[MONITOR] BACKPRESSURE: skipping poll (queue ${scanQueue.length} >= ${MAX_SCAN_QUEUE})`);
-    return;
+  // Backpressure removed: polling ALWAYS runs regardless of queue depth.
+  // The queue can grow unbounded in memory (entries are ~300 bytes, 100K = 30MB).
+  // This prevents the data loss scenario where the CouchDB seq advances but
+  // queued items are not persisted — packages would be permanently invisible.
+  if (scanQueue.length > 5_000) {
+    console.log(`[MONITOR] QUEUE_DEPTH: ${scanQueue.length} items — polling continues (no backpressure skip)`);
   }
 
   const timestamp = new Date().toISOString().slice(0, 19).replace('T', ' ');

package/src/monitor/queue.js

@@ -106,9 +106,19 @@ const { archiveSuspectTarball } = require('./tarball-archive.js');
 // From ./deferred-sandbox.js
 const { enqueueDeferred } = require('./deferred-sandbox.js');
 
-// --- Constants ---
+// --- Adaptive concurrency ---
 
-const SCAN_CONCURRENCY = Math.max(1, parseInt(process.env.MUADDIB_SCAN_CONCURRENCY, 10) || 8);
+const { BASE_CONCURRENCY, MIN_CONCURRENCY, MAX_CONCURRENCY } = require('./adaptive-concurrency.js');
+
+// SCAN_CONCURRENCY kept as getter for backward compatibility (tests, logging)
+let _targetConcurrency = BASE_CONCURRENCY;
+const SCAN_CONCURRENCY = BASE_CONCURRENCY; // legacy export — tests check this value
+let _activeWorkers = 0;
+const _workerPromises = new Set();
+
+function getTargetConcurrency() { return _targetConcurrency; }
+function setTargetConcurrency(n) { _targetConcurrency = Math.max(MIN_CONCURRENCY, Math.min(MAX_CONCURRENCY, n)); }
+function getActiveWorkers() { return _activeWorkers; }
 const SCAN_TIMEOUT_MS = 300_000; // 5 minutes per package (3 sandbox runs × 90s + static scan headroom)
 const STATIC_SCAN_TIMEOUT_MS = 45_000; // 45s for static analysis only
 const LARGE_PACKAGE_SIZE = 10 * 1024 * 1024; // 10MB
@@ -967,30 +977,65 @@ async function processQueueItem(item, stats, dailyAlerts, recentlyScanned, downl
 }
 
 /**
- * Worker-pool consumer for the scan queue.
- * Runs up to SCAN_CONCURRENCY scans in parallel. Each worker pulls from the
- * shared scanQueue until it's empty. Node.js is single-threaded so
- * scanQueue.shift() is atomic — no race conditions between workers.
+ * Spawn a single worker that pulls from scanQueue until:
+ *   - queue is empty, OR
+ *   - activeWorkers exceeds targetConcurrency (soft drain on scale-down)
+ *
+ * Workers are fire-and-forget: they run as background promises tracked
+ * in _workerPromises. Node.js is single-threaded so scanQueue.shift()
+ * is atomic — no race conditions between workers.
  */
-async function processQueue(scanQueue, stats, dailyAlerts, recentlyScanned, downloadsCache, sandboxAvailable) {
-  if (scanQueue.length === 0) return;
-
-  if (SCAN_CONCURRENCY > 1 && scanQueue.length > 1) {
-    console.log(`[MONITOR] Processing ${scanQueue.length} queued packages (concurrency: ${SCAN_CONCURRENCY})`);
-  }
-
-  async function worker() {
-    while (scanQueue.length > 0) {
+async function _spawnWorker(scanQueue, stats, dailyAlerts, recentlyScanned, downloadsCache, sandboxAvailable) {
+  _activeWorkers++;
+  try {
+    while (scanQueue.length > 0 && _activeWorkers <= _targetConcurrency) {
       const item = scanQueue.shift();
+      if (!item) break;
       await processQueueItem(item, stats, dailyAlerts, recentlyScanned, downloadsCache, scanQueue, sandboxAvailable);
     }
+  } finally {
+    _activeWorkers--;
   }
+}
 
-  const workers = [];
-  for (let i = 0; i < Math.min(SCAN_CONCURRENCY, scanQueue.length); i++) {
-    workers.push(worker());
+/**
+ * Ensure the target number of workers are running. Non-blocking: spawns
+ * missing workers as background promises. Called from the daemon main loop
+ * every PROCESS_LOOP_INTERVAL (2s), and after concurrency adjustments.
+ */
+function ensureWorkers(scanQueue, stats, dailyAlerts, recentlyScanned, downloadsCache, sandboxAvailable) {
+  if (scanQueue.length === 0) return;
+  const toSpawn = Math.min(_targetConcurrency - _activeWorkers, scanQueue.length);
+  if (toSpawn <= 0) return;
+
+  console.log(`[MONITOR] Spawning ${toSpawn} worker(s) (active: ${_activeWorkers}, target: ${_targetConcurrency}, queue: ${scanQueue.length})`);
+  for (let i = 0; i < toSpawn; i++) {
+    const p = _spawnWorker(scanQueue, stats, dailyAlerts, recentlyScanned, downloadsCache, sandboxAvailable)
+      .catch(err => console.error('[MONITOR] Worker error:', err.message))
+      .finally(() => _workerPromises.delete(p));
+    _workerPromises.add(p);
   }
-  await Promise.all(workers);
+}
+
+/**
+ * Wait for all active workers to finish. Used for:
+ *   - Graceful shutdown (drain in-flight scans)
+ *   - Tests (backward-compatible await)
+ */
+async function drainWorkers() {
+  if (_workerPromises.size === 0) return;
+  await Promise.all(_workerPromises);
+}
+
+/**
+ * Backward-compatible processQueue: ensure workers + await completion.
+ * Used by tests and the initial sequential scan at startup.
+ * The daemon main loop uses ensureWorkers() directly (non-blocking).
+ */
+async function processQueue(scanQueue, stats, dailyAlerts, recentlyScanned, downloadsCache, sandboxAvailable) {
+  if (scanQueue.length === 0) return;
+  ensureWorkers(scanQueue, stats, dailyAlerts, recentlyScanned, downloadsCache, sandboxAvailable);
+  await drainWorkers();
 }
 
 /**
@@ -1195,6 +1240,13 @@ module.exports = {
   TEST_FILE_PATTERN,
   SCAN_WORKER_PATH,
 
+  // Adaptive concurrency
+  getTargetConcurrency,
+  setTargetConcurrency,
+  getActiveWorkers,
+  ensureWorkers,
+  drainWorkers,
+
   // Functions
   isBundledToolingOnly,
   recordTrainingSample,