muaddib-scanner 2.10.80 → 2.10.82

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "muaddib-scanner",
-  "version": "2.10.80",
+  "version": "2.10.82",
   "description": "Supply-chain threat detection & response for npm & PyPI/Python",
   "main": "src/index.js",
   "bin": {

package/src/monitor/adaptive-concurrency.js

@@ -17,7 +17,7 @@ const os = require('os');
 
 const MIN_CONCURRENCY = 4;
 const BASE_CONCURRENCY = Math.max(MIN_CONCURRENCY, parseInt(process.env.MUADDIB_SCAN_CONCURRENCY, 10) || 8);
-const MAX_CONCURRENCY = Math.max(BASE_CONCURRENCY, parseInt(process.env.MUADDIB_MAX_CONCURRENCY, 10) || 32);
+const MAX_CONCURRENCY = Math.max(BASE_CONCURRENCY, parseInt(process.env.MUADDIB_MAX_CONCURRENCY, 10) || 16);
 const ADJUST_INTERVAL_MS = 30_000;
 
 // Queue depth thresholds
@@ -37,6 +37,12 @@ const TIMEOUT_RATE_MIN_SAMPLES = 20;
 let _prevScanned = 0;
 let _prevTimeouts = 0;
 
+// Throughput plateau detection: if we scaled up but throughput didn't increase,
+// we've hit I/O saturation (npm registry rate limiting, disk contention).
+// More workers would make it worse — scale back instead.
+let _prevThroughput = 0;
+let _lastScaleDirection = 0; // +1 = scaled up, -1 = scaled down, 0 = stable
+
 /**
  * Compute new target concurrency from system signals.
  * Uses stats deltas (not cumulative) for timeout rate — avoids stale data.
@@ -74,21 +80,39 @@ function computeTarget(current, queueDepth, stats) {
   // Priority 2: High timeout rate — system saturated, adding workers makes it worse
   if (timeoutRate > TIMEOUT_RATE_THRESHOLD) {
     const target = clamp(current - 2);
+    _prevThroughput = scannedDelta;
+    _lastScaleDirection = target < current ? -1 : 0;
     return { target, reason: `high_timeout_rate (${(timeoutRate * 100).toFixed(0)}%, ${timeoutDelta}/${scannedDelta})` };
   }
 
-  // Priority 3: Queue depth — scale up for backlog, down toward base when idle
+  // Priority 3: Throughput plateau — scaled up last tick but throughput flat/down.
+  // This catches I/O saturation: more workers = more concurrent HTTP to npm registry
+  // = rate limiting + contention = scan times 10s→90s = throughput drops.
+  // Scale back instead of continuing to add workers.
+  if (_lastScaleDirection > 0 && _prevThroughput > 0 && scannedDelta > 0 && scannedDelta <= _prevThroughput) {
+    const prevTp = _prevThroughput;
+    _prevThroughput = scannedDelta;
+    _lastScaleDirection = -1;
+    return { target: clamp(current - 2), reason: `throughput_plateau (${prevTp}→${scannedDelta} scans/30s, more workers didn't help)` };
+  }
+
+  // Priority 4: Queue depth — scale up for backlog, down toward base when idle
   if (queueDepth > QUEUE_BACKLOG_THRESHOLD) {
     const target = clamp(current + 4);
+    // Record throughput at the point of scale-up — next tick compares against this
+    _prevThroughput = scannedDelta;
+    _lastScaleDirection = target > current ? 1 : 0;
     return { target, reason: `backlog (queue=${queueDepth})` };
   }
 
   if (queueDepth < QUEUE_IDLE_THRESHOLD) {
     // Converge toward BASE, not MIN — normal traffic needs BASE capacity
     const target = Math.max(BASE_CONCURRENCY, clamp(current - 2));
+    _lastScaleDirection = target < current ? -1 : 0;
     return { target, reason: `idle (queue=${queueDepth})` };
   }
 
+  _lastScaleDirection = 0;
   return { target: current, reason: 'stable' };
 }
 
@@ -102,6 +126,8 @@ function clamp(n) {
 function resetDeltas() {
   _prevScanned = 0;
   _prevTimeouts = 0;
+  _prevThroughput = 0;
+  _lastScaleDirection = 0;
 }
 
 module.exports = {

package/src/monitor/daemon.js

@@ -480,15 +480,37 @@ async function startMonitor(options, stats, dailyAlerts, recentlyScanned, downlo
     console.log('[MONITOR] Deferred sandbox worker started (30s interval, dedicated slot)');
   }
 
-  // Initial poll + scan (sequential for first run)
+  // ─── Initial poll ───
+  // Fills the queue with pending packages. Processing starts in the main loop
+  // via ensureWorkers (non-blocking) — NOT await processQueue (blocking).
+  // A blocking processQueue here would prevent adaptive concurrency from
+  // firing until the entire initial batch is drained at BASE_CONCURRENCY.
   await poll(state, scanQueue, stats);
   // Atomicity fix: persist queue AND seq together after each poll.
   // Previously, seq was saved inside pollNpmChanges() but queue persisted
-  // every 60s ��� crash between the two lost queued items permanently.
+  // every 60s — crash between the two lost queued items permanently.
   persistQueue(scanQueue, state);
   saveNpmSeq(state.npmLastSeq);
   saveState(state, stats);
-  await processQueue(scanQueue, stats, dailyAlerts, recentlyScanned, downloadsCache, sandboxAvailableRef.value);
+  console.log(`[MONITOR] Initial poll complete — ${scanQueue.length} packages queued for processing`);
+
+  // ─── Adaptive concurrency ───
+  // Set up BEFORE the main loop so it fires during the initial batch.
+  // Adjusts scan worker count every 30s based on queue depth, memory, timeout rate.
+  // Scale-up is aggressive (+4) during backlog, scale-down is gradual (-2) when idle.
+  concurrencyAdjustHandle = setInterval(() => {
+    if (!running) return;
+    const current = getTargetConcurrency();
+    const { target, reason } = computeTarget(current, scanQueue.length, stats);
+    if (target !== current) {
+      console.log(`[MONITOR] ADAPTIVE: concurrency ${current} → ${target} (${reason}, active=${getActiveWorkers()})`);
+      setTargetConcurrency(target);
+      // Immediately spawn new workers if scaling up (don't wait for next loop tick)
+      if (target > current) {
+        ensureWorkers(scanQueue, stats, dailyAlerts, recentlyScanned, downloadsCache, sandboxAvailableRef.value);
+      }
+    }
+  }, ADJUST_INTERVAL_MS);
 
   // ─── Decoupled polling ───
   // Poll runs on its own interval, independent of processing.
@@ -523,23 +545,6 @@ async function startMonitor(options, stats, dailyAlerts, recentlyScanned, downlo
     persistDeferredQueue(); // Piggyback: persist deferred sandbox queue on same interval
   }, QUEUE_PERSIST_INTERVAL);
 
-  // ─── Adaptive concurrency ───
-  // Adjusts scan worker count every 30s based on queue depth, memory, timeout rate.
-  // Scale-up is aggressive (+4) during backlog, scale-down is gradual (-2) when idle.
-  concurrencyAdjustHandle = setInterval(() => {
-    if (!running) return;
-    const current = getTargetConcurrency();
-    const { target, reason } = computeTarget(current, scanQueue.length, stats);
-    if (target !== current) {
-      console.log(`[MONITOR] ADAPTIVE: concurrency ${current} → ${target} (${reason}, active=${getActiveWorkers()})`);
-      setTargetConcurrency(target);
-      // Immediately spawn new workers if scaling up (don't wait for next loop tick)
-      if (target > current) {
-        ensureWorkers(scanQueue, stats, dailyAlerts, recentlyScanned, downloadsCache, sandboxAvailableRef.value);
-      }
-    }
-  }, ADJUST_INTERVAL_MS);
-
   // ─── Continuous processing loop ───
   // Non-blocking: ensureWorkers spawns fire-and-forget background workers.
   // This loop tops up workers every 2s AND runs housekeeping (memory, daily report)

package/src/monitor/ingestion.js

@@ -643,13 +643,20 @@ async function pollPyPI(state, scanQueue) {
  * @param {Array} scanQueue - Mutable scan queue array
  * @param {Object} stats - Mutable stats object
  */
+const SOFT_BACKPRESSURE_THRESHOLD = 10_000;
+
 async function poll(state, scanQueue, stats) {
-  // Backpressure removed: polling ALWAYS runs regardless of queue depth.
-  // The queue can grow unbounded in memory (entries are ~300 bytes, 100K = 30MB).
-  // This prevents the data loss scenario where the CouchDB seq advances but
-  // queued items are not persisted — packages would be permanently invisible.
+  // Soft backpressure: skip poll when queue is very deep.
+  // Safe because: CouchDB seq is NOT advanced (stays in memory only, persisted
+  // by daemon.js AFTER poll returns) — next poll resumes from the same point.
+  // Combined with adaptive concurrency: workers scale up → queue drains → poll resumes.
+  // This prevents the queue from growing to 30-40K during catch-up (OOM risk).
+  if (scanQueue.length >= SOFT_BACKPRESSURE_THRESHOLD) {
+    console.log(`[MONITOR] BACKPRESSURE: skipping poll (queue ${scanQueue.length} >= ${SOFT_BACKPRESSURE_THRESHOLD}) — seq not advanced, 0 packages lost`);
+    return;
+  }
   if (scanQueue.length > 5_000) {
-    console.log(`[MONITOR] QUEUE_DEPTH: ${scanQueue.length} items — polling continues (no backpressure skip)`);
+    console.log(`[MONITOR] QUEUE_DEPTH: ${scanQueue.length} items — polling continues`);
   }
 
   const timestamp = new Date().toISOString().slice(0, 19).replace('T', ' ');

package/src/shared/download.js

@@ -171,6 +171,12 @@ function downloadToFile(url, destPath, timeoutMs = DOWNLOAD_TIMEOUT) {
             }
             return doRequest(absoluteLocation, redirectCount + 1);
           }
+          if (res.statusCode === 429) {
+            res.resume();
+            // Signal rate limiter to back off — drains tokens, forces ~1s pause
+            try { require('./http-limiter.js').signal429(); } catch {}
+            return reject(new Error(`HTTP 429 rate limited for ${requestUrl}`));
+          }
           if (res.statusCode < 200 || res.statusCode >= 300) {
             res.resume();
             return reject(new Error(`HTTP ${res.statusCode} for ${requestUrl}`));

package/src/shared/http-limiter.js

@@ -1,29 +1,35 @@
 'use strict';
 
 /**
- * Centralized HTTP concurrency limiter for npm registry requests.
+ * Centralized HTTP concurrency + rate limiter for npm registry requests.
  *
- * With 16 monitor workers × 7+ HTTP requests/package, uncapped concurrency
- * reaches 112+ simultaneous requests — well above npm's implicit rate limit.
- * This module caps ALL registry.npmjs.org requests to a single semaphore
- * so that no more than REGISTRY_SEMAPHORE_MAX requests are in-flight at once.
+ * Two layers of protection:
+ *   1. Concurrency semaphore (REGISTRY_SEMAPHORE_MAX = 10) — caps in-flight requests
+ *   2. Rate limiter (RATE_LIMIT_PER_SEC = 30) — caps requests/second via token bucket
  *
- * Consumers: temporal-analysis.js, temporal-ast-diff.js, monitor.js (getNpmLatestTarball),
- *            npm-registry.js (fetchWithRetry to registry.npmjs.org).
+ * Without rate limiting, 10 concurrent slots × fast-completing requests = 100+ req/s
+ * bursts that trigger npm 429 responses → exponential backoff → scan times 10s→90s.
+ *
+ * Consumers: queue.js (downloadToFile), temporal-analysis.js, npm-registry.js.
  * NOT covered: api.npmjs.org (different server), replicate.npmjs.com (CouchDB changes stream).
  */
 
 const REGISTRY_SEMAPHORE_MAX = 10;
+const RATE_LIMIT_PER_SEC = 30;
+
+// --- Concurrency semaphore ---
 
 const _semaphore = { active: 0, queue: [] };
 
 function acquireRegistrySlot() {
   if (_semaphore.active < REGISTRY_SEMAPHORE_MAX) {
     _semaphore.active++;
-    return Promise.resolve();
+    return _acquireRateToken();
   }
   return new Promise(resolve => {
-    _semaphore.queue.push(resolve);
+    _semaphore.queue.push(() => {
+      _acquireRateToken().then(resolve);
+    });
   });
 }
 
@@ -36,9 +42,64 @@ function releaseRegistrySlot() {
   }
 }
 
+// --- Token bucket rate limiter ---
+// Refills RATE_LIMIT_PER_SEC tokens per second. Each request consumes 1 token.
+// If no tokens available, waits until the next refill.
+
+let _tokens = RATE_LIMIT_PER_SEC;
+let _lastRefill = Date.now();
+
+function _refillTokens() {
+  const now = Date.now();
+  const elapsed = now - _lastRefill;
+  if (elapsed >= 1000) {
+    _tokens = Math.min(RATE_LIMIT_PER_SEC, _tokens + Math.floor(elapsed / 1000) * RATE_LIMIT_PER_SEC);
+    _lastRefill = now;
+  }
+}
+
+function _acquireRateToken() {
+  _refillTokens();
+  if (_tokens > 0) {
+    _tokens--;
+    return Promise.resolve();
+  }
+  // Wait until next refill
+  const waitMs = 1000 - (Date.now() - _lastRefill);
+  return new Promise(resolve => {
+    setTimeout(() => {
+      _refillTokens();
+      _tokens = Math.max(0, _tokens - 1);
+      resolve();
+    }, Math.max(10, waitMs));
+  });
+}
+
+// --- 429 backoff helper ---
+// Call this when a 429 response is received. Drains all tokens to force
+// a ~1s pause on subsequent requests (token bucket naturally refills).
+
+let _backoffCount = 0;
+
+function signal429() {
+  _tokens = 0;
+  _lastRefill = Date.now() + 1000; // Force 1s pause
+  _backoffCount++;
+  if (_backoffCount % 10 === 1) {
+    console.warn(`[HTTP-LIMITER] 429 rate limited by npm registry (total: ${_backoffCount})`);
+  }
+}
+
+function getBackoffCount() {
+  return _backoffCount;
+}
+
 function resetLimiter() {
   _semaphore.active = 0;
   _semaphore.queue.length = 0;
+  _tokens = RATE_LIMIT_PER_SEC;
+  _lastRefill = Date.now();
+  _backoffCount = 0;
 }
 
 function getActiveSemaphore() {
@@ -47,8 +108,11 @@ function getActiveSemaphore() {
 
 module.exports = {
   REGISTRY_SEMAPHORE_MAX,
+  RATE_LIMIT_PER_SEC,
   acquireRegistrySlot,
   releaseRegistrySlot,
+  signal429,
+  getBackoffCount,
   resetLimiter,
   getActiveSemaphore
 };