muaddib-scanner 2.11.124 → 2.11.125

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "muaddib-scanner",
-  "version": "2.11.124",
+  "version": "2.11.125",
   "description": "Supply-chain threat detection & response for npm & PyPI/Python",
   "main": "src/index.js",
   "bin": {

package/{self-scan-v2.11.124.json → self-scan-v2.11.125.json}

@@ -1,6 +1,6 @@
 {
   "target": "node_modules",
-  "timestamp": "2026-06-19T13:30:20.182Z",
+  "timestamp": "2026-06-19T14:00:25.646Z",
   "threats": [
     {
       "type": "string_mutation_obfuscation",

package/src/monitor/ingestion.js

@@ -115,7 +115,7 @@ function httpsGet(url, timeoutMs = 30_000, deadlineMs = Math.max(timeoutMs * 2,
         // Coordinated backoff: drain the SHARED token bucket so every in-flight registry fetch
         // slows together. This high-volume packument/changes path must signal 429 like the
         // metadata path (npm-registry.js) does — not just acquire a slot (CLAUDE.md 429 storm).
-        try { require('../shared/http-limiter.js').signal429(); } catch { /* limiter best-effort */ }
+        try { const _l = require('../shared/http-limiter.js'); _l.signal429(_l.hostForUrl(url)); } catch { /* limiter best-effort */ }
         return done(new Error(`HTTP 429 rate limited for ${url}`));
       }
       if (res.statusCode < 200 || res.statusCode >= 300) {
@@ -177,7 +177,7 @@ function httpsPost(url, body, headers = {}, timeoutMs = 30_000, deadlineMs = Mat
     req = _deps.https.request(options, (res) => {
       if (res.statusCode === 429) {
         res.resume();
-        try { require('../shared/http-limiter.js').signal429(); } catch { /* limiter best-effort */ }
+        try { const _l = require('../shared/http-limiter.js'); _l.signal429(_l.hostForUrl(url)); } catch { /* limiter best-effort */ }
         return done(new Error(`HTTP 429 rate limited for POST ${url}`));
       }
       if (res.statusCode < 200 || res.statusCode >= 300) {

package/src/monitor/queue.js

@@ -84,7 +84,7 @@ const {
 
 // From ./ingestion.js
 const { getNpmLatestTarball, getPyPITarballUrl } = require('./ingestion.js');
-const { enqueueScan, dequeueScan } = require('./scan-queue.js');
+const { enqueueScan, dequeueScan, shouldPullNewest } = require('./scan-queue.js');
 
 // From ./tarball-archive.js
 const { archiveSuspectTarball } = require('./tarball-archive.js');
@@ -98,6 +98,7 @@ const { BASE_CONCURRENCY, MIN_CONCURRENCY, MAX_CONCURRENCY } = require('./adapti
 
 // SCAN_CONCURRENCY kept as getter for backward compatibility (tests, logging)
 let _targetConcurrency = BASE_CONCURRENCY;
+let _dequeueSeq = 0; // shared monotonic cursor for fresh-first round-robin (Phase 2)
 const SCAN_CONCURRENCY = BASE_CONCURRENCY; // legacy export — tests check this value
 let _activeWorkers = 0;
 const _workerPromises = new Set();
@@ -1780,7 +1781,10 @@ async function _spawnWorker(scanQueue, stats, dailyAlerts, recentlyScanned, down
       // admission (1 scan when nothing is in flight) can still flow.
       if (isGovernorFrozen() && (getGovernorState().outstandingCount > 0 || _activeWorkers > 1)) break;
       // AUDIT A2: FIFO by default; priority dequeue when MUADDIB_PRIORITY_DEQUEUE=1.
-      const item = dequeueScan(scanQueue);
+      // Phase 2: when MUADDIB_FRESH_FIRST=1 and a backlog exists, most lanes take the
+      // newest item (fresh-first) while a reserved few keep draining the oldest.
+      const newest = shouldPullNewest(_dequeueSeq++, scanQueue.length, _targetConcurrency);
+      const item = dequeueScan(scanQueue, { newest });
       if (!item) break;
       _inFlightItems.add(item);
       try {

package/src/monitor/scan-queue.js

@@ -201,6 +201,26 @@ const PRIORITY_DEQUEUE_WINDOW = (() => {
   return Number.isFinite(v) && v > 0 ? v : 2048;
 })();
 
+// ── Fresh-first scheduling (throughput plan Phase 2; gated OFF by default) ───
+// When MUADDIB_FRESH_FIRST=1 AND a real backlog exists (queue > split threshold),
+// most worker lanes dequeue the NEWEST item (tail) so a freshly-published package
+// is scanned in minutes instead of aging ~hours behind a FIFO backlog, while a
+// reserved `drainLanes` keep draining the OLDEST (anti-starvation). Below the
+// threshold all lanes stay strict FIFO ("back to the normal flow" once drained).
+// Pure ordering change — every item is still scanned; inert until the flag is on.
+const FRESH_FIRST = (() => {
+  const v = process.env.MUADDIB_FRESH_FIRST;
+  return v === '1' || v === 'true';
+})();
+const BACKLOG_DRAIN_LANES = (() => {
+  const v = parseInt(process.env.MUADDIB_BACKLOG_DRAIN_LANES, 10);
+  return Number.isFinite(v) && v > 0 ? v : 2;
+})();
+const BACKLOG_SPLIT_THRESHOLD = (() => {
+  const v = parseInt(process.env.MUADDIB_BACKLOG_SPLIT_THRESHOLD, 10);
+  return Number.isFinite(v) && v > 0 ? v : 5000;
+})();
+
 function _isPriority(item) {
   return !!(item && (item.firstPublish || item.isIOCMatch || (item.isBurst && !item.isATOBurstExtra)));
 }
@@ -213,8 +233,12 @@ function _isPriority(item) {
  * @param {{priority?: boolean, window?: number}} [opts] test overrides
  */
 function dequeueScan(scanQueue, opts = {}) {
+  if (scanQueue.length === 0) return scanQueue.shift();
+  // Fresh-first lanes take the NEWEST item (tail). Takes precedence over priority:
+  // the freshest publish is the one we most want to be first on.
+  if (opts.newest) return scanQueue.pop();
   const priority = opts.priority !== undefined ? opts.priority : PRIORITY_DEQUEUE;
-  if (!priority || scanQueue.length === 0) return scanQueue.shift();
+  if (!priority) return scanQueue.shift();
   const win = Math.min(scanQueue.length, opts.window || PRIORITY_DEQUEUE_WINDOW);
   for (let i = 0; i < win; i++) {
     if (_isPriority(scanQueue[i])) return i === 0 ? scanQueue.shift() : scanQueue.splice(i, 1)[0];
@@ -222,4 +246,23 @@ function dequeueScan(scanQueue, opts = {}) {
   return scanQueue.shift();
 }
 
-module.exports = { enqueueScan, evictFromScanQueueBulk, dequeueScan, isProtected: _isProtected, MAX_SCAN_QUEUE };
+/**
+ * Fresh-first dequeue decision (Phase 2, gated by MUADDIB_FRESH_FIRST). True → this
+ * dequeue should take the NEWEST item (tail); false → oldest (FIFO/priority head).
+ * Splits only above BACKLOG_SPLIT_THRESHOLD; below it everything stays FIFO. Of
+ * `target` concurrent lanes, `drainLanes` keep pulling oldest (anti-starvation), the
+ * rest go fresh. `seq` is a shared monotonic dequeue counter (single-threaded →
+ * consistent). Pure; `opts` override the env gates for tests.
+ */
+function shouldPullNewest(seq, queueLen, target, opts = {}) {
+  const enabled = opts.enabled !== undefined ? opts.enabled : FRESH_FIRST;
+  if (!enabled) return false;
+  const threshold = opts.threshold !== undefined ? opts.threshold : BACKLOG_SPLIT_THRESHOLD;
+  if (queueLen <= threshold) return false;
+  const drainLanes = opts.drainLanes !== undefined ? opts.drainLanes : BACKLOG_DRAIN_LANES;
+  const lanes = Math.max(2, target || 2);
+  const drain = Math.min(drainLanes, lanes - 1);   // always leave ≥1 fresh lane
+  return (seq % lanes) >= drain;                    // first `drain` slots → oldest; rest → newest
+}
+
+module.exports = { enqueueScan, evictFromScanQueueBulk, dequeueScan, shouldPullNewest, isProtected: _isProtected, MAX_SCAN_QUEUE };

package/src/scanner/npm-registry.js

@@ -1,6 +1,6 @@
 const { NPM_PACKAGE_REGEX } = require('../shared/constants.js');
 const { debugLog } = require('../utils.js');
-const { acquireRegistrySlot, releaseRegistrySlot, awaitRateToken, signal429, hostForUrl } = require('../shared/http-limiter.js');
+const { acquireRegistrySlot, releaseRegistrySlot, awaitRateToken, signal429, hostForUrl, isHostBackedOff } = require('../shared/http-limiter.js');
 const { registryAuthHeaders } = require('../shared/registry-auth.js');
 const { computeAdvancedRegistrySignals } = require('../integrations/registry-signals.js');
 
@@ -223,8 +223,30 @@ async function getPackageMetadata(packageName) {
     return count;
   }
 
+  // Downloads-count (api.npmjs.org) is a SEPARATE, aggressively rate-limited host and
+  // is OFF the detection path — it only feeds the reputation/ML weekly_downloads
+  // feature. The ingest pre-resolve already fetches + 24h-caches it (classify.js
+  // downloadsCache). So prefer the warm cache; if cold AND api.npmjs.org is in 429
+  // backoff, SKIP rather than park the scan on the token gate (a level-12 60s pause
+  // was stalling every cold scan ~20s). Same graceful degradation as the author-count
+  // kill-switch: weekly_downloads stays absent, never blocks throughput.
+  let cachedDownloads;
+  try {
+    const { downloadsCache, DOWNLOADS_CACHE_TTL } = require('../monitor/classify.js');
+    const hit = downloadsCache.get(packageName);
+    if (hit && (Date.now() - hit.fetchedAt) < DOWNLOADS_CACHE_TTL && hit.downloads >= 0) {
+      cachedDownloads = hit.downloads;
+    }
+  } catch { /* classify unavailable (scanner-only context) — fall through to a guarded fetch */ }
+
+  const downloadsPromise = (typeof cachedDownloads === 'number')
+    ? Promise.resolve({ downloads: cachedDownloads })
+    : (isHostBackedOff(hostForUrl(downloadsUrl))
+        ? Promise.resolve(null)            // host hammered → skip; do not stall the scan
+        : fetchWithRetry(downloadsUrl, { noRetryOn429: true }));
+
   const [downloadsData, authorPackageCount] = await Promise.all([
-    fetchWithRetry(downloadsUrl, { noRetryOn429: true }),    // api.npmjs.org — rate-limited; best-effort single shot (no retry storm, correct-host backoff)
+    downloadsPromise,                // api.npmjs.org — cached / skipped-under-backoff / best-effort
     getAuthorPackageCount()          // registry.npmjs.org search — cached + kill-switchable
   ]);
 

package/src/shared/http-limiter.js

@@ -162,6 +162,19 @@ function hostForUrl(url) {
   try { return new URL(url).hostname || DEFAULT_HOST; } catch { return DEFAULT_HOST; }
 }
 
+/**
+ * True when a host is in (or near) a 429 backoff — fetching it now would park on
+ * the token gate. Lets best-effort callers (downloads-count) SKIP the fetch under a
+ * storm instead of stalling the scan. "Near" = an active pause OR a high backoff
+ * level (the host is 429ing essentially every probe). Pure read: never allocates a
+ * bucket (an unseen host is, by definition, not backed off).
+ */
+function isHostBackedOff(host, levelThreshold = 6) {
+  const b = _buckets.get(host || DEFAULT_HOST);
+  if (!b) return false;
+  return Date.now() < b.bo.pauseUntil || b.bo.level >= levelThreshold;
+}
+
 function _effectiveRate(level = 0) {
   let rate = RATE_LIMIT_PER_SEC;
   if (BOOT_SLOWSTART_MS > 0 && Date.now() - _bootAt < BOOT_SLOWSTART_MS) {
@@ -463,6 +476,7 @@ module.exports = {
   getBrainState,
   computeBackoffTransition,
   hostForUrl,
+  isHostBackedOff,
   resetLimiter,
   _restartBootSlowStartForTests,
   getActiveSemaphore