muaddib-scanner 2.11.57 → 2.11.59

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "muaddib-scanner",
-  "version": "2.11.57",
+  "version": "2.11.59",
   "description": "Supply-chain threat detection & response for npm & PyPI/Python",
   "main": "src/index.js",
   "bin": {

package/{self-scan-v2.11.57.json → self-scan-v2.11.59.json}

@@ -1,6 +1,6 @@
 {
   "target": "node_modules",
-  "timestamp": "2026-06-04T20:24:41.702Z",
+  "timestamp": "2026-06-05T06:40:48.592Z",
   "threats": [
     {
       "type": "string_mutation_obfuscation",

package/src/monitor/queue.js

@@ -445,6 +445,24 @@ async function scanPackage(name, version, ecosystem, tarballUrl, registryMeta, s
     // ML Phase 2a: Count JS files and detect test presence for enriched features
     const { fileCountTotal, hasTests } = countPackageFiles(extractedDir);
 
+    // Hoisted before the worker spawn (per-worker 429-storm fix): fetch the npm
+    // registry metadata ONCE on the main thread. The shared http-limiter coordinates
+    // it and the temporal cache is warm (npm-registry.js reads it first), so only
+    // weekly_downloads + author hit the network. Passed to the worker via scanContext
+    // so the worker's processor consumes it instead of re-fetching on its OWN module-
+    // level limiter — N worker_threads = N uncoordinated limiters → ~Nx npm throughput
+    // → 429 bursts. Also reused below (ML / first-publish / training records /
+    // reputation) — previously this was a SECOND main-side fetch after the worker.
+    let npmRegistryMeta = null;
+    if (ecosystem === 'npm') {
+      try {
+        const { getPackageMetadata } = require('../scanner/npm-registry.js');
+        npmRegistryMeta = await getPackageMetadata(name);
+      } catch (err) {
+        console.error(`[ML] npm registry fetch failed for ${name}: ${err.message}`);
+      }
+    }
+
     let result;
     try {
       // scanContext: feeds monitor-side info (name/version/ecosystem) and the
@@ -463,6 +481,11 @@ async function scanPackage(name, version, ecosystem, tarballUrl, registryMeta, s
         // the full 20-scanner pipeline (unchanged behaviour).
         scanMode: (meta && meta.scanMode) || 'full'
       };
+      // Hand the main-thread-fetched metadata to the worker so its processor skips
+      // the per-worker getPackageMetadata fetch (429-storm fix). npm only; the key
+      // is set even when null ("main already tried, don't refetch"). pypi leaves it
+      // absent so the worker takes the unchanged CLI/else-if path.
+      if (ecosystem === 'npm') scanContext.npmRegistryMeta = npmRegistryMeta;
       result = await runScanInWorker(extractedDir, STATIC_SCAN_TIMEOUT_MS, scanContext, signal);
     } catch (staticErr) {
       if (/static scan timeout/i.test(staticErr.message)) {
@@ -494,22 +517,11 @@ async function scanPackage(name, version, ecosystem, tarballUrl, registryMeta, s
     // First-publish detection: used for sandbox priority below
     const isFirstPublish = cacheTrigger && cacheTrigger.reason === 'first_publish';
 
-    // Fetch npm registry metadata for ALL npm packages (not just those with findings).
-    // Needed for: (1) isFirstPublishHighRisk decision, (2) ML classifier features,
-    // (3) JSONL training records — clean packages MUST have metadata to prevent
-    // data leakage (model learning "metadata=0 → clean" instead of behavioral signals).
-    // Cost: near-zero for npm packages because temporal checks (line ~1014) already
-    // pre-fetch registry metadata into temporal-analysis._metadataCache, and
-    // getPackageMetadata() reads this cache first (npm-registry.js:87-95).
-    let npmRegistryMeta = null;
-    if (ecosystem === 'npm') {
-      try {
-        const { getPackageMetadata } = require('../scanner/npm-registry.js');
-        npmRegistryMeta = await getPackageMetadata(name);
-      } catch (err) {
-        console.error(`[ML] npm registry fetch failed for ${name}: ${err.message}`);
-      }
-    }
+    // npm registry metadata was fetched ONCE before the worker spawn (hoisted above
+    // to feed scanContext.npmRegistryMeta) and is reused here for: isFirstPublishHigh-
+    // Risk, ML classifier features, JSONL training records, and reputation scoring.
+    // Clean packages MUST carry metadata to prevent training-data leakage (model
+    // learning "metadata=0 → clean" instead of behavioral signals).
 
     // First-publish sandbox priority: sandbox even with 0 static findings
     // if the package is from a new/unknown maintainer without a linked repository.

package/src/pipeline/processor.js

@@ -191,6 +191,31 @@ async function process(threats, targetPath, options, pythonDeps, warnings, scann
   // 3 metadata-dependent gates (METADATA_FACTOR, MATURE_CAP, DELTA_MODE) in
   // one shot. Individual gates can still be turned off via their own =0 flag.
   if (
+    packageName &&
+    _pkgMeta &&
+    options &&
+    Object.prototype.hasOwnProperty.call(options, 'npmRegistryMeta')
+  ) {
+    // The monitor fetched the registry metadata ONCE on the main thread (shared
+    // http-limiter, warm temporal cache) and passed it via scanContext.npmRegistry-
+    // Meta. Consume it here instead of re-fetching: a per-worker getPackageMetadata()
+    // runs on the worker's OWN module-level limiter, so N worker_threads = N
+    // uncoordinated limiters → ~Nx npm throughput → 429 storms. Strict semantics —
+    // the key being present (even null) means "main already handled it"; a null
+    // value (main fetch failed) leaves the gates to no-op, identical to a failed
+    // fetch (best-effort metadata signals stay silent). CLI / `muaddib replay`
+    // never set the key → the else-if fetch path below is unchanged.
+    const injected = options.npmRegistryMeta;
+    if (injected) {
+      // Attach the scanned version (getPackageMetadata never sets it) so
+      // applyMatureStableCap can require scan_version === latest_version — a
+      // historical compromised version must not inherit live "stable" reputation.
+      if (injected.scan_version == null && packageVersion != null) {
+        injected.scan_version = packageVersion;
+      }
+      _pkgMeta.npmRegistryMeta = injected;
+    }
+  } else if (
     packageName &&
     _pkgMeta &&
     globalThis.process.env.MUADDIB_NO_REGISTRY_FETCH !== '1' &&

package/src/scanner/npm-registry.js

@@ -1,14 +1,16 @@
 const { NPM_PACKAGE_REGEX } = require('../shared/constants.js');
 const { debugLog } = require('../utils.js');
-const { acquireRegistrySlot, releaseRegistrySlot } = require('../shared/http-limiter.js');
+const { acquireRegistrySlot, releaseRegistrySlot, signal429 } = require('../shared/http-limiter.js');
 const { computeAdvancedRegistrySignals } = require('../integrations/registry-signals.js');
 
 const REGISTRY_URL = 'https://registry.npmjs.org';
 const DOWNLOADS_URL = 'https://api.npmjs.org/downloads/point/last-week';
 const SEARCH_URL = 'https://registry.npmjs.org/-/v1/search';
 
-const REQUEST_TIMEOUT = 10000; // 10 seconds
-const MAX_RETRIES = 3;
+// Env-tunable; defaults preserve prior behavior except MAX_RETRIES (3 → 5) for more headroom
+// under sustained 429s during a large evaluate burst.
+const REQUEST_TIMEOUT = Math.max(1000, parseInt(process.env.MUADDIB_REGISTRY_TIMEOUT_MS, 10) || 10000); // 10s default
+const MAX_RETRIES = Math.max(1, parseInt(process.env.MUADDIB_REGISTRY_RETRIES, 10) || 5);
 
 /**
  * Create a timeout signal, with fallback for older Node versions.
@@ -31,10 +33,12 @@ async function fetchWithRetry(url) {
       response = await fetch(url, { signal });
     } catch {
       cleanup();
-      // REG-001: Retry on timeout/abort instead of returning null immediately
+      // REG-001: Retry on timeout/abort instead of returning null immediately.
+      // Jittered exponential backoff avoids synchronized retry storms across the
+      // (up to MUADDIB_REGISTRY_CONCURRENCY) concurrent fetches.
       if (attempt < MAX_RETRIES - 1) {
         const backoff = Math.min(1000 * Math.pow(2, attempt), 8000);
-        await new Promise(r => setTimeout(r, backoff));
+        await new Promise(r => setTimeout(r, Math.round(backoff * (0.5 + Math.random() * 0.5))));
       }
       continue;
     }
@@ -48,12 +52,16 @@ async function fetchWithRetry(url) {
       return null;
     }
 
-    // 429 = rate limit, respect Retry-After header (capped at 30s)
+    // 429 = rate limit. Drain the SHARED token bucket so EVERY in-flight request
+    // (not just this one) backs off together — fixes the thundering-herd 429 storm
+    // that left ~17% of packages metadata-less in a local evaluate run. Then honor
+    // Retry-After (capped at 30s) with jitter so retries don't re-synchronize.
     if (response.status === 429) {
       try { await response.text(); } catch (e) { debugLog('response drain failed:', e.message); }
+      try { signal429(); } catch { /* limiter is best-effort */ }
       const retryAfter = parseInt(response.headers.get('retry-after'), 10);
-      const delay = Math.min(retryAfter && retryAfter > 0 ? retryAfter * 1000 : 2000, 30000);
-      await new Promise(r => setTimeout(r, delay));
+      const base = Math.min(retryAfter && retryAfter > 0 ? retryAfter * 1000 : 2000, 30000);
+      await new Promise(r => setTimeout(r, Math.round(base * (0.5 + Math.random() * 0.5))));
       continue;
     }
 

package/src/scanner/trusted-dep-diff.js

@@ -29,6 +29,7 @@
 const fs = require('fs');
 const path = require('path');
 const https = require('https');
+const { acquireRegistrySlot, releaseRegistrySlot, signal429 } = require('../shared/http-limiter.js');
 
 const TRUSTED_DEP_AGE_THRESHOLD_MS = 7 * 24 * 60 * 60 * 1000; // 7 days
 
@@ -80,7 +81,17 @@ function httpsGet(url, timeoutMs = 30_000) {
 async function checkDepDiff(name, newVersion) {
   const findings = [];
   try {
-    const body = await httpsGet(`https://registry.npmjs.org/${encodeURIComponent(name)}`, PACKUMENT_TIMEOUT_MS);
+    // Route through the shared http-limiter (concurrency + token bucket + 429
+    // backoff) instead of a raw uncoordinated httpsGet — this scanner runs inside
+    // the monitor worker_threads, where an unbounded fetch joins the per-worker
+    // 429 storm. finally-release keeps the semaphore balanced even on reject.
+    await acquireRegistrySlot();
+    let body;
+    try {
+      body = await httpsGet(`https://registry.npmjs.org/${encodeURIComponent(name)}`, PACKUMENT_TIMEOUT_MS);
+    } finally {
+      releaseRegistrySlot();
+    }
     const packument = JSON.parse(body);
 
     if (!packument.versions || !packument.time) return findings;
@@ -107,13 +118,20 @@ async function checkDepDiff(name, newVersion) {
     for (const dep of addedDeps) {
       let ageMs = null;
       try {
-        const depBody = await httpsGet(`https://registry.npmjs.org/${encodeURIComponent(dep)}`, DEP_AGE_TIMEOUT_MS);
+        await acquireRegistrySlot();
+        let depBody;
+        try {
+          depBody = await httpsGet(`https://registry.npmjs.org/${encodeURIComponent(dep)}`, DEP_AGE_TIMEOUT_MS);
+        } finally {
+          releaseRegistrySlot();
+        }
         const depData = JSON.parse(depBody);
         const created = depData.time && depData.time.created;
         if (created) {
           ageMs = Date.now() - new Date(created).getTime();
         }
       } catch (err) {
+        if (/HTTP 429/.test(err.message)) { try { signal429(); } catch { /* limiter best-effort */ } }
         console.log(`[SCANNER] trusted-dep-diff: could not check age of dependency ${dep}: ${err.message}`);
       }
 
@@ -152,6 +170,7 @@ async function checkDepDiff(name, newVersion) {
 
     return findings;
   } catch (err) {
+    if (/HTTP 429/.test(err.message)) { try { signal429(); } catch { /* limiter best-effort */ } }
     console.log(`[SCANNER] trusted-dep-diff: check failed for ${name}@${newVersion}: ${err.message}`);
     return findings;
   }

package/src/shared/http-limiter.js

@@ -4,8 +4,8 @@
  * Centralized HTTP concurrency + rate limiter for npm registry requests.
  *
  * Two layers of protection:
- *   1. Concurrency semaphore (REGISTRY_SEMAPHORE_MAX = 10) — caps in-flight requests
- *   2. Rate limiter (RATE_LIMIT_PER_SEC = 30) — caps requests/second via token bucket
+ *   1. Concurrency semaphore (REGISTRY_SEMAPHORE_MAX, default 20, env MUADDIB_REGISTRY_CONCURRENCY) — caps in-flight requests
+ *   2. Rate limiter (RATE_LIMIT_PER_SEC, default 30, env MUADDIB_REGISTRY_RATE) — caps requests/second via token bucket
  *
  * Without rate limiting, 10 concurrent slots × fast-completing requests = 100+ req/s
  * bursts that trigger npm 429 responses → exponential backoff → scan times 10s→90s.
@@ -14,8 +14,11 @@
  * NOT covered: api.npmjs.org (different server), replicate.npmjs.com (CouchDB changes stream).
  */
 
-const REGISTRY_SEMAPHORE_MAX = 20;
-const RATE_LIMIT_PER_SEC = 30;
+// Env-tunable so a constrained client (e.g. local/Windows `evaluate` runs that hit npm 429s during
+// the ~1644-request metadata burst over 548 packages) can dial the burst down without code edits.
+// Defaults preserve prior behavior (20 in-flight / 30 req/s).
+const REGISTRY_SEMAPHORE_MAX = Math.max(1, parseInt(process.env.MUADDIB_REGISTRY_CONCURRENCY, 10) || 20);
+const RATE_LIMIT_PER_SEC = Math.max(1, parseInt(process.env.MUADDIB_REGISTRY_RATE, 10) || 30);
 
 // --- Concurrency semaphore ---