muaddib-scanner 2.11.58 → 2.11.59

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "muaddib-scanner",
-  "version": "2.11.58",
+  "version": "2.11.59",
   "description": "Supply-chain threat detection & response for npm & PyPI/Python",
   "main": "src/index.js",
   "bin": {

package/{self-scan-v2.11.58.json → self-scan-v2.11.59.json}

@@ -1,6 +1,6 @@
 {
   "target": "node_modules",
-  "timestamp": "2026-06-04T21:33:40.755Z",
+  "timestamp": "2026-06-05T06:40:48.592Z",
   "threats": [
     {
       "type": "string_mutation_obfuscation",

package/src/monitor/queue.js

@@ -445,6 +445,24 @@ async function scanPackage(name, version, ecosystem, tarballUrl, registryMeta, s
     // ML Phase 2a: Count JS files and detect test presence for enriched features
     const { fileCountTotal, hasTests } = countPackageFiles(extractedDir);
 
+    // Hoisted before the worker spawn (per-worker 429-storm fix): fetch the npm
+    // registry metadata ONCE on the main thread. The shared http-limiter coordinates
+    // it and the temporal cache is warm (npm-registry.js reads it first), so only
+    // weekly_downloads + author hit the network. Passed to the worker via scanContext
+    // so the worker's processor consumes it instead of re-fetching on its OWN module-
+    // level limiter — N worker_threads = N uncoordinated limiters → ~Nx npm throughput
+    // → 429 bursts. Also reused below (ML / first-publish / training records /
+    // reputation) — previously this was a SECOND main-side fetch after the worker.
+    let npmRegistryMeta = null;
+    if (ecosystem === 'npm') {
+      try {
+        const { getPackageMetadata } = require('../scanner/npm-registry.js');
+        npmRegistryMeta = await getPackageMetadata(name);
+      } catch (err) {
+        console.error(`[ML] npm registry fetch failed for ${name}: ${err.message}`);
+      }
+    }
+
     let result;
     try {
       // scanContext: feeds monitor-side info (name/version/ecosystem) and the
@@ -463,6 +481,11 @@ async function scanPackage(name, version, ecosystem, tarballUrl, registryMeta, s
         // the full 20-scanner pipeline (unchanged behaviour).
         scanMode: (meta && meta.scanMode) || 'full'
       };
+      // Hand the main-thread-fetched metadata to the worker so its processor skips
+      // the per-worker getPackageMetadata fetch (429-storm fix). npm only; the key
+      // is set even when null ("main already tried, don't refetch"). pypi leaves it
+      // absent so the worker takes the unchanged CLI/else-if path.
+      if (ecosystem === 'npm') scanContext.npmRegistryMeta = npmRegistryMeta;
       result = await runScanInWorker(extractedDir, STATIC_SCAN_TIMEOUT_MS, scanContext, signal);
     } catch (staticErr) {
       if (/static scan timeout/i.test(staticErr.message)) {
@@ -494,22 +517,11 @@ async function scanPackage(name, version, ecosystem, tarballUrl, registryMeta, s
     // First-publish detection: used for sandbox priority below
     const isFirstPublish = cacheTrigger && cacheTrigger.reason === 'first_publish';
 
-    // Fetch npm registry metadata for ALL npm packages (not just those with findings).
-    // Needed for: (1) isFirstPublishHighRisk decision, (2) ML classifier features,
-    // (3) JSONL training records — clean packages MUST have metadata to prevent
-    // data leakage (model learning "metadata=0 → clean" instead of behavioral signals).
-    // Cost: near-zero for npm packages because temporal checks (line ~1014) already
-    // pre-fetch registry metadata into temporal-analysis._metadataCache, and
-    // getPackageMetadata() reads this cache first (npm-registry.js:87-95).
-    let npmRegistryMeta = null;
-    if (ecosystem === 'npm') {
-      try {
-        const { getPackageMetadata } = require('../scanner/npm-registry.js');
-        npmRegistryMeta = await getPackageMetadata(name);
-      } catch (err) {
-        console.error(`[ML] npm registry fetch failed for ${name}: ${err.message}`);
-      }
-    }
+    // npm registry metadata was fetched ONCE before the worker spawn (hoisted above
+    // to feed scanContext.npmRegistryMeta) and is reused here for: isFirstPublishHigh-
+    // Risk, ML classifier features, JSONL training records, and reputation scoring.
+    // Clean packages MUST carry metadata to prevent training-data leakage (model
+    // learning "metadata=0 → clean" instead of behavioral signals).
 
     // First-publish sandbox priority: sandbox even with 0 static findings
     // if the package is from a new/unknown maintainer without a linked repository.

package/src/pipeline/processor.js

@@ -191,6 +191,31 @@ async function process(threats, targetPath, options, pythonDeps, warnings, scann
   // 3 metadata-dependent gates (METADATA_FACTOR, MATURE_CAP, DELTA_MODE) in
   // one shot. Individual gates can still be turned off via their own =0 flag.
   if (
+    packageName &&
+    _pkgMeta &&
+    options &&
+    Object.prototype.hasOwnProperty.call(options, 'npmRegistryMeta')
+  ) {
+    // The monitor fetched the registry metadata ONCE on the main thread (shared
+    // http-limiter, warm temporal cache) and passed it via scanContext.npmRegistry-
+    // Meta. Consume it here instead of re-fetching: a per-worker getPackageMetadata()
+    // runs on the worker's OWN module-level limiter, so N worker_threads = N
+    // uncoordinated limiters → ~Nx npm throughput → 429 storms. Strict semantics —
+    // the key being present (even null) means "main already handled it"; a null
+    // value (main fetch failed) leaves the gates to no-op, identical to a failed
+    // fetch (best-effort metadata signals stay silent). CLI / `muaddib replay`
+    // never set the key → the else-if fetch path below is unchanged.
+    const injected = options.npmRegistryMeta;
+    if (injected) {
+      // Attach the scanned version (getPackageMetadata never sets it) so
+      // applyMatureStableCap can require scan_version === latest_version — a
+      // historical compromised version must not inherit live "stable" reputation.
+      if (injected.scan_version == null && packageVersion != null) {
+        injected.scan_version = packageVersion;
+      }
+      _pkgMeta.npmRegistryMeta = injected;
+    }
+  } else if (
     packageName &&
     _pkgMeta &&
     globalThis.process.env.MUADDIB_NO_REGISTRY_FETCH !== '1' &&

package/src/scanner/trusted-dep-diff.js

@@ -29,6 +29,7 @@
 const fs = require('fs');
 const path = require('path');
 const https = require('https');
+const { acquireRegistrySlot, releaseRegistrySlot, signal429 } = require('../shared/http-limiter.js');
 
 const TRUSTED_DEP_AGE_THRESHOLD_MS = 7 * 24 * 60 * 60 * 1000; // 7 days
 
@@ -80,7 +81,17 @@ function httpsGet(url, timeoutMs = 30_000) {
 async function checkDepDiff(name, newVersion) {
   const findings = [];
   try {
-    const body = await httpsGet(`https://registry.npmjs.org/${encodeURIComponent(name)}`, PACKUMENT_TIMEOUT_MS);
+    // Route through the shared http-limiter (concurrency + token bucket + 429
+    // backoff) instead of a raw uncoordinated httpsGet — this scanner runs inside
+    // the monitor worker_threads, where an unbounded fetch joins the per-worker
+    // 429 storm. finally-release keeps the semaphore balanced even on reject.
+    await acquireRegistrySlot();
+    let body;
+    try {
+      body = await httpsGet(`https://registry.npmjs.org/${encodeURIComponent(name)}`, PACKUMENT_TIMEOUT_MS);
+    } finally {
+      releaseRegistrySlot();
+    }
     const packument = JSON.parse(body);
 
     if (!packument.versions || !packument.time) return findings;
@@ -107,13 +118,20 @@ async function checkDepDiff(name, newVersion) {
     for (const dep of addedDeps) {
       let ageMs = null;
       try {
-        const depBody = await httpsGet(`https://registry.npmjs.org/${encodeURIComponent(dep)}`, DEP_AGE_TIMEOUT_MS);
+        await acquireRegistrySlot();
+        let depBody;
+        try {
+          depBody = await httpsGet(`https://registry.npmjs.org/${encodeURIComponent(dep)}`, DEP_AGE_TIMEOUT_MS);
+        } finally {
+          releaseRegistrySlot();
+        }
         const depData = JSON.parse(depBody);
         const created = depData.time && depData.time.created;
         if (created) {
           ageMs = Date.now() - new Date(created).getTime();
         }
       } catch (err) {
+        if (/HTTP 429/.test(err.message)) { try { signal429(); } catch { /* limiter best-effort */ } }
         console.log(`[SCANNER] trusted-dep-diff: could not check age of dependency ${dep}: ${err.message}`);
       }
 
@@ -152,6 +170,7 @@ async function checkDepDiff(name, newVersion) {
 
     return findings;
   } catch (err) {
+    if (/HTTP 429/.test(err.message)) { try { signal429(); } catch { /* limiter best-effort */ } }
     console.log(`[SCANNER] trusted-dep-diff: check failed for ${name}@${newVersion}: ${err.message}`);
     return findings;
   }