muaddib-scanner 2.10.20 → 2.10.21

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "muaddib-scanner",
-  "version": "2.10.20",
+  "version": "2.10.21",
   "description": "Supply-chain threat detection & response for npm & PyPI/Python",
   "main": "src/index.js",
   "bin": {

package/src/scanner/npm-registry.js

@@ -1,5 +1,6 @@
 const { NPM_PACKAGE_REGEX } = require('../shared/constants.js');
 const { debugLog } = require('../utils.js');
+const { acquireRegistrySlot, releaseRegistrySlot } = require('../shared/http-limiter.js');
 
 const REGISTRY_URL = 'https://registry.npmjs.org';
 const DOWNLOADS_URL = 'https://api.npmjs.org/downloads/point/last-week';
@@ -94,7 +95,12 @@ async function getPackageMetadata(packageName) {
   }
   if (!meta) {
     const registryUrl = REGISTRY_URL + '/' + encodeURIComponent(packageName);
-    meta = await fetchWithRetry(registryUrl);
+    await acquireRegistrySlot();
+    try {
+      meta = await fetchWithRetry(registryUrl);
+    } finally {
+      releaseRegistrySlot();
+    }
   }
   if (!meta) return null;
 
@@ -121,9 +127,16 @@ async function getPackageMetadata(packageName) {
     ? SEARCH_URL + '?text=maintainer:' + encodeURIComponent(maintainer) + '&size=1'
     : null;
 
+  async function fetchAuthorWithSlot() {
+    if (!authorUrl) return null;
+    await acquireRegistrySlot();
+    try { return await fetchWithRetry(authorUrl); }
+    finally { releaseRegistrySlot(); }
+  }
+
   const [downloadsData, authorData] = await Promise.all([
-    fetchWithRetry(downloadsUrl),
-    authorUrl ? fetchWithRetry(authorUrl) : Promise.resolve(null)
+    fetchWithRetry(downloadsUrl),    // api.npmjs.org — no semaphore needed
+    fetchAuthorWithSlot()            // registry.npmjs.org — semaphore protected
   ]);
 
   const weeklyDownloads = downloadsData?.downloads ?? 0;

package/src/shared/http-limiter.js

@@ -0,0 +1,54 @@
+'use strict';
+
+/**
+ * Centralized HTTP concurrency limiter for npm registry requests.
+ *
+ * With 16 monitor workers × 7+ HTTP requests/package, uncapped concurrency
+ * reaches 112+ simultaneous requests — well above npm's implicit rate limit.
+ * This module caps ALL registry.npmjs.org requests to a single semaphore
+ * so that no more than REGISTRY_SEMAPHORE_MAX requests are in-flight at once.
+ *
+ * Consumers: temporal-analysis.js, temporal-ast-diff.js, monitor.js (getNpmLatestTarball),
+ *            npm-registry.js (fetchWithRetry to registry.npmjs.org).
+ * NOT covered: api.npmjs.org (different server), replicate.npmjs.com (CouchDB changes stream).
+ */
+
+const REGISTRY_SEMAPHORE_MAX = 10;
+
+const _semaphore = { active: 0, queue: [] };
+
+function acquireRegistrySlot() {
+  if (_semaphore.active < REGISTRY_SEMAPHORE_MAX) {
+    _semaphore.active++;
+    return Promise.resolve();
+  }
+  return new Promise(resolve => {
+    _semaphore.queue.push(resolve);
+  });
+}
+
+function releaseRegistrySlot() {
+  if (_semaphore.queue.length > 0) {
+    const next = _semaphore.queue.shift();
+    next(); // Transfers slot to next waiter (active count stays the same)
+  } else {
+    _semaphore.active--;
+  }
+}
+
+function resetLimiter() {
+  _semaphore.active = 0;
+  _semaphore.queue.length = 0;
+}
+
+function getActiveSemaphore() {
+  return _semaphore;
+}
+
+module.exports = {
+  REGISTRY_SEMAPHORE_MAX,
+  acquireRegistrySlot,
+  releaseRegistrySlot,
+  resetLimiter,
+  getActiveSemaphore
+};

package/src/temporal-analysis.js

@@ -1,4 +1,5 @@
 const https = require('https');
+const { acquireRegistrySlot, releaseRegistrySlot, resetLimiter, getActiveSemaphore, REGISTRY_SEMAPHORE_MAX } = require('./shared/http-limiter.js');
 
 const REGISTRY_URL = 'https://registry.npmjs.org';
 const TIMEOUT_MS = 10_000;
@@ -13,31 +14,6 @@ const METADATA_CACHE_TTL = 5 * 60 * 1000; // 5 minutes
 const NEGATIVE_CACHE_TTL = 60 * 1000; // 60 seconds for failed fetches
 const METADATA_CACHE_MAX = 200;
 
-// HTTP semaphore: limits concurrent requests to npm registry to prevent throttling.
-// With 16 monitor workers × 7 requests/package, uncapped concurrency hits 112 simultaneous
-// requests — well above npm's implicit rate limit. Cap at 10 to stay under the threshold.
-const HTTP_SEMAPHORE_MAX = 10;
-const _httpSemaphore = { active: 0, queue: [] };
-
-function _acquireHttpSlot() {
-  if (_httpSemaphore.active < HTTP_SEMAPHORE_MAX) {
-    _httpSemaphore.active++;
-    return Promise.resolve();
-  }
-  return new Promise(resolve => {
-    _httpSemaphore.queue.push(resolve);
-  });
-}
-
-function _releaseHttpSlot() {
-  if (_httpSemaphore.queue.length > 0) {
-    const next = _httpSemaphore.queue.shift();
-    next(); // Transfers the slot to the next waiter (active count stays the same)
-  } else {
-    _httpSemaphore.active--;
-  }
-}
-
 const LIFECYCLE_SCRIPTS = [
   'preinstall',
   'install',
@@ -52,10 +28,10 @@ const LIFECYCLE_SCRIPTS = [
 /**
  * Raw HTTP fetch — always hits the npm registry. Use fetchPackageMetadata() instead,
  * which adds caching, inflight dedup, and semaphore.
- * Acquires an HTTP semaphore slot before making the request.
+ * Acquires a shared HTTP semaphore slot before making the request.
  */
 async function _fetchPackageMetadataImpl(packageName) {
-  await _acquireHttpSlot();
+  await acquireRegistrySlot();
   try {
     return await _fetchPackageMetadataHttp(packageName);
   } catch (err) {
@@ -67,7 +43,7 @@ async function _fetchPackageMetadataImpl(packageName) {
     _metadataCache.set(packageName, { data: null, error: true, fetchedAt: Date.now() });
     throw err;
   } finally {
-    _releaseHttpSlot();
+    releaseRegistrySlot();
   }
 }
 
@@ -183,13 +159,12 @@ function fetchPackageMetadata(packageName) {
 }
 
 /**
- * Clear the metadata cache and reset semaphore. Exported for tests and monitor reset.
+ * Clear the metadata cache and reset shared semaphore. Exported for tests and monitor reset.
  */
 function clearMetadataCache() {
   _metadataCache.clear();
   _inflightRequests.clear();
-  _httpSemaphore.active = 0;
-  _httpSemaphore.queue.length = 0;
+  resetLimiter();
 }
 
 /**
@@ -365,9 +340,10 @@ module.exports = {
   // Exposed for tests only
   _metadataCache,
   _inflightRequests,
-  _httpSemaphore,
   METADATA_CACHE_TTL,
   METADATA_CACHE_MAX,
   NEGATIVE_CACHE_TTL,
-  HTTP_SEMAPHORE_MAX
+  // Re-export shared semaphore for backward compat with existing tests
+  _httpSemaphore: getActiveSemaphore(),
+  HTTP_SEMAPHORE_MAX: REGISTRY_SEMAPHORE_MAX
 };

package/src/temporal-ast-diff.js

@@ -7,6 +7,7 @@ const walk = require('acorn-walk');
 const { findJsFiles, forEachSafeFile, debugLog } = require('./utils.js');
 const { fetchPackageMetadata, getLatestVersions } = require('./temporal-analysis.js');
 const { downloadToFile, extractTarGz, sanitizePackageName } = require('./shared/download.js');
+const { acquireRegistrySlot, releaseRegistrySlot } = require('./shared/http-limiter.js');
 
 const { MAX_FILE_SIZE, getMaxFileSize, ACORN_OPTIONS, safeParse } = require('./shared/constants.js');
 
@@ -36,11 +37,21 @@ const PATTERN_SEVERITY = {
 
 /**
  * Fetch version-specific metadata from npm registry.
+ * Acquires a shared HTTP semaphore slot to prevent registry throttling.
  * @param {string} packageName
  * @param {string} version
  * @returns {Promise<object>}
  */
-function fetchVersionMetadata(packageName, version) {
+async function fetchVersionMetadata(packageName, version) {
+  await acquireRegistrySlot();
+  try {
+    return await _fetchVersionMetadataHttp(packageName, version);
+  } finally {
+    releaseRegistrySlot();
+  }
+}
+
+function _fetchVersionMetadataHttp(packageName, version) {
   const encodedName = encodeURIComponent(packageName).replace('%40', '@');
   const url = `${REGISTRY_URL}/${encodedName}/${encodeURIComponent(version)}`;
   const urlObj = new URL(url);
@@ -99,7 +110,13 @@ async function fetchPackageTarball(packageName, version) {
   let extractedDir;
   try {
     const tgzPath = path.join(tmpDir, 'package.tar.gz');
-    await downloadToFile(tarballUrl, tgzPath);
+    // Tarball downloads go through the shared semaphore (npm CDN)
+    await acquireRegistrySlot();
+    try {
+      await downloadToFile(tarballUrl, tgzPath);
+    } finally {
+      releaseRegistrySlot();
+    }
     extractedDir = extractTarGz(tgzPath, tmpDir);
   } catch (err) {
     try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch (e) { debugLog('tmpDir cleanup failed:', e.message); }