muaddib-scanner 2.10.19 → 2.10.21

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "muaddib-scanner",
-  "version": "2.10.19",
+  "version": "2.10.21",
   "description": "Supply-chain threat detection & response for npm & PyPI/Python",
   "main": "src/index.js",
   "bin": {

package/src/scanner/npm-registry.js

@@ -1,5 +1,6 @@
 const { NPM_PACKAGE_REGEX } = require('../shared/constants.js');
 const { debugLog } = require('../utils.js');
+const { acquireRegistrySlot, releaseRegistrySlot } = require('../shared/http-limiter.js');
 
 const REGISTRY_URL = 'https://registry.npmjs.org';
 const DOWNLOADS_URL = 'https://api.npmjs.org/downloads/point/last-week';
@@ -94,7 +95,12 @@ async function getPackageMetadata(packageName) {
   }
   if (!meta) {
     const registryUrl = REGISTRY_URL + '/' + encodeURIComponent(packageName);
-    meta = await fetchWithRetry(registryUrl);
+    await acquireRegistrySlot();
+    try {
+      meta = await fetchWithRetry(registryUrl);
+    } finally {
+      releaseRegistrySlot();
+    }
   }
   if (!meta) return null;
 
@@ -121,9 +127,16 @@ async function getPackageMetadata(packageName) {
     ? SEARCH_URL + '?text=maintainer:' + encodeURIComponent(maintainer) + '&size=1'
     : null;
 
+  async function fetchAuthorWithSlot() {
+    if (!authorUrl) return null;
+    await acquireRegistrySlot();
+    try { return await fetchWithRetry(authorUrl); }
+    finally { releaseRegistrySlot(); }
+  }
+
   const [downloadsData, authorData] = await Promise.all([
-    fetchWithRetry(downloadsUrl),
-    authorUrl ? fetchWithRetry(authorUrl) : Promise.resolve(null)
+    fetchWithRetry(downloadsUrl),    // api.npmjs.org — no semaphore needed
+    fetchAuthorWithSlot()            // registry.npmjs.org — semaphore protected
   ]);
 
   const weeklyDownloads = downloadsData?.downloads ?? 0;

package/src/shared/http-limiter.js

@@ -0,0 +1,54 @@
+'use strict';
+
+/**
+ * Centralized HTTP concurrency limiter for npm registry requests.
+ *
+ * With 16 monitor workers × 7+ HTTP requests/package, uncapped concurrency
+ * reaches 112+ simultaneous requests — well above npm's implicit rate limit.
+ * This module caps ALL registry.npmjs.org requests to a single semaphore
+ * so that no more than REGISTRY_SEMAPHORE_MAX requests are in-flight at once.
+ *
+ * Consumers: temporal-analysis.js, temporal-ast-diff.js, monitor.js (getNpmLatestTarball),
+ *            npm-registry.js (fetchWithRetry to registry.npmjs.org).
+ * NOT covered: api.npmjs.org (different server), replicate.npmjs.com (CouchDB changes stream).
+ */
+
+const REGISTRY_SEMAPHORE_MAX = 10;
+
+const _semaphore = { active: 0, queue: [] };
+
+function acquireRegistrySlot() {
+  if (_semaphore.active < REGISTRY_SEMAPHORE_MAX) {
+    _semaphore.active++;
+    return Promise.resolve();
+  }
+  return new Promise(resolve => {
+    _semaphore.queue.push(resolve);
+  });
+}
+
+function releaseRegistrySlot() {
+  if (_semaphore.queue.length > 0) {
+    const next = _semaphore.queue.shift();
+    next(); // Transfers slot to next waiter (active count stays the same)
+  } else {
+    _semaphore.active--;
+  }
+}
+
+function resetLimiter() {
+  _semaphore.active = 0;
+  _semaphore.queue.length = 0;
+}
+
+function getActiveSemaphore() {
+  return _semaphore;
+}
+
+module.exports = {
+  REGISTRY_SEMAPHORE_MAX,
+  acquireRegistrySlot,
+  releaseRegistrySlot,
+  resetLimiter,
+  getActiveSemaphore
+};

package/src/temporal-analysis.js

@@ -1,4 +1,5 @@
 const https = require('https');
+const { acquireRegistrySlot, releaseRegistrySlot, resetLimiter, getActiveSemaphore, REGISTRY_SEMAPHORE_MAX } = require('./shared/http-limiter.js');
 
 const REGISTRY_URL = 'https://registry.npmjs.org';
 const TIMEOUT_MS = 10_000;
@@ -6,9 +7,11 @@ const MAX_RESPONSE_SIZE = 50 * 1024 * 1024; // 50MB (some packages have lots of
 
 // Metadata cache: avoids duplicate HTTP requests when multiple temporal modules
 // fetch the same package metadata within a short window (monitor pipeline).
-const _metadataCache = new Map(); // packageName → { data, fetchedAt }
+// Entries with error=true are negative cache (shorter TTL) to avoid retry storms.
+const _metadataCache = new Map(); // packageName → { data, fetchedAt, error? }
 const _inflightRequests = new Map(); // packageName → Promise
 const METADATA_CACHE_TTL = 5 * 60 * 1000; // 5 minutes
+const NEGATIVE_CACHE_TTL = 60 * 1000; // 60 seconds for failed fetches
 const METADATA_CACHE_MAX = 200;
 
 const LIFECYCLE_SCRIPTS = [
@@ -24,9 +27,30 @@ const LIFECYCLE_SCRIPTS = [
 
 /**
  * Raw HTTP fetch — always hits the npm registry. Use fetchPackageMetadata() instead,
- * which adds caching and inflight dedup.
+ * which adds caching, inflight dedup, and semaphore.
+ * Acquires a shared HTTP semaphore slot before making the request.
  */
-function _fetchPackageMetadataImpl(packageName) {
+async function _fetchPackageMetadataImpl(packageName) {
+  await acquireRegistrySlot();
+  try {
+    return await _fetchPackageMetadataHttp(packageName);
+  } catch (err) {
+    // Negative cache: store failure for 60s to prevent retry storms
+    if (_metadataCache.size >= METADATA_CACHE_MAX) {
+      const oldestKey = _metadataCache.keys().next().value;
+      _metadataCache.delete(oldestKey);
+    }
+    _metadataCache.set(packageName, { data: null, error: true, fetchedAt: Date.now() });
+    throw err;
+  } finally {
+    releaseRegistrySlot();
+  }
+}
+
+/**
+ * Low-level HTTP request to npm registry. No caching, no semaphore.
+ */
+function _fetchPackageMetadataHttp(packageName) {
   const encodedName = encodeURIComponent(packageName).replace('%40', '@');
   const url = `${REGISTRY_URL}/${encodedName}`;
   const urlObj = new URL(url);
@@ -103,16 +127,23 @@ function _fetchPackageMetadataImpl(packageName) {
 }
 
 /**
- * Fetch full package metadata from the npm registry with caching and inflight dedup.
- * Multiple callers requesting the same package within 5 minutes share one HTTP request.
+ * Fetch full package metadata from the npm registry with caching, inflight dedup,
+ * negative cache, and HTTP semaphore. Multiple callers requesting the same package
+ * within 5 minutes share one HTTP request. Failed fetches are cached for 60s.
  * @param {string} packageName - npm package name (scoped or unscoped)
  * @returns {Promise<object>} Full registry metadata (versions, time, maintainers, etc.)
  */
 function fetchPackageMetadata(packageName) {
-  // Check cache first (TTL-based)
+  // Check cache first (TTL-based, positive + negative)
   const cached = _metadataCache.get(packageName);
-  if (cached && (Date.now() - cached.fetchedAt) < METADATA_CACHE_TTL) {
-    return Promise.resolve(cached.data);
+  if (cached) {
+    const ttl = cached.error ? NEGATIVE_CACHE_TTL : METADATA_CACHE_TTL;
+    if ((Date.now() - cached.fetchedAt) < ttl) {
+      if (cached.error) {
+        return Promise.reject(new Error(`Negative cache hit for ${packageName} (failed ${Math.round((Date.now() - cached.fetchedAt) / 1000)}s ago)`));
+      }
+      return Promise.resolve(cached.data);
+    }
   }
 
   // Dedup inflight requests — if the same package is already being fetched, reuse that Promise
@@ -128,11 +159,12 @@ function fetchPackageMetadata(packageName) {
 }
 
 /**
- * Clear the metadata cache. Exported for tests and monitor reset.
+ * Clear the metadata cache and reset shared semaphore. Exported for tests and monitor reset.
  */
 function clearMetadataCache() {
   _metadataCache.clear();
   _inflightRequests.clear();
+  resetLimiter();
 }
 
 /**
@@ -309,5 +341,9 @@ module.exports = {
   _metadataCache,
   _inflightRequests,
   METADATA_CACHE_TTL,
-  METADATA_CACHE_MAX
+  METADATA_CACHE_MAX,
+  NEGATIVE_CACHE_TTL,
+  // Re-export shared semaphore for backward compat with existing tests
+  _httpSemaphore: getActiveSemaphore(),
+  HTTP_SEMAPHORE_MAX: REGISTRY_SEMAPHORE_MAX
 };

package/src/temporal-ast-diff.js

@@ -7,6 +7,7 @@ const walk = require('acorn-walk');
 const { findJsFiles, forEachSafeFile, debugLog } = require('./utils.js');
 const { fetchPackageMetadata, getLatestVersions } = require('./temporal-analysis.js');
 const { downloadToFile, extractTarGz, sanitizePackageName } = require('./shared/download.js');
+const { acquireRegistrySlot, releaseRegistrySlot } = require('./shared/http-limiter.js');
 
 const { MAX_FILE_SIZE, getMaxFileSize, ACORN_OPTIONS, safeParse } = require('./shared/constants.js');
 
@@ -36,11 +37,21 @@ const PATTERN_SEVERITY = {
 
 /**
  * Fetch version-specific metadata from npm registry.
+ * Acquires a shared HTTP semaphore slot to prevent registry throttling.
  * @param {string} packageName
  * @param {string} version
  * @returns {Promise<object>}
  */
-function fetchVersionMetadata(packageName, version) {
+async function fetchVersionMetadata(packageName, version) {
+  await acquireRegistrySlot();
+  try {
+    return await _fetchVersionMetadataHttp(packageName, version);
+  } finally {
+    releaseRegistrySlot();
+  }
+}
+
+function _fetchVersionMetadataHttp(packageName, version) {
   const encodedName = encodeURIComponent(packageName).replace('%40', '@');
   const url = `${REGISTRY_URL}/${encodedName}/${encodeURIComponent(version)}`;
   const urlObj = new URL(url);
@@ -99,7 +110,13 @@ async function fetchPackageTarball(packageName, version) {
   let extractedDir;
   try {
     const tgzPath = path.join(tmpDir, 'package.tar.gz');
-    await downloadToFile(tarballUrl, tgzPath);
+    // Tarball downloads go through the shared semaphore (npm CDN)
+    await acquireRegistrySlot();
+    try {
+      await downloadToFile(tarballUrl, tgzPath);
+    } finally {
+      releaseRegistrySlot();
+    }
     extractedDir = extractTarGz(tgzPath, tmpDir);
   } catch (err) {
     try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch (e) { debugLog('tmpDir cleanup failed:', e.message); }

package/src/utils.js

@@ -28,8 +28,10 @@ let _filesCapped = false;
  * File content cache — read each file once, reused across all scanners in a single scan.
  * Key = absolute file path, Value = file content string.
  * Cleared between scans via clearFileListCache().
+ * Capped at 500 entries to prevent OOM during evaluate (200 packages sequential).
  */
 const _fileContentCache = new Map();
+const _FILE_CONTENT_CACHE_MAX = 500;
 
 function setExtraExcludes(dirs, scanRoot) {
   _extraExcludedDirs = Array.isArray(dirs) ? dirs : [];
@@ -339,7 +341,8 @@ function forEachSafeFile(files, callback) {
       content = fs.readFileSync(file, 'utf8');
     } catch { continue; }
 
-    // Cache for subsequent scanners
+    // Cache for subsequent scanners (evict all if over cap to prevent OOM in evaluate loops)
+    if (_fileContentCache.size >= _FILE_CONTENT_CACHE_MAX) _fileContentCache.clear();
     _fileContentCache.set(file, content);
     callback(file, content);
   }