muaddib-scanner 2.10.18 → 2.10.20

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "muaddib-scanner",
-  "version": "2.10.18",
+  "version": "2.10.20",
   "description": "Supply-chain threat detection & response for npm & PyPI/Python",
   "main": "src/index.js",
   "bin": {

package/src/scanner/npm-registry.js

@@ -79,9 +79,23 @@ async function getPackageMetadata(packageName) {
   // Validate package name before building URL
   if (!NPM_PACKAGE_REGEX.test(packageName)) return null;
 
-  // 1. Registry metadata
-  const registryUrl = REGISTRY_URL + '/' + encodeURIComponent(packageName);
-  const meta = await fetchWithRetry(registryUrl);
+  // 1. Registry metadata — read from temporal-analysis cache if warm (monitor pipeline
+  // pre-fetches metadata for temporal checks). Only reads the Map, never fires HTTP.
+  // Falls back to own fetchWithRetry (with retries + 429 handling) on cache miss.
+  let meta = null;
+  try {
+    const { _metadataCache, METADATA_CACHE_TTL } = require('../temporal-analysis.js');
+    const cached = _metadataCache.get(packageName);
+    if (cached && (Date.now() - cached.fetchedAt) < METADATA_CACHE_TTL) {
+      meta = cached.data;
+    }
+  } catch {
+    // temporal-analysis not available — fall through to fetchWithRetry
+  }
+  if (!meta) {
+    const registryUrl = REGISTRY_URL + '/' + encodeURIComponent(packageName);
+    meta = await fetchWithRetry(registryUrl);
+  }
   if (!meta) return null;
 
   const createdAt = meta.time?.created || null;

package/src/temporal-analysis.js

@@ -4,6 +4,40 @@ const REGISTRY_URL = 'https://registry.npmjs.org';
 const TIMEOUT_MS = 10_000;
 const MAX_RESPONSE_SIZE = 50 * 1024 * 1024; // 50MB (some packages have lots of versions)
 
+// Metadata cache: avoids duplicate HTTP requests when multiple temporal modules
+// fetch the same package metadata within a short window (monitor pipeline).
+// Entries with error=true are negative cache (shorter TTL) to avoid retry storms.
+const _metadataCache = new Map(); // packageName → { data, fetchedAt, error? }
+const _inflightRequests = new Map(); // packageName → Promise
+const METADATA_CACHE_TTL = 5 * 60 * 1000; // 5 minutes
+const NEGATIVE_CACHE_TTL = 60 * 1000; // 60 seconds for failed fetches
+const METADATA_CACHE_MAX = 200;
+
+// HTTP semaphore: limits concurrent requests to npm registry to prevent throttling.
+// With 16 monitor workers × 7 requests/package, uncapped concurrency hits 112 simultaneous
+// requests — well above npm's implicit rate limit. Cap at 10 to stay under the threshold.
+const HTTP_SEMAPHORE_MAX = 10;
+const _httpSemaphore = { active: 0, queue: [] };
+
+function _acquireHttpSlot() {
+  if (_httpSemaphore.active < HTTP_SEMAPHORE_MAX) {
+    _httpSemaphore.active++;
+    return Promise.resolve();
+  }
+  return new Promise(resolve => {
+    _httpSemaphore.queue.push(resolve);
+  });
+}
+
+function _releaseHttpSlot() {
+  if (_httpSemaphore.queue.length > 0) {
+    const next = _httpSemaphore.queue.shift();
+    next(); // Transfers the slot to the next waiter (active count stays the same)
+  } else {
+    _httpSemaphore.active--;
+  }
+}
+
 const LIFECYCLE_SCRIPTS = [
   'preinstall',
   'install',
@@ -16,11 +50,31 @@ const LIFECYCLE_SCRIPTS = [
 ];
 
 /**
- * Fetch full package metadata from the npm registry.
- * @param {string} packageName - npm package name (scoped or unscoped)
- * @returns {Promise<object>} Full registry metadata (versions, time, maintainers, etc.)
+ * Raw HTTP fetch — always hits the npm registry. Use fetchPackageMetadata() instead,
+ * which adds caching, inflight dedup, and semaphore.
+ * Acquires an HTTP semaphore slot before making the request.
  */
-function fetchPackageMetadata(packageName) {
+async function _fetchPackageMetadataImpl(packageName) {
+  await _acquireHttpSlot();
+  try {
+    return await _fetchPackageMetadataHttp(packageName);
+  } catch (err) {
+    // Negative cache: store failure for 60s to prevent retry storms
+    if (_metadataCache.size >= METADATA_CACHE_MAX) {
+      const oldestKey = _metadataCache.keys().next().value;
+      _metadataCache.delete(oldestKey);
+    }
+    _metadataCache.set(packageName, { data: null, error: true, fetchedAt: Date.now() });
+    throw err;
+  } finally {
+    _releaseHttpSlot();
+  }
+}
+
+/**
+ * Low-level HTTP request to npm registry. No caching, no semaphore.
+ */
+function _fetchPackageMetadataHttp(packageName) {
   const encodedName = encodeURIComponent(packageName).replace('%40', '@');
   const url = `${REGISTRY_URL}/${encodedName}`;
   const urlObj = new URL(url);
@@ -68,7 +122,15 @@ function fetchPackageMetadata(packageName) {
       res.on('end', () => {
         if (destroyed) return;
         try {
-          resolve(JSON.parse(data));
+          const parsed = JSON.parse(data);
+          // Store in cache on successful fetch
+          if (_metadataCache.size >= METADATA_CACHE_MAX) {
+            // Evict oldest entry
+            const oldestKey = _metadataCache.keys().next().value;
+            _metadataCache.delete(oldestKey);
+          }
+          _metadataCache.set(packageName, { data: parsed, fetchedAt: Date.now() });
+          resolve(parsed);
         } catch (e) {
           reject(new Error(`Invalid JSON from registry for ${packageName}: ${e.message}`));
         }
@@ -88,6 +150,48 @@ function fetchPackageMetadata(packageName) {
   });
 }
 
+/**
+ * Fetch full package metadata from the npm registry with caching, inflight dedup,
+ * negative cache, and HTTP semaphore. Multiple callers requesting the same package
+ * within 5 minutes share one HTTP request. Failed fetches are cached for 60s.
+ * @param {string} packageName - npm package name (scoped or unscoped)
+ * @returns {Promise<object>} Full registry metadata (versions, time, maintainers, etc.)
+ */
+function fetchPackageMetadata(packageName) {
+  // Check cache first (TTL-based, positive + negative)
+  const cached = _metadataCache.get(packageName);
+  if (cached) {
+    const ttl = cached.error ? NEGATIVE_CACHE_TTL : METADATA_CACHE_TTL;
+    if ((Date.now() - cached.fetchedAt) < ttl) {
+      if (cached.error) {
+        return Promise.reject(new Error(`Negative cache hit for ${packageName} (failed ${Math.round((Date.now() - cached.fetchedAt) / 1000)}s ago)`));
+      }
+      return Promise.resolve(cached.data);
+    }
+  }
+
+  // Dedup inflight requests — if the same package is already being fetched, reuse that Promise
+  if (_inflightRequests.has(packageName)) {
+    return _inflightRequests.get(packageName);
+  }
+
+  const promise = _fetchPackageMetadataImpl(packageName).finally(() => {
+    _inflightRequests.delete(packageName);
+  });
+  _inflightRequests.set(packageName, promise);
+  return promise;
+}
+
+/**
+ * Clear the metadata cache and reset semaphore. Exported for tests and monitor reset.
+ */
+function clearMetadataCache() {
+  _metadataCache.clear();
+  _inflightRequests.clear();
+  _httpSemaphore.active = 0;
+  _httpSemaphore.queue.length = 0;
+}
+
 /**
  * Extract lifecycle scripts from a package.json object.
  * @param {object} packageJson - A package.json object (or a version entry from registry metadata)
@@ -253,8 +357,17 @@ async function detectSuddenLifecycleChange(packageName) {
 
 module.exports = {
   fetchPackageMetadata,
+  clearMetadataCache,
   getLifecycleScripts,
   compareLifecycleScripts,
   getLatestVersions,
-  detectSuddenLifecycleChange
+  detectSuddenLifecycleChange,
+  // Exposed for tests only
+  _metadataCache,
+  _inflightRequests,
+  _httpSemaphore,
+  METADATA_CACHE_TTL,
+  METADATA_CACHE_MAX,
+  NEGATIVE_CACHE_TTL,
+  HTTP_SEMAPHORE_MAX
 };

package/src/utils.js

@@ -28,8 +28,10 @@ let _filesCapped = false;
  * File content cache — read each file once, reused across all scanners in a single scan.
  * Key = absolute file path, Value = file content string.
  * Cleared between scans via clearFileListCache().
+ * Capped at 500 entries to prevent OOM during evaluate (200 packages sequential).
  */
 const _fileContentCache = new Map();
+const _FILE_CONTENT_CACHE_MAX = 500;
 
 function setExtraExcludes(dirs, scanRoot) {
   _extraExcludedDirs = Array.isArray(dirs) ? dirs : [];
@@ -339,7 +341,8 @@ function forEachSafeFile(files, callback) {
       content = fs.readFileSync(file, 'utf8');
     } catch { continue; }
 
-    // Cache for subsequent scanners
+    // Cache for subsequent scanners (evict all if over cap to prevent OOM in evaluate loops)
+    if (_fileContentCache.size >= _FILE_CONTENT_CACHE_MAX) _fileContentCache.clear();
     _fileContentCache.set(file, content);
     callback(file, content);
   }