muaddib-scanner 2.10.17 → 2.10.19

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "muaddib-scanner",
-  "version": "2.10.17",
+  "version": "2.10.19",
   "description": "Supply-chain threat detection & response for npm & PyPI/Python",
   "main": "src/index.js",
   "bin": {

package/src/index.js

@@ -27,7 +27,7 @@ const { buildModuleGraph, annotateTaintedExports, detectCrossFileFlows, annotate
 const { computeReachableFiles } = require('./scanner/reachability.js');
 const { runTemporalAnalyses } = require('./temporal-runner.js');
 const { formatOutput } = require('./output-formatter.js');
-const { setExtraExcludes, getExtraExcludes, Spinner, listInstalledPackages, clearFileListCache, debugLog } = require('./utils.js');
+const { setExtraExcludes, getExtraExcludes, Spinner, listInstalledPackages, clearFileListCache, wasFilesCapped, debugLog } = require('./utils.js');
 const { SEVERITY_WEIGHTS, RISK_THRESHOLDS, MAX_RISK_SCORE, isPackageLevelThreat, computeGroupScore, applyFPReductions, applyCompoundBoosts, calculateRiskScore, applyConfigOverrides, resetConfigOverrides, getSeverityWeights } = require('./scoring.js');
 const { resolveConfig } = require('./config.js');
 const { buildIntentPairs } = require('./intent-graph.js');
@@ -480,6 +480,11 @@ async function run(targetPath, options = {}) {
     aiConfigThreats
   ] = scanResult;
 
+  // Emit warning if file count cap was hit
+  if (wasFilesCapped()) {
+    warnings.push('File count cap reached (500 files) — some files were not scanned. Root-level files were prioritized.');
+  }
+
   // Stop spinner now that scanning is complete
   if (spinner) {
     spinner.succeed(`[MUADDIB] Scanned ${targetPath}`);

package/src/scan-worker.js

@@ -0,0 +1,31 @@
+'use strict';
+
+/**
+ * Worker thread entry point for static analysis.
+ * Runs the full scan pipeline in an isolated V8 isolate.
+ * The parent can call worker.terminate() to kill synchronous code
+ * (V8::TerminateExecution) — this is the only way to enforce a real
+ * timeout on synchronous AST parsing and tree-walking.
+ *
+ * Communication:
+ *   parentPort.postMessage({ type: 'result', data: scanResult })
+ *   parentPort.postMessage({ type: 'error', message: string })
+ */
+
+const { parentPort, workerData } = require('worker_threads');
+
+if (!parentPort) {
+  // Not running as a worker — exit gracefully
+  process.exit(1);
+}
+
+const { run } = require('./index.js');
+
+(async () => {
+  try {
+    const result = await run(workerData.extractedDir, { _capture: true });
+    parentPort.postMessage({ type: 'result', data: result });
+  } catch (err) {
+    parentPort.postMessage({ type: 'error', message: err.message || String(err) });
+  }
+})();

package/src/scanner/npm-registry.js

@@ -79,9 +79,23 @@ async function getPackageMetadata(packageName) {
   // Validate package name before building URL
   if (!NPM_PACKAGE_REGEX.test(packageName)) return null;
 
-  // 1. Registry metadata
-  const registryUrl = REGISTRY_URL + '/' + encodeURIComponent(packageName);
-  const meta = await fetchWithRetry(registryUrl);
+  // 1. Registry metadata — read from temporal-analysis cache if warm (monitor pipeline
+  // pre-fetches metadata for temporal checks). Only reads the Map, never fires HTTP.
+  // Falls back to own fetchWithRetry (with retries + 429 handling) on cache miss.
+  let meta = null;
+  try {
+    const { _metadataCache, METADATA_CACHE_TTL } = require('../temporal-analysis.js');
+    const cached = _metadataCache.get(packageName);
+    if (cached && (Date.now() - cached.fetchedAt) < METADATA_CACHE_TTL) {
+      meta = cached.data;
+    }
+  } catch {
+    // temporal-analysis not available — fall through to fetchWithRetry
+  }
+  if (!meta) {
+    const registryUrl = REGISTRY_URL + '/' + encodeURIComponent(packageName);
+    meta = await fetchWithRetry(registryUrl);
+  }
   if (!meta) return null;
 
   const createdAt = meta.time?.created || null;

package/src/scanner/obfuscation.js

@@ -2,8 +2,10 @@ const fs = require('fs');
 const path = require('path');
 const { findFiles, forEachSafeFile } = require('../utils.js');
 
-// node_modules NOT excluded: detect obfuscated code in dependencies
-const OBF_EXCLUDED_DIRS = ['.git', '.muaddib-cache'];
+// node_modules NOT excluded: detect obfuscated code in dependencies.
+// dist/build/out/output excluded: bundled output is always flagged as isPackageOutput (LOW)
+// and costs significant processing time on large SDKs.
+const OBF_EXCLUDED_DIRS = ['.git', '.muaddib-cache', 'dist', 'build', 'out', 'output'];
 
 function detectObfuscation(targetPath) {
   const threats = [];

package/src/shared/constants.js

@@ -100,22 +100,50 @@ const ACORN_OPTIONS = { ecmaVersion: 2024, sourceType: 'module', allowHashBang:
 
 const acorn = require('acorn');
 
+/**
+ * AST parse cache — same content+options returns the same AST.
+ * Scanners do not mutate AST nodes (verified: only read comparisons).
+ * Cleared between scans via clearASTCache().
+ * Key = code.length + '|' + optionsKey + '|' + code.slice(0,128) + code.slice(-64)
+ * (length-prefixed partial key for fast Map lookup; collisions resolved by full WeakRef check)
+ */
+const _astCache = new Map();
+const _AST_CACHE_MAX = 600; // Max entries (one scan ≈ 500 files max)
+
 /**
  * Parse JS source with module-mode fallback to script-mode.
  * `const package = ...` is valid in script mode but reserved in module mode.
+ * Results are cached for reuse across scanners within the same scan.
  * Returns AST or null if both modes fail.
  */
 function safeParse(code, extraOptions = {}) {
+  // Build cache key: options signature + content fingerprint
+  const optKey = Object.keys(extraOptions).length === 0 ? '' : JSON.stringify(extraOptions);
+  const cacheKey = code.length + '|' + optKey + '|' + code.slice(0, 128) + code.slice(-64);
+
+  const cached = _astCache.get(cacheKey);
+  if (cached !== undefined) return cached;
+
   const opts = { ...ACORN_OPTIONS, ...extraOptions };
+  let ast = null;
   try {
-    return acorn.parse(code, opts);
+    ast = acorn.parse(code, opts);
   } catch {
     try {
-      return acorn.parse(code, { ...opts, sourceType: 'script' });
+      ast = acorn.parse(code, { ...opts, sourceType: 'script' });
     } catch {
-      return null;
+      ast = null;
     }
   }
+
+  // Cache the result (including null for unparseable files)
+  if (_astCache.size >= _AST_CACHE_MAX) _astCache.clear();
+  _astCache.set(cacheKey, ast);
+  return ast;
+}
+
+function clearASTCache() {
+  _astCache.clear();
 }
 
-module.exports = { REHABILITATED_PACKAGES, NPM_PACKAGE_REGEX, MAX_TARBALL_SIZE, DOWNLOAD_TIMEOUT, MAX_FILE_SIZE, ACORN_OPTIONS, safeParse, getMaxFileSize, setMaxFileSize, resetMaxFileSize };
+module.exports = { REHABILITATED_PACKAGES, NPM_PACKAGE_REGEX, MAX_TARBALL_SIZE, DOWNLOAD_TIMEOUT, MAX_FILE_SIZE, ACORN_OPTIONS, safeParse, clearASTCache, getMaxFileSize, setMaxFileSize, resetMaxFileSize };

package/src/temporal-analysis.js

@@ -4,6 +4,13 @@ const REGISTRY_URL = 'https://registry.npmjs.org';
 const TIMEOUT_MS = 10_000;
 const MAX_RESPONSE_SIZE = 50 * 1024 * 1024; // 50MB (some packages have lots of versions)
 
+// Metadata cache: avoids duplicate HTTP requests when multiple temporal modules
+// fetch the same package metadata within a short window (monitor pipeline).
+const _metadataCache = new Map(); // packageName → { data, fetchedAt }
+const _inflightRequests = new Map(); // packageName → Promise
+const METADATA_CACHE_TTL = 5 * 60 * 1000; // 5 minutes
+const METADATA_CACHE_MAX = 200;
+
 const LIFECYCLE_SCRIPTS = [
   'preinstall',
   'install',
@@ -16,11 +23,10 @@ const LIFECYCLE_SCRIPTS = [
 ];
 
 /**
- * Fetch full package metadata from the npm registry.
- * @param {string} packageName - npm package name (scoped or unscoped)
- * @returns {Promise<object>} Full registry metadata (versions, time, maintainers, etc.)
+ * Raw HTTP fetch — always hits the npm registry. Use fetchPackageMetadata() instead,
+ * which adds caching and inflight dedup.
  */
-function fetchPackageMetadata(packageName) {
+function _fetchPackageMetadataImpl(packageName) {
   const encodedName = encodeURIComponent(packageName).replace('%40', '@');
   const url = `${REGISTRY_URL}/${encodedName}`;
   const urlObj = new URL(url);
@@ -68,7 +74,15 @@ function fetchPackageMetadata(packageName) {
       res.on('end', () => {
         if (destroyed) return;
         try {
-          resolve(JSON.parse(data));
+          const parsed = JSON.parse(data);
+          // Store in cache on successful fetch
+          if (_metadataCache.size >= METADATA_CACHE_MAX) {
+            // Evict oldest entry
+            const oldestKey = _metadataCache.keys().next().value;
+            _metadataCache.delete(oldestKey);
+          }
+          _metadataCache.set(packageName, { data: parsed, fetchedAt: Date.now() });
+          resolve(parsed);
         } catch (e) {
           reject(new Error(`Invalid JSON from registry for ${packageName}: ${e.message}`));
         }
@@ -88,6 +102,39 @@ function fetchPackageMetadata(packageName) {
   });
 }
 
+/**
+ * Fetch full package metadata from the npm registry with caching and inflight dedup.
+ * Multiple callers requesting the same package within 5 minutes share one HTTP request.
+ * @param {string} packageName - npm package name (scoped or unscoped)
+ * @returns {Promise<object>} Full registry metadata (versions, time, maintainers, etc.)
+ */
+function fetchPackageMetadata(packageName) {
+  // Check cache first (TTL-based)
+  const cached = _metadataCache.get(packageName);
+  if (cached && (Date.now() - cached.fetchedAt) < METADATA_CACHE_TTL) {
+    return Promise.resolve(cached.data);
+  }
+
+  // Dedup inflight requests — if the same package is already being fetched, reuse that Promise
+  if (_inflightRequests.has(packageName)) {
+    return _inflightRequests.get(packageName);
+  }
+
+  const promise = _fetchPackageMetadataImpl(packageName).finally(() => {
+    _inflightRequests.delete(packageName);
+  });
+  _inflightRequests.set(packageName, promise);
+  return promise;
+}
+
+/**
+ * Clear the metadata cache. Exported for tests and monitor reset.
+ */
+function clearMetadataCache() {
+  _metadataCache.clear();
+  _inflightRequests.clear();
+}
+
 /**
  * Extract lifecycle scripts from a package.json object.
  * @param {object} packageJson - A package.json object (or a version entry from registry metadata)
@@ -253,8 +300,14 @@ async function detectSuddenLifecycleChange(packageName) {
 
 module.exports = {
   fetchPackageMetadata,
+  clearMetadataCache,
   getLifecycleScripts,
   compareLifecycleScripts,
   getLatestVersions,
-  detectSuddenLifecycleChange
+  detectSuddenLifecycleChange,
+  // Exposed for tests only
+  _metadataCache,
+  _inflightRequests,
+  METADATA_CACHE_TTL,
+  METADATA_CACHE_MAX
 };

package/src/utils.js

@@ -1,12 +1,14 @@
 const fs = require('fs');
 const path = require('path');
-const { MAX_FILE_SIZE, getMaxFileSize } = require('./shared/constants.js');
+const { MAX_FILE_SIZE, getMaxFileSize, clearASTCache } = require('./shared/constants.js');
 
 /**
  * Directories excluded from scanning.
- * Only skip dependency/VCS/cache dirs - never skip user source code.
+ * Skips dependency/VCS/cache dirs and bundled output (dist/build/out).
+ * Bundled output is minified, huge, and produces FPs without security value.
+ * Obfuscation scanner uses its own OBF_EXCLUDED_DIRS to intentionally scan these.
  */
-const EXCLUDED_DIRS = ['node_modules', '.git', '.muaddib-cache'];
+const EXCLUDED_DIRS = ['node_modules', '.git', '.muaddib-cache', 'dist', 'build', 'out', 'output'];
 
 /**
  * Extra directories to exclude (set at runtime via --exclude flag).
@@ -20,6 +22,14 @@ let _scanRoot = '';
  * Cleared between scans via clearFileListCache().
  */
 const _fileListCache = new Map();
+let _filesCapped = false;
+
+/**
+ * File content cache — read each file once, reused across all scanners in a single scan.
+ * Key = absolute file path, Value = file content string.
+ * Cleared between scans via clearFileListCache().
+ */
+const _fileContentCache = new Map();
 
 function setExtraExcludes(dirs, scanRoot) {
   _extraExcludedDirs = Array.isArray(dirs) ? dirs : [];
@@ -59,6 +69,13 @@ function isDevFile(relativePath) {
   return DEV_PATTERNS.some(pattern => pattern.test(relativePath));
 }
 
+/**
+ * Maximum number of files to scan per package.
+ * Malware packages rarely have >50 JS files; 500 is a generous safety margin.
+ * Prevents large SDKs (1000+ files) from monopolizing scan time.
+ */
+const MAX_SCAN_FILES = 500;
+
 /**
  * Generic recursive file finder with symlink protection and depth limit.
  * @param {string} dir - Starting directory
@@ -66,6 +83,7 @@ function isDevFile(relativePath) {
  * @param {string[]} [options.extensions=['.js']] - File extensions to match
  * @param {string[]} [options.excludedDirs=EXCLUDED_DIRS] - Dirs to skip
  * @param {number} [options.maxDepth=100] - Max recursion depth
+ * @param {number} [options.maxFiles=MAX_SCAN_FILES] - Max files to return (0=unlimited)
  * @param {string[]} [options.results=[]] - Accumulator (internal)
  * @param {Set} [options.visitedInodes=new Set()] - Symlink loop detection (note: inode tracking
  *   is unreliable on Windows where stat.ino may be 0; maxDepth serves as fallback protection)
@@ -77,6 +95,7 @@ function findFiles(dir, options = {}) {
     extensions = ['.js'],
     excludedDirs = EXCLUDED_DIRS,
     maxDepth = 100,
+    maxFiles = MAX_SCAN_FILES,
     results = [],
     visitedInodes = new Set(),
     visitedPaths = new Set(),
@@ -90,6 +109,21 @@ function findFiles(dir, options = {}) {
     const cached = _fileListCache.get(cacheKey);
     if (cached) return [...cached]; // return copy to prevent mutation
     const result = _findFilesImpl(dir, { extensions, excludedDirs, maxDepth, results, visitedInodes, visitedPaths, depth });
+
+    // Apply file count cap: sort by depth (shallowest first) so root-level files
+    // (most likely to contain malicious entry points) are prioritized.
+    if (maxFiles > 0 && result.length > maxFiles) {
+      result.sort((a, b) => {
+        const depthA = a.split(path.sep).length;
+        const depthB = b.split(path.sep).length;
+        return depthA - depthB;
+      });
+      const capped = result.slice(0, maxFiles);
+      _fileListCache.set(cacheKey, [...capped]);
+      _filesCapped = true;
+      return capped;
+    }
+
     _fileListCache.set(cacheKey, [...result]);
     return result;
   }
@@ -188,6 +222,13 @@ function findJsFiles(dir, results = []) {
 
 function clearFileListCache() {
   _fileListCache.clear();
+  _fileContentCache.clear();
+  clearASTCache();
+  _filesCapped = false;
+}
+
+function wasFilesCapped() {
+  return _filesCapped;
 }
 
 /**
@@ -278,9 +319,17 @@ class Spinner {
 /**
  * Iterates files with size guard and error handling.
  * Calls callback(file, content) for each readable file under MAX_FILE_SIZE.
+ * File contents are cached in _fileContentCache for reuse across scanners.
  */
 function forEachSafeFile(files, callback) {
   for (const file of files) {
+    // Check content cache first
+    const cached = _fileContentCache.get(file);
+    if (cached !== undefined) {
+      callback(file, cached);
+      continue;
+    }
+
     try {
       const stat = fs.statSync(file);
       if (stat.size > getMaxFileSize()) continue;
@@ -289,6 +338,9 @@ function forEachSafeFile(files, callback) {
     try {
       content = fs.readFileSync(file, 'utf8');
     } catch { continue; }
+
+    // Cache for subsequent scanners
+    _fileContentCache.set(file, content);
     callback(file, content);
   }
 }
@@ -332,11 +384,13 @@ function debugLog(...args) {
 
 module.exports = {
   EXCLUDED_DIRS,
+  MAX_SCAN_FILES,
   DEV_PATTERNS,
   isDevFile,
   findFiles,
   findJsFiles,
   clearFileListCache,
+  wasFilesCapped,
   escapeHtml,
   getCallName,
   Spinner,