muaddib-scanner 2.10.21 → 2.10.22

package/scripts/benchmark.js

@@ -0,0 +1,326 @@
+#!/usr/bin/env node
+'use strict';
+
+/**
+ * MUAD'DIB Performance Benchmark
+ *
+ * Generates synthetic projects of different sizes and measures:
+ * - Scan time (wall-clock via process.hrtime.bigint())
+ * - Peak memory usage (process.memoryUsage())
+ * - Per-scanner breakdown (via _capture mode timing)
+ *
+ * Usage: node scripts/benchmark.js [--runs N] [--sizes small,medium,large]
+ */
+
+const fs = require('fs');
+const path = require('path');
+const os = require('os');
+
+// ----- Config -----
+const RUNS = parseInt(process.argv.find((a, i) => process.argv[i - 1] === '--runs') || '3', 10);
+const SIZE_ARG = process.argv.find((a, i) => process.argv[i - 1] === '--sizes') || 'small,medium,large';
+const SIZES = SIZE_ARG.split(',').map(s => s.trim());
+
+const SIZE_CONFIGS = {
+  small:  { files: 10,  label: '10 JS files' },
+  medium: { files: 100, label: '100 JS files' },
+  large:  { files: 500, label: '500 JS files (cap test)' }
+};
+
+// ----- Synthetic file templates -----
+// Mix of benign-looking code with varied patterns to exercise all scanners
+const TEMPLATES = [
+  // Standard module
+  (i) => `'use strict';
+const path = require('path');
+const fs = require('fs');
+
+function process${i}(input) {
+  const result = input.toString().trim();
+  return path.resolve(result);
+}
+
+module.exports = { process${i} };
+`,
+  // HTTP client usage
+  (i) => `'use strict';
+const https = require('https');
+
+function fetch${i}(url) {
+  return new Promise((resolve, reject) => {
+    https.get(url, (res) => {
+      let data = '';
+      res.on('data', (chunk) => { data += chunk; });
+      res.on('end', () => resolve(JSON.parse(data)));
+    }).on('error', reject);
+  });
+}
+
+module.exports = { fetch${i} };
+`,
+  // Crypto usage
+  (i) => `'use strict';
+const crypto = require('crypto');
+
+function hash${i}(data) {
+  return crypto.createHash('sha256').update(data).digest('hex');
+}
+
+function verify${i}(data, expected) {
+  const actual = hash${i}(data);
+  return crypto.timingSafeEqual(Buffer.from(actual), Buffer.from(expected));
+}
+
+module.exports = { hash${i}, verify${i} };
+`,
+  // Config/util module
+  (i) => `'use strict';
+const os = require('os');
+
+const CONFIG_${i} = {
+  tmpDir: os.tmpdir(),
+  cpus: os.cpus().length,
+  platform: os.platform(),
+  arch: os.arch()
+};
+
+function getConfig${i}() {
+  return { ...CONFIG_${i} };
+}
+
+module.exports = { getConfig${i}, CONFIG_${i} };
+`,
+  // Event emitter pattern
+  (i) => `'use strict';
+const { EventEmitter } = require('events');
+
+class Service${i} extends EventEmitter {
+  constructor() {
+    super();
+    this.data = new Map();
+  }
+
+  add(key, value) {
+    this.data.set(key, value);
+    this.emit('added', { key, value });
+  }
+
+  remove(key) {
+    this.data.delete(key);
+    this.emit('removed', { key });
+  }
+}
+
+module.exports = { Service${i} };
+`
+];
+
+// ----- Generate synthetic project -----
+function generateProject(tmpDir, fileCount) {
+  fs.mkdirSync(tmpDir, { recursive: true });
+
+  // Create package.json
+  fs.writeFileSync(path.join(tmpDir, 'package.json'), JSON.stringify({
+    name: `bench-project-${fileCount}`,
+    version: '1.0.0',
+    description: 'Synthetic benchmark project',
+    main: 'index.js'
+  }, null, 2));
+
+  // Create subdirectories for realism
+  const dirs = ['', 'src', 'lib', 'utils', 'helpers'];
+  for (const d of dirs) {
+    if (d) fs.mkdirSync(path.join(tmpDir, d), { recursive: true });
+  }
+
+  // Create JS files
+  for (let i = 0; i < fileCount; i++) {
+    const template = TEMPLATES[i % TEMPLATES.length];
+    const dir = dirs[i % dirs.length];
+    const filePath = path.join(tmpDir, dir, `module-${i}.js`);
+    fs.writeFileSync(filePath, template(i));
+  }
+
+  // Create index.js that references some modules
+  const imports = Array.from({ length: Math.min(10, fileCount) }, (_, i) => {
+    const dir = dirs[i % dirs.length];
+    const rel = dir ? `./${dir}/module-${i}` : `./module-${i}`;
+    return `const m${i} = require('${rel}');`;
+  }).join('\n');
+  fs.writeFileSync(path.join(tmpDir, 'index.js'), `'use strict';\n${imports}\n\nconsole.log('Loaded modules');\n`);
+
+  return tmpDir;
+}
+
+// ----- Cleanup -----
+function cleanup(dir) {
+  try {
+    fs.rmSync(dir, { recursive: true, force: true });
+  } catch { /* ignore */ }
+}
+
+// ----- Run benchmark -----
+async function benchmark() {
+  // Lazy-load the scanner
+  const { run } = require('../src/index.js');
+  const { clearFileListCache } = require('../src/utils.js');
+  const { clearASTCache } = require('../src/shared/constants.js');
+
+  console.log('='.repeat(70));
+  console.log('  MUAD\'DIB Performance Benchmark');
+  console.log(`  Runs per size: ${RUNS}  |  Sizes: ${SIZES.join(', ')}`);
+  console.log(`  Node ${process.version}  |  ${os.cpus()[0]?.model || 'unknown CPU'}  |  ${os.platform()}`);
+  console.log('='.repeat(70));
+  console.log();
+
+  const results = {};
+
+  for (const sizeName of SIZES) {
+    const config = SIZE_CONFIGS[sizeName];
+    if (!config) {
+      console.error(`Unknown size: ${sizeName}`);
+      continue;
+    }
+
+    console.log(`--- ${sizeName.toUpperCase()}: ${config.label} ---`);
+
+    const tmpDir = path.join(os.tmpdir(), `muaddib-bench-${sizeName}-${Date.now()}`);
+    generateProject(tmpDir, config.files);
+
+    const times = [];
+    const memories = [];
+
+    for (let r = 0; r < RUNS; r++) {
+      // Clear caches between runs for fair measurement
+      clearFileListCache();
+      clearASTCache();
+
+      // Force GC if available
+      if (global.gc) global.gc();
+
+      const memBefore = process.memoryUsage();
+      const start = process.hrtime.bigint();
+
+      try {
+        await run(tmpDir, { _capture: true });
+      } catch (err) {
+        console.error(`  Run ${r + 1} error: ${err.message}`);
+      }
+
+      const end = process.hrtime.bigint();
+      const memAfter = process.memoryUsage();
+
+      const durationMs = Number(end - start) / 1e6;
+      const heapDelta = memAfter.heapUsed - memBefore.heapUsed;
+      const rssAfter = memAfter.rss;
+
+      times.push(durationMs);
+      memories.push({ heapDelta, rss: rssAfter, heapUsed: memAfter.heapUsed });
+
+      console.log(`  Run ${r + 1}/${RUNS}: ${durationMs.toFixed(0)}ms  |  heap: ${(memAfter.heapUsed / 1024 / 1024).toFixed(1)}MB  |  RSS: ${(rssAfter / 1024 / 1024).toFixed(1)}MB`);
+    }
+
+    // Stats
+    times.sort((a, b) => a - b);
+    const median = times[Math.floor(times.length / 2)];
+    const mean = times.reduce((a, b) => a + b, 0) / times.length;
+    const min = times[0];
+    const max = times[times.length - 1];
+    const peakRss = Math.max(...memories.map(m => m.rss));
+    const peakHeap = Math.max(...memories.map(m => m.heapUsed));
+
+    results[sizeName] = { median, mean, min, max, peakRss, peakHeap, runs: RUNS, files: config.files };
+
+    console.log(`  => median: ${median.toFixed(0)}ms  mean: ${mean.toFixed(0)}ms  min: ${min.toFixed(0)}ms  max: ${max.toFixed(0)}ms`);
+    console.log(`  => peak heap: ${(peakHeap / 1024 / 1024).toFixed(1)}MB  peak RSS: ${(peakRss / 1024 / 1024).toFixed(1)}MB`);
+    console.log();
+
+    cleanup(tmpDir);
+  }
+
+  // ----- Per-scanner timing (single run on medium) -----
+  console.log('--- SCANNER BREAKDOWN (medium, single run) ---');
+  const scannerTmpDir = path.join(os.tmpdir(), `muaddib-bench-scanner-${Date.now()}`);
+  const scannerFiles = SIZE_CONFIGS.medium?.files || 100;
+  generateProject(scannerTmpDir, scannerFiles);
+  clearFileListCache();
+  clearASTCache();
+
+  // Monkey-patch Promise.allSettled to measure per-scanner time
+  const origAllSettled = Promise.allSettled.bind(Promise);
+  const scannerTimings = [];
+
+  // We'll measure by wrapping run() and parsing its internal flow
+  // Simpler approach: time each scanner individually
+  const scannerModules = [
+    { name: 'PackageJson', mod: '../src/scanner/package.js', fn: 'scanPackageJson' },
+    { name: 'ShellScripts', mod: '../src/scanner/shell.js', fn: 'scanShellScripts' },
+    { name: 'AST', mod: '../src/scanner/ast.js', fn: 'analyzeAST' },
+    { name: 'Obfuscation', mod: '../src/scanner/obfuscation.js', fn: 'detectObfuscation' },
+    { name: 'Dependencies', mod: '../src/scanner/dependencies.js', fn: 'scanDependencies' },
+    { name: 'Hashes', mod: '../src/scanner/hash.js', fn: 'scanHashes' },
+    { name: 'DataFlow', mod: '../src/scanner/dataflow.js', fn: 'analyzeDataFlow' },
+    { name: 'Typosquat', mod: '../src/scanner/typosquat.js', fn: 'scanTyposquatting' },
+    { name: 'GitHubActions', mod: '../src/scanner/github-actions.js', fn: 'scanGitHubActions' },
+    { name: 'Entropy', mod: '../src/scanner/entropy.js', fn: 'scanEntropy' },
+    { name: 'AIConfig', mod: '../src/scanner/ai-config.js', fn: 'scanAIConfig' }
+  ];
+
+  for (const s of scannerModules) {
+    try {
+      const mod = require(s.mod);
+      const fn = mod[s.fn];
+      if (!fn) {
+        scannerTimings.push({ name: s.name, ms: 0, note: 'not found' });
+        continue;
+      }
+
+      clearFileListCache(); // each scanner gets fresh file list
+      const start = process.hrtime.bigint();
+      try {
+        await fn(scannerTmpDir, {});
+      } catch { /* some scanners may throw on benign input */ }
+      const end = process.hrtime.bigint();
+      const ms = Number(end - start) / 1e6;
+      scannerTimings.push({ name: s.name, ms });
+    } catch (err) {
+      scannerTimings.push({ name: s.name, ms: 0, note: err.message });
+    }
+  }
+
+  // Sort by time descending
+  scannerTimings.sort((a, b) => b.ms - a.ms);
+  const totalScannerMs = scannerTimings.reduce((sum, s) => sum + s.ms, 0);
+
+  for (const s of scannerTimings) {
+    const pct = totalScannerMs > 0 ? ((s.ms / totalScannerMs) * 100).toFixed(1) : '0.0';
+    const bar = '#'.repeat(Math.max(1, Math.round(s.ms / totalScannerMs * 40)));
+    console.log(`  ${s.name.padEnd(15)} ${s.ms.toFixed(0).padStart(6)}ms  ${pct.padStart(5)}%  ${bar}${s.note ? ` (${s.note})` : ''}`);
+  }
+  console.log(`  ${'TOTAL'.padEnd(15)} ${totalScannerMs.toFixed(0).padStart(6)}ms`);
+  console.log();
+
+  cleanup(scannerTmpDir);
+
+  // ----- Summary table -----
+  console.log('='.repeat(70));
+  console.log('  SUMMARY');
+  console.log('='.repeat(70));
+  console.log(`  ${'Size'.padEnd(10)} ${'Files'.padStart(6)} ${'Median'.padStart(8)} ${'Mean'.padStart(8)} ${'Min'.padStart(8)} ${'Max'.padStart(8)} ${'Heap'.padStart(8)} ${'RSS'.padStart(8)}`);
+  console.log(`  ${'-'.repeat(10)} ${'-'.repeat(6)} ${'-'.repeat(8)} ${'-'.repeat(8)} ${'-'.repeat(8)} ${'-'.repeat(8)} ${'-'.repeat(8)} ${'-'.repeat(8)}`);
+  for (const [name, r] of Object.entries(results)) {
+    console.log(`  ${name.padEnd(10)} ${String(r.files).padStart(6)} ${(r.median.toFixed(0) + 'ms').padStart(8)} ${(r.mean.toFixed(0) + 'ms').padStart(8)} ${(r.min.toFixed(0) + 'ms').padStart(8)} ${(r.max.toFixed(0) + 'ms').padStart(8)} ${((r.peakHeap / 1024 / 1024).toFixed(1) + 'MB').padStart(8)} ${((r.peakRss / 1024 / 1024).toFixed(1) + 'MB').padStart(8)}`);
+  }
+  console.log();
+
+  // Slowest scanner
+  if (scannerTimings.length > 0) {
+    console.log(`  Slowest scanner: ${scannerTimings[0].name} (${scannerTimings[0].ms.toFixed(0)}ms, ${((scannerTimings[0].ms / totalScannerMs) * 100).toFixed(1)}% of total)`);
+  }
+  console.log();
+}
+
+benchmark().catch(err => {
+  console.error('Benchmark failed:', err);
+  process.exit(1);
+});

package/src/index.js

@@ -27,7 +27,7 @@ const { buildModuleGraph, annotateTaintedExports, detectCrossFileFlows, annotate
 const { computeReachableFiles } = require('./scanner/reachability.js');
 const { runTemporalAnalyses } = require('./temporal-runner.js');
 const { formatOutput } = require('./output-formatter.js');
-const { setExtraExcludes, getExtraExcludes, Spinner, listInstalledPackages, clearFileListCache, wasFilesCapped, debugLog } = require('./utils.js');
+const { setExtraExcludes, getExtraExcludes, Spinner, listInstalledPackages, clearFileListCache, wasFilesCapped, getOverflowFiles, debugLog } = require('./utils.js');
 const { SEVERITY_WEIGHTS, RISK_THRESHOLDS, MAX_RISK_SCORE, isPackageLevelThreat, computeGroupScore, applyFPReductions, applyCompoundBoosts, calculateRiskScore, applyConfigOverrides, resetConfigOverrides, getSeverityWeights } = require('./scoring.js');
 const { resolveConfig } = require('./config.js');
 const { buildIntentPairs } = require('./intent-graph.js');
@@ -480,9 +480,39 @@ async function run(targetPath, options = {}) {
     aiConfigThreats
   ] = scanResult;
 
-  // Emit warning if file count cap was hit
+  // Emit warning if file count cap was hit + quick-scan overflow files
+  const quickScanThreats = [];
   if (wasFilesCapped()) {
-    warnings.push('File count cap reached (500 files) — some files were not scanned. Root-level files were prioritized.');
+    warnings.push('File count cap reached (500 files) — overflow files scanned in quick-scan mode (lifecycle + child_process only).');
+    const overflowFiles = getOverflowFiles();
+    const QUICK_SCAN_PATTERNS = [
+      { re: /\brequire\s*\(\s*['"]child_process['"]\s*\)/, type: 'dangerous_exec', severity: 'HIGH', label: 'require("child_process")' },
+      { re: /\brequire\s*\(\s*['"]node:child_process['"]\s*\)/, type: 'dangerous_exec', severity: 'HIGH', label: 'require("node:child_process")' },
+      { re: /\b(?:exec|execSync|spawn|spawnSync)\s*\(/, type: 'dangerous_exec', severity: 'HIGH', label: 'exec/spawn call' },
+      { re: /\bprocess\.mainModule\b/, type: 'dynamic_require', severity: 'HIGH', label: 'process.mainModule' },
+      { re: /\bModule\._load\b/, type: 'module_load_bypass', severity: 'CRITICAL', label: 'Module._load' }
+    ];
+    for (const filePath of overflowFiles) {
+      try {
+        const stat = fs.statSync(filePath);
+        if (stat.size > getMaxFileSize()) continue;
+        const content = fs.readFileSync(filePath, 'utf8');
+        const relFile = path.relative(targetPath, filePath);
+        for (const pat of QUICK_SCAN_PATTERNS) {
+          if (pat.re.test(content)) {
+            quickScanThreats.push({
+              type: pat.type,
+              severity: pat.severity,
+              message: `[quick-scan] ${pat.label} detected in overflow file.`,
+              file: relFile
+            });
+          }
+        }
+      } catch { /* skip unreadable files */ }
+    }
+    if (quickScanThreats.length > 0) {
+      debugLog(`Quick-scan found ${quickScanThreats.length} threats in ${overflowFiles.length} overflow files`);
+    }
   }
 
   // Stop spinner now that scanning is complete
@@ -494,6 +524,7 @@ async function run(targetPath, options = {}) {
     ...packageThreats,
     ...shellThreats,
     ...astThreats,
+    ...quickScanThreats,
     ...obfuscationThreats,
     ...dependencyThreats,
     ...hashThreats,

package/src/ioc/scraper.js

@@ -7,7 +7,6 @@ const AdmZip = require('adm-zip');
 const IOC_FILE = path.join(__dirname, 'data/iocs.json');
 const COMPACT_IOC_FILE = path.join(__dirname, 'data/iocs-compact.json');
 const HOME_IOC_FILE = path.join(os.homedir(), '.muaddib', 'data', 'iocs.json');
-const STATIC_IOCS_FILE = path.join(__dirname, '../../data/static-iocs.json');
 const { generateCompactIOCs, NEVER_WILDCARD } = require('./updater.js');
 const { Spinner } = require('../utils.js');
 const { NPM_PACKAGE_REGEX } = require('../shared/constants.js');
@@ -144,17 +143,6 @@ function parseCSV(csvContent, hasHeader = true) {
   return results;
 }
 
-function loadStaticIOCs() {
-  try {
-    if (fs.existsSync(STATIC_IOCS_FILE)) {
-      return JSON.parse(fs.readFileSync(STATIC_IOCS_FILE, 'utf8'));
-    }
-  } catch (e) {
-    console.log(`[WARN] Error loading static-iocs.json: ${e.message}`);
-  }
-  return { socket: [], phylum: [], npmRemoved: [] };
-}
-
 const MAX_REDIRECTS = 5;
 const MAX_RESPONSE_SIZE = 200 * 1024 * 1024; // 200MB
 const MAX_ENTRY_UNCOMPRESSED = 50 * 1024 * 1024;   // 50MB per zip entry
@@ -972,124 +960,6 @@ async function scrapeGitHubAdvisory() {
   return packages;
 }
 
-// ============================================
-// SOURCE 5: Static IOCs (Socket, Phylum, npm removed)
-// Local file maintained manually
-// ============================================
-async function scrapeStaticIOCs() {
-  console.log('[SCRAPER] Static IOCs (local file)...');
-  const packages = [];
-  const staticIOCs = loadStaticIOCs();
-  
-  // Socket.dev reports
-  for (const pkg of staticIOCs.socket || []) {
-    if (!pkg.version) continue; // Skip entries without version — avoids wildcard cascade
-    packages.push({
-      id: `SOCKET-${pkg.name}`,
-      name: pkg.name,
-      version: pkg.version,
-      severity: pkg.severity || 'critical',
-      confidence: 'high',
-      source: 'socket-dev',
-      description: pkg.description || 'Malicious package reported by Socket.dev',
-      references: ['https://socket.dev/npm/package/' + pkg.name],
-      mitre: 'T1195.002',
-      freshness: createFreshness('socket', 'high')
-    });
-  }
-  
-  // Phylum Research
-  for (const pkg of staticIOCs.phylum || []) {
-    if (!pkg.version) continue; // Skip entries without version — avoids wildcard cascade
-    packages.push({
-      id: `PHYLUM-${pkg.name}`,
-      name: pkg.name,
-      version: pkg.version,
-      severity: pkg.severity || 'critical',
-      confidence: 'high',
-      source: 'phylum',
-      description: pkg.description || 'Malicious package reported by Phylum Research',
-      references: ['https://blog.phylum.io'],
-      mitre: 'T1195.002',
-      freshness: createFreshness('phylum', 'high')
-    });
-  }
-  
-  // npm removed packages
-  for (const pkg of staticIOCs.npmRemoved || []) {
-    if (!pkg.version) continue; // Skip entries without version — avoids wildcard cascade
-    packages.push({
-      id: `NPM-REMOVED-${pkg.name}`,
-      name: pkg.name,
-      version: pkg.version,
-      severity: 'critical',
-      confidence: 'high',
-      source: 'npm-removed',
-      description: 'Removed from npm: ' + (pkg.reason || 'security violation'),
-      references: ['https://www.npmjs.com/policies/security'],
-      mitre: 'T1195.002',
-      freshness: createFreshness('npm-removed', 'medium')
-    });
-  }
-  
-  console.log(`[SCRAPER]   ${packages.length} packages`);
-  return packages;
-}
-
-// ============================================
-// SOURCE 6: Snyk Known Malware
-// Historical attacks database
-// ============================================
-async function scrapeSnykMalware() {
-  console.log('[SCRAPER] Snyk Malware DB...');
-  const packages = [];
-  
-  const knownSnykMalware = [
-    { name: 'event-stream', version: '3.3.6', description: 'Flatmap-stream backdoor (2018)' },
-    { name: 'flatmap-stream', version: '*', description: 'Malicious dependency of event-stream' },
-    { name: 'eslint-scope', version: '3.7.2', description: 'Credential theft (2018)' },
-    { name: 'eslint-config-eslint', version: '5.0.2', description: 'Credential theft (2018)' },
-    { name: 'getcookies', version: '*', description: 'Backdoor malware' },
-    { name: 'mailparser', version: '2.3.0', description: 'Compromised version' },
-    { name: 'node-ipc', version: '10.1.1', description: 'Protestware - file deletion' },
-    { name: 'node-ipc', version: '10.1.2', description: 'Protestware - file deletion' },
-    { name: 'node-ipc', version: '10.1.3', description: 'Protestware - file deletion' },
-    { name: 'colors', version: '1.4.1', description: 'Protestware - infinite loop' },
-    { name: 'colors', version: '1.4.2', description: 'Protestware - infinite loop' },
-    { name: 'faker', version: '6.6.6', description: 'Protestware - breaking change' },
-    { name: 'ua-parser-js', version: '0.7.29', description: 'Cryptominer injection' },
-    { name: 'ua-parser-js', version: '0.8.0', description: 'Cryptominer injection' },
-    { name: 'ua-parser-js', version: '1.0.0', description: 'Cryptominer injection' },
-    { name: 'coa', version: '2.0.3', description: 'Malicious version' },
-    { name: 'coa', version: '2.0.4', description: 'Malicious version' },
-    { name: 'coa', version: '2.1.1', description: 'Malicious version' },
-    { name: 'coa', version: '2.1.3', description: 'Malicious version' },
-    { name: 'coa', version: '3.0.1', description: 'Malicious version' },
-    { name: 'coa', version: '3.1.3', description: 'Malicious version' },
-    { name: 'rc', version: '1.2.9', description: 'Malicious version' },
-    { name: 'rc', version: '1.3.9', description: 'Malicious version' },
-    { name: 'rc', version: '2.3.9', description: 'Malicious version' },
-  ];
-  
-  for (const pkg of knownSnykMalware) {
-    packages.push({
-      id: ('SNYK-' + pkg.name + '-' + pkg.version).replace(/[^a-zA-Z0-9-]/g, '-'),
-      name: pkg.name,
-      version: pkg.version,
-      severity: 'critical',
-      confidence: 'high',
-      source: 'snyk-known',
-      description: pkg.description,
-      references: ['https://snyk.io/advisor'],
-      mitre: 'T1195.002',
-      freshness: createFreshness('snyk', 'high')
-    });
-  }
-  
-  console.log(`[SCRAPER]   ${packages.length} packages`);
-  return packages;
-}
-
 // ============================================
 // MAIN SCRAPER
 // ============================================
@@ -1152,8 +1022,6 @@ async function runScraper() {
     scrapeDatadogIOCs(),
     scrapeOSSFMaliciousPackages(osvResult.knownIds),
     scrapeGitHubAdvisory(),
-    scrapeStaticIOCs(),
-    scrapeSnykMalware(),
     scrapeOSVPyPIDataDump()
   ]);
 
@@ -1161,9 +1029,7 @@ async function runScraper() {
   const datadogResult = results[1];
   const ossfPackages = results[2];
   const githubPackages = results[3];
-  const staticPackages = results[4];
-  const snykPackages = results[5];
-  const pypiPackages = results[6];
+  const pypiPackages = results[4];
 
   // Log aggregated warnings
   if (_noVersionSkipCount > 0) {
@@ -1176,9 +1042,7 @@ async function runScraper() {
     ...shaiHuludResult.packages,
     ...datadogResult.packages,
     ...ossfPackages,
-    ...githubPackages,
-    ...staticPackages,
-    ...snykPackages
+    ...githubPackages
   ];
 
   // Merge all hashes
@@ -1326,8 +1190,7 @@ async function runScraper() {
     'github-advisory',
     'socket-dev',
     'phylum',
-    'npm-removed',
-    'snyk-known'
+    'npm-removed'
   ];
 
   // Save enriched (full) IOCs — atomic write via .tmp + rename
@@ -1428,7 +1291,7 @@ module.exports = {
   runScraper, scrapeShaiHuludDetector, scrapeDatadogIOCs,
   // Pure utility functions (exported for testing)
   parseCSVLine, parseCSV, extractVersions, parseOSVEntry,
-  createFreshness, isAllowedRedirect, loadStaticIOCs,
+  createFreshness, isAllowedRedirect,
   validateIOCEntry,
   getNoVersionSkipCount, resetNoVersionSkipCount,
   CONFIDENCE_ORDER, ALLOWED_REDIRECT_DOMAINS,

package/src/ioc/updater.js

@@ -209,7 +209,7 @@ function loadCachedIOCs() {
     files: yamlIOCs.files.map(function(f) { return f.name; })
   };
 
-  // Priority 2: Local scraped IOCs (full enriched file)
+  // Priority 2a: Local scraped IOCs (full enriched file)
   if (fs.existsSync(LOCAL_IOC_FILE)) {
     try {
       const localIOCs = JSON.parse(fs.readFileSync(LOCAL_IOC_FILE, 'utf8'));
@@ -217,8 +217,10 @@ function loadCachedIOCs() {
     } catch (e) {
       console.log('[WARN] Failed to load IOC database (iocs.json): ' + e.message);
     }
-  } else if (fs.existsSync(LOCAL_COMPACT_FILE)) {
-    // Priority 2b: Compact file (shipped in npm, lightweight)
+  }
+
+  // Priority 2b: Always merge compact file (contains manual IOCs not in full — TeamPCP, LiteLLM, etc.)
+  if (fs.existsSync(LOCAL_COMPACT_FILE)) {
     try {
       const compactData = JSON.parse(fs.readFileSync(LOCAL_COMPACT_FILE, 'utf8'));
       const expandedIOCs = expandCompactIOCs(compactData);
@@ -315,7 +317,9 @@ function createOptimizedIOCs(iocs) {
       const versions = pkg.version.split(',').map(v => v.trim()).filter(Boolean);
       for (const ver of versions) {
         const entry = Object.assign({}, pkg, { version: ver });
-        if (ver === '*') pypiWildcardPackages.add(pkg.name);
+        if (ver === '*' && !NEVER_WILDCARD_PYPI.has(pkg.name)) {
+          pypiWildcardPackages.add(pkg.name);
+        }
         if (!pypiPackagesMap.has(pkg.name)) pypiPackagesMap.set(pkg.name, []);
         pypiPackagesMap.get(pkg.name).push(entry);
       }
@@ -323,7 +327,10 @@ function createOptimizedIOCs(iocs) {
     }
 
     if (pkg.version === '*') {
-      pypiWildcardPackages.add(pkg.name);
+      // Defense-in-depth: NEVER_WILDCARD_PYPI packages must not be wildcarded
+      if (!NEVER_WILDCARD_PYPI.has(pkg.name)) {
+        pypiWildcardPackages.add(pkg.name);
+      }
     }
 
     if (!pypiPackagesMap.has(pkg.name)) {
@@ -377,6 +384,13 @@ const NEVER_WILDCARD = new Set([
   'posthog-node', 'posthog-js', 'ngx-bootstrap', '@asyncapi/specs'
 ]);
 
+// PyPI equivalent: legitimate packages where only specific versions were compromised.
+// These are among the most popular PyPI packages — wildcarding them would cause mass FPs.
+const NEVER_WILDCARD_PYPI = new Set([
+  'flask', 'django', 'requests', 'numpy', 'pandas',
+  'scipy', 'tensorflow', 'torch', 'fastapi', 'uvicorn'
+]);
+
 function generateCompactIOCs(fullIOCs) {
   const wildcards = [];
   const versioned = Object.create(null);
@@ -416,6 +430,7 @@ function generateCompactIOCs(fullIOCs) {
 
   for (const p of fullIOCs.pypi_packages || []) {
     if (p.version === '*') {
+      if (NEVER_WILDCARD_PYPI.has(p.name)) continue;
       pypiWildcards.push(p.name);
     } else {
       if (!pypiVersioned[p.name]) pypiVersioned[p.name] = [];
@@ -481,9 +496,13 @@ function expandCompactIOCs(compact) {
     }
   }
 
-  // Expand PyPI wildcards
+  // Expand PyPI wildcards (deduplicate via Set, enforce NEVER_WILDCARD_PYPI)
   const pypiPackages = [];
+  const seenPyPIWildcards = new Set();
   for (const name of compact.pypi_wildcards || []) {
+    if (seenPyPIWildcards.has(name)) continue;
+    if (NEVER_WILDCARD_PYPI.has(name)) continue;
+    seenPyPIWildcards.add(name);
     pypiPackages.push({ name: name, version: '*', severity: defaultSev });
   }
 
@@ -593,4 +612,4 @@ function verifyIOCHMAC(data, hmac) {
   }
 }
 
-module.exports = { updateIOCs, loadCachedIOCs, invalidateCache, generateCompactIOCs, expandCompactIOCs, mergeIOCs, createOptimizedIOCs, generateIOCHMAC, verifyIOCHMAC, checkIOCStaleness, NEVER_WILDCARD };
+module.exports = { updateIOCs, loadCachedIOCs, invalidateCache, generateCompactIOCs, expandCompactIOCs, mergeIOCs, createOptimizedIOCs, generateIOCHMAC, verifyIOCHMAC, checkIOCStaleness, NEVER_WILDCARD, NEVER_WILDCARD_PYPI };