muaddib-scanner 2.10.100 → 2.11.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "muaddib-scanner",
-  "version": "2.10.100",
+  "version": "2.11.0",
   "description": "Supply-chain threat detection & response for npm & PyPI/Python",
   "main": "src/index.js",
   "bin": {

package/src/ioc/scraper.js

@@ -1037,7 +1037,8 @@ async function runScraper() {
     scrapeDatadogIOCs(),
     scrapeOSSFMaliciousPackages(osvResult.knownIds),
     scrapeGitHubAdvisory(),
-    scrapeOSVPyPIDataDump()
+    scrapeOSVPyPIDataDump(),
+    scrapeAikidoMalwareFeed()
   ]);
 
   const shaiHuludResult = results[0];
@@ -1045,6 +1046,7 @@ async function runScraper() {
   const ossfPackages = results[2];
   const githubPackages = results[3];
   const pypiPackages = results[4];
+  const aikidoResult = results[5];
 
   // Log aggregated warnings
   if (_noVersionSkipCount > 0) {
@@ -1057,7 +1059,8 @@ async function runScraper() {
     ...shaiHuludResult.packages,
     ...datadogResult.packages,
     ...ossfPackages,
-    ...githubPackages
+    ...githubPackages,
+    ...aikidoResult.packages
   ];
 
   // Merge all hashes
@@ -1069,7 +1072,7 @@ async function runScraper() {
   // Smart deduplication: build map of best entry per key
   // For duplicates, keep the one with highest confidence, then most recent date
   const dedupSpinner = new Spinner();
-  dedupSpinner.start('Deduplicating ' + allPackages.length + ' npm + ' + pypiPackages.length + ' PyPI entries...');
+  dedupSpinner.start('Deduplicating ' + allPackages.length + ' npm + ' + (pypiPackages.length + (aikidoResult.pypi_packages || []).length) + ' PyPI entries...');
   const dedupMap = new Map();
 
   // Seed with existing IOCs (with sanitization of stale comma-in-version entries)
@@ -1091,11 +1094,34 @@ async function runScraper() {
     dedupMap.set(key, pkg);
   }
 
-  // Merge new IOCs with smart replacement (with input validation)
+  // Merge new IOCs with smart replacement (with input validation).
+  // Source-aware: each entry accumulates a `sources: [{name, added_at}]` array
+  // tracking every feed that reported this (name, version). A package
+  // reported by >= 3 distinct sources is treated as confidence-max
+  // (used by `getSourceConfidence` for webhook gating).
   let addedPackages = 0;
   let upgradedPackages = 0;
   let skippedInvalid = 0;
   let skippedNeverWildcard = 0;
+  function appendSource(target, pkg) {
+    if (!Array.isArray(target.sources)) target.sources = [];
+    const newSrc = pkg.source || (pkg.freshness && pkg.freshness.source) || 'unknown';
+    if (!target.sources.some(s => s.name === newSrc)) {
+      target.sources.push({
+        name: newSrc,
+        added_at: (pkg.freshness && pkg.freshness.added_at) || pkg.published || new Date().toISOString()
+      });
+    }
+  }
+  function seedSources(pkg) {
+    if (!Array.isArray(pkg.sources)) {
+      const src = pkg.source || (pkg.freshness && pkg.freshness.source) || 'unknown';
+      pkg.sources = [{
+        name: src,
+        added_at: (pkg.freshness && pkg.freshness.added_at) || pkg.published || new Date().toISOString()
+      }];
+    }
+  }
   for (const pkg of allPackages) {
     if (!validateIOCEntry(pkg.name, pkg.version, 'npm')) {
       skippedInvalid++;
@@ -1108,21 +1134,28 @@ async function runScraper() {
     }
     const key = pkg.name + '@' + pkg.version;
     if (!dedupMap.has(key)) {
+      seedSources(pkg);
       dedupMap.set(key, pkg);
       addedPackages++;
     } else {
       const existing = dedupMap.get(key);
+      // Always accumulate source attribution before any replacement decision.
+      seedSources(existing);
+      appendSource(existing, pkg);
       const existingConf = CONFIDENCE_ORDER[existing.confidence] || 0;
       const newConf = CONFIDENCE_ORDER[pkg.confidence] || 0;
       if (newConf > existingConf) {
-        dedupMap.set(key, pkg);
+        // Replace with the higher-confidence entry but preserve the merged sources list
+        const mergedSources = existing.sources;
+        dedupMap.set(key, Object.assign({}, pkg, { sources: mergedSources }));
         upgradedPackages++;
       } else if (newConf === existingConf) {
         // Same confidence: keep most recent
         const existingDate = existing.published || (existing.freshness && existing.freshness.added_at) || '';
         const newDate = pkg.published || (pkg.freshness && pkg.freshness.added_at) || '';
         if (newDate > existingDate) {
-          dedupMap.set(key, pkg);
+          const mergedSources = existing.sources;
+          dedupMap.set(key, Object.assign({}, pkg, { sources: mergedSources }));
           upgradedPackages++;
         }
       }
@@ -1139,21 +1172,27 @@ async function runScraper() {
     pypiDedupMap.set(key, pkg);
   }
   let addedPyPIPackages = 0;
-  for (const pkg of pypiPackages) {
+  // Merge Aikido PyPI feed into the same loop
+  const allPyPIPackages = pypiPackages.concat(aikidoResult.pypi_packages || []);
+  for (const pkg of allPyPIPackages) {
     if (!validateIOCEntry(pkg.name, pkg.version, 'pypi')) {
       skippedInvalid++;
       continue;
     }
     const key = pkg.name + '@' + pkg.version;
     if (!pypiDedupMap.has(key)) {
+      seedSources(pkg);
       pypiDedupMap.set(key, pkg);
       addedPyPIPackages++;
     } else {
       const existing = pypiDedupMap.get(key);
+      seedSources(existing);
+      appendSource(existing, pkg);
       const existingConf = CONFIDENCE_ORDER[existing.confidence] || 0;
       const newConf = CONFIDENCE_ORDER[pkg.confidence] || 0;
       if (newConf > existingConf) {
-        pypiDedupMap.set(key, pkg);
+        const mergedSources = existing.sources;
+        pypiDedupMap.set(key, Object.assign({}, pkg, { sources: mergedSources }));
       }
     }
   }
@@ -1298,6 +1337,80 @@ async function runScraper() {
   };
 }
 
+// ============================================
+// SOURCE 6: Aikido Open Source Malware Feed (npm + PyPI)
+// Free flat JSON feed at malware-list.aikido.dev. Each entry:
+//   { package_name, version, reason: 'MALWARE'|'TELEMETRY'|'PROTESTWARE' }
+// Source: https://github.com/AikidoSec/safe-chain (open-source consumer)
+// ============================================
+async function scrapeAikidoMalwareFeed() {
+  console.log('[SCRAPER] Aikido Open Source Malware Feed...');
+  const npmPackages = [];
+  const pypiPackages = [];
+
+  // npm
+  try {
+    const { status, data } = await fetchJSON('https://malware-list.aikido.dev/malware_predictions.json');
+    if (status === 200 && Array.isArray(data)) {
+      for (const entry of data) {
+        if (!entry || typeof entry.package_name !== 'string') continue;
+        // Only keep MALWARE; TELEMETRY/PROTESTWARE are policy decisions, not security
+        if (entry.reason !== 'MALWARE') continue;
+        const ver = (entry.version && entry.version !== '') ? String(entry.version) : '*';
+        npmPackages.push({
+          id: 'AIKIDO-' + entry.package_name + '-' + ver,
+          name: entry.package_name,
+          version: ver,
+          severity: 'critical',
+          confidence: 'high',
+          source: 'aikido',
+          description: 'Flagged by Aikido Open Source Malware Feed',
+          references: ['https://malware-list.aikido.dev/malware_predictions.json',
+                       'https://www.aikido.dev/code/malware-detection-in-dependencies'],
+          mitre: 'T1195.002',
+          freshness: createFreshness('aikido', 'high')
+        });
+      }
+      console.log('[SCRAPER]   ' + npmPackages.length + ' npm MALWARE entries from Aikido');
+    } else {
+      console.log('[SCRAPER]   Aikido npm feed: HTTP ' + status);
+    }
+  } catch (e) {
+    console.log('[SCRAPER]   Aikido npm error: ' + e.message);
+  }
+
+  // PyPI
+  try {
+    const { status, data } = await fetchJSON('https://malware-list.aikido.dev/malware_pypi.json');
+    if (status === 200 && Array.isArray(data)) {
+      for (const entry of data) {
+        if (!entry || typeof entry.package_name !== 'string') continue;
+        if (entry.reason !== 'MALWARE') continue;
+        const ver = (entry.version && entry.version !== '') ? String(entry.version) : '*';
+        pypiPackages.push({
+          id: 'AIKIDO-PYPI-' + entry.package_name + '-' + ver,
+          name: entry.package_name,
+          version: ver,
+          severity: 'critical',
+          confidence: 'high',
+          source: 'aikido',
+          description: 'Flagged by Aikido Open Source Malware Feed',
+          references: ['https://malware-list.aikido.dev/malware_pypi.json'],
+          mitre: 'T1195.002',
+          freshness: createFreshness('aikido', 'high')
+        });
+      }
+      console.log('[SCRAPER]   ' + pypiPackages.length + ' PyPI MALWARE entries from Aikido');
+    } else {
+      console.log('[SCRAPER]   Aikido PyPI feed: HTTP ' + status);
+    }
+  } catch (e) {
+    console.log('[SCRAPER]   Aikido PyPI error: ' + e.message);
+  }
+
+  return { packages: npmPackages, pypi_packages: pypiPackages };
+}
+
 // ============================================
 // SOURCE 5: OSV.dev Lightweight API
 // Used by `muaddib update` (fast, no zip download)
@@ -1388,9 +1501,36 @@ async function queryOSVBatch(packageNames) {
 function getNoVersionSkipCount() { return _noVersionSkipCount; }
 function resetNoVersionSkipCount() { _noVersionSkipCount = 0; }
 
+/**
+ * Source-aware confidence: a package reported by N distinct feeds is more
+ * confident than one reported by a single source. Used by webhook gating
+ * and the /diff command to prioritize multi-confirmed alerts.
+ *
+ * Tiers:
+ *   N >= 3 → 'high'   (cross-confirmed, alert immediately)
+ *   N === 2 → 'medium' (single-corroboration, alert with sandbox confirm)
+ *   N <= 1 → 'low'    (single-feed only, log + sandbox before alert)
+ *
+ * @param {object} pkg - IOC package entry (with optional `sources` array)
+ * @returns {{ tier: 'high'|'medium'|'low', count: number, sources: string[] }}
+ */
+function getSourceConfidence(pkg) {
+  if (!pkg) return { tier: 'low', count: 0, sources: [] };
+  const sources = Array.isArray(pkg.sources) && pkg.sources.length > 0
+    ? pkg.sources.map(s => s.name || 'unknown')
+    : [pkg.source || (pkg.freshness && pkg.freshness.source) || 'unknown'];
+  const unique = Array.from(new Set(sources));
+  let tier = 'low';
+  if (unique.length >= 3) tier = 'high';
+  else if (unique.length === 2) tier = 'medium';
+  return { tier, count: unique.length, sources: unique };
+}
+
 module.exports = {
   runScraper, scrapeShaiHuludDetector, scrapeDatadogIOCs,
+  scrapeAikidoMalwareFeed,
   scrapeOSVLightweightAPI, queryOSVBatch,
+  getSourceConfidence,
   // Pure utility functions (exported for testing)
   parseCSVLine, parseCSV, extractVersions, parseOSVEntry,
   createFreshness, isAllowedRedirect,

package/src/ioc/updater.js

@@ -136,7 +136,9 @@ function mergeIOCs(target, source) {
     target._hashSet = new Set(target.hashes);
     target._markerSet = new Set(target.markers);
     target._fileSet = new Set(target.files);
+    target._stringIocSet = new Set((target.stringIocs || []).map(s => s.string));
   }
+  if (!target.stringIocs) target.stringIocs = [];
 
   let added = 0;
 
@@ -184,6 +186,16 @@ function mergeIOCs(target, source) {
     }
   }
 
+  // Merge string IOCs (YARA-style)
+  for (const sIoc of source.stringIocs || []) {
+    const literal = sIoc && typeof sIoc.string === 'string' ? sIoc.string : null;
+    if (!literal) continue;
+    if (!target._stringIocSet.has(literal)) {
+      target.stringIocs.push(sIoc);
+      target._stringIocSet.add(literal);
+    }
+  }
+
   return added;
 }
 
@@ -207,7 +219,9 @@ function loadCachedIOCs() {
     pypi_packages: [],
     hashes: yamlIOCs.hashes.map(function(h) { return h.sha256; }),
     markers: yamlIOCs.markers.map(function(m) { return m.pattern; }),
-    files: yamlIOCs.files.map(function(f) { return f.name; })
+    files: yamlIOCs.files.map(function(f) { return f.name; }),
+    // string-IOCs from string-iocs.yaml (YARA-style high-precision artifacts)
+    stringIocs: Array.isArray(yamlIOCs.stringIocs) ? [...yamlIOCs.stringIocs] : []
   };
 
   // Priority 2a: Local scraped IOCs (full enriched file)
@@ -349,6 +363,11 @@ function createOptimizedIOCs(iocs) {
   // Set for suspicious files
   const filesSet = new Set(iocs.files);
 
+  // String IOCs (YARA-style): keep both array (for metadata) and Map keyed by string
+  // for O(1) campaign lookup once a substring match has been confirmed.
+  const stringIocsArr = Array.isArray(iocs.stringIocs) ? iocs.stringIocs : [];
+  const stringIocsMap = new Map(stringIocsArr.map(s => [s.string, s]));
+
   return {
     // Optimized structures (npm)
     packagesMap,
@@ -360,6 +379,9 @@ function createOptimizedIOCs(iocs) {
     hashesSet,
     markersSet,
     filesSet,
+    // String IOCs (YARA-style)
+    stringIocs: stringIocsArr,
+    stringIocsMap,
     // Original arrays for compatibility
     packages: iocs.packages,
     pypi_packages: iocs.pypi_packages || [],

package/src/ioc/yaml-loader.js

@@ -34,7 +34,10 @@ function loadYAMLIOCs() {
     packages: [],
     hashes: [],
     markers: [],
-    files: []
+    files: [],
+    // string-IOCs (YARA-style high-precision artifacts) — see iocs/string-iocs.yaml
+    // Each entry: { string, campaign, severity, source, description }
+    stringIocs: []
   };
 
   // Dedup sets for O(1) lookup during loading
@@ -42,6 +45,7 @@ function loadYAMLIOCs() {
   const seenHashes = new Set();
   const seenMarkers = new Set();
   const seenFiles = new Set();
+  const seenStrings = new Set();
 
   // Charger packages.yaml
   loadPackagesYAML(path.join(IOCS_DIR, 'packages.yaml'), iocs, seenPkgs);
@@ -52,9 +56,53 @@ function loadYAMLIOCs() {
   // Charger hashes.yaml
   loadHashesYAML(path.join(IOCS_DIR, 'hashes.yaml'), iocs, seenHashes, seenMarkers, seenFiles);
 
+  // Charger string-iocs.yaml (YARA-style)
+  loadStringIocsYAML(path.join(IOCS_DIR, 'string-iocs.yaml'), iocs, seenStrings);
+
   return iocs;
 }
 
+/**
+ * Load YARA-style string IOCs from string-iocs.yaml.
+ * Each entry must satisfy the inclusion criteria documented in that file:
+ *   - length >= 6 chars
+ *   - confirmed in >= 1 sample malware
+ *   - absent from benign corpus
+ *   - unique enough that substring match is decisive
+ * Length floor enforced here (defense-in-depth) — anything shorter is dropped
+ * silently with a single warning to avoid spamming the console.
+ */
+function loadStringIocsYAML(filePath, iocs, seenStrings) {
+  if (!fs.existsSync(filePath)) return;
+  const MIN_STRING_LEN = 6;
+  let dropped = 0;
+
+  try {
+    const data = yaml.load(readVerifiedYAML(filePath), { schema: yaml.JSON_SCHEMA });
+    if (!data || !Array.isArray(data.strings)) return;
+
+    for (const s of data.strings) {
+      if (!s || typeof s.string !== 'string') { dropped++; continue; }
+      const literal = s.string;
+      if (literal.length < MIN_STRING_LEN) { dropped++; continue; }
+      if (seenStrings.has(literal)) continue;
+      seenStrings.add(literal);
+      iocs.stringIocs.push({
+        string: literal,
+        campaign: typeof s.campaign === 'string' ? s.campaign : 'unknown',
+        severity: (s.severity === 'HIGH' || s.severity === 'MEDIUM') ? s.severity : 'CRITICAL',
+        source: typeof s.source === 'string' ? s.source : '',
+        description: typeof s.description === 'string' ? s.description : ''
+      });
+    }
+    if (dropped > 0) {
+      console.error(`[WARN] string-iocs.yaml: ${dropped} entries dropped (missing string field or below ${MIN_STRING_LEN} chars)`);
+    }
+  } catch (e) {
+    console.error('[WARN] Erreur parsing string-iocs.yaml:', e.message);
+  }
+}
+
 function loadPackagesYAML(filePath, iocs, seenPkgs) {
   if (!fs.existsSync(filePath)) return;
 

package/src/monitor/daemon.js

@@ -5,7 +5,7 @@ const os = require('os');
 const v8 = require('v8');
 const { isDockerAvailable, SANDBOX_CONCURRENCY_MAX } = require('../sandbox/index.js');
 const { setVerboseMode, isSandboxEnabled, isCanaryEnabled, isLlmDetectiveEnabled, getLlmDetectiveMode, DOWNLOADS_CACHE_TTL } = require('./classify.js');
-const { loadState, saveState, loadDailyStats, saveDailyStats, purgeTarballCache, getParisHour, atomicWriteFileSync, saveNpmSeq, ALERTS_FILE } = require('./state.js');
+const { loadState, saveState, loadDailyStats, saveDailyStats, purgeTarballCache, getParisHour, atomicWriteFileSync, saveNpmSeq, ALERTS_FILE, runStateMigrations } = require('./state.js');
 const { isTemporalEnabled, isTemporalAstEnabled, isTemporalPublishEnabled, isTemporalMaintainerEnabled } = require('./temporal.js');
 const { pendingGrouped, flushScopeGroup, sendDailyReport, DAILY_REPORT_HOUR, alertedPackageRules } = require('./webhook.js');
 const { poll } = require('./ingestion.js');
@@ -14,6 +14,12 @@ const { computeTarget, ADJUST_INTERVAL_MS, BASE_CONCURRENCY, resetDeltas } = req
 const { startHealthcheck } = require('./healthcheck.js');
 const { startDeferredWorker, stopDeferredWorker, persistDeferredQueue, restoreDeferredQueue, clearDeferredQueue } = require('./deferred-sandbox.js');
 const { clearMetadataCache } = require('../scanner/temporal-analysis.js');
+// Caches not previously cleared by handleMemoryPressure (OOM fix). These live
+// in the main thread and are populated by temporal-ast-diff and the typosquat
+// scanner, neither of which runs in the static-scan worker.
+const { clearMetadataCache: clearTyposquatMetadataCache } = require('../scanner/typosquat.js');
+const { clearFileListCache } = require('../utils.js');
+const { clearASTCache } = require('../shared/constants.js');
 
 const POLL_INTERVAL = 60_000;
 const PROCESS_LOOP_INTERVAL = 2_000;    // Queue check interval when empty
@@ -401,6 +407,13 @@ function handleMemoryPressure(level, ratio, recentlyScanned, downloadsCache, sca
     console.error(`[MONITOR] MEMORY PRESSURE CRITICAL: heap at ${pct}% — stopping ingestion, clearing scanner caches`);
     // temporal-analysis._metadataCache (200 entries × full npm registry metadata)
     try { clearMetadataCache(); } catch {}
+    // typosquat metadataCache (500 entries × npm registry metadata for typosquat scoring)
+    try { clearTyposquatMetadataCache(); } catch {}
+    // utils._fileListCache, utils._fileContentCache, shared/constants._astCache
+    // — populated by temporal-ast-diff (main-thread tarball download + AST parse).
+    // Each AST entry can be MB-sized for bundled outputs.
+    try { clearFileListCache(); } catch {}
+    try { clearASTCache(); } catch {}
     // pendingGrouped webhook buffers
     for (const [scope, group] of pendingGrouped) {
       clearTimeout(group.timer);
@@ -567,6 +580,13 @@ async function startMonitor(options, stats, dailyAlerts, recentlyScanned, downlo
   // External healthcheck (Healthchecks.io) — sends /start ping now, heartbeat every 10 min
   const healthcheck = startHealthcheck();
 
+  // OOM fix: convert legacy detections.json / temporal-detections.json into
+  // append-only JSONL on first boot after upgrade. Idempotent and safe to call
+  // every boot (skips when JSONL already exists).
+  try { runStateMigrations(); } catch (err) {
+    console.error(`[MONITOR] runStateMigrations failed: ${err.message}`);
+  }
+
   const state = loadState(stats);
   loadDailyStats(stats, dailyAlerts); // Restore counters from previous run (survives restarts)
   console.log(`[MONITOR] State loaded — npm last: ${state.npmLastPackage || 'none'}, pypi last: ${state.pypiLastPackage || 'none'}, npm seq: ${state.npmLastSeq || 'none'}`);