muaddib-scanner 2.2.23 → 2.2.25

package/README.fr.md

@@ -285,7 +285,7 @@ Ajoutez à `.pre-commit-config.yaml` :
 ```yaml
 repos:
   - repo: https://github.com/DNSZLSK/muad-dib
-    rev: v2.2.21
+    rev: v2.2.24
     hooks:
       - id: muaddib-scan        # Scanner toutes les menaces
       # - id: muaddib-diff      # Ou: seulement les nouvelles
@@ -640,7 +640,7 @@ Les alertes apparaissent dans Security > Code scanning alerts.
 ## Architecture
 
 ```
-MUAD'DIB 2.2.21 Scanner
+MUAD'DIB 2.2.24 Scanner
 |
 +-- IOC Match (225 000+ packages, JSON DB)
 |   +-- OSV.dev npm dump (200K+ entrées MAL-*)
@@ -748,7 +748,7 @@ Output (CLI, JSON, HTML, SARIF, Webhook, Threat Feed)
 - **ADR** (Adversarial Detection Rate) : taux de detection sur 78 samples malveillants evasifs — 38 adversariaux (4 vagues red team + 3 bypasses) + 40 holdouts (5 batches de 10, testant obfuscation, dataflow inter-module, etc.)
 - **Holdout** (pre-tuning) : taux de detection sur 10 samples jamais vus avec regles gelees (mesure de generalisation)
 
-Datasets : 529 npm + 132 PyPI packages benins, 78 samples adversariaux/holdout, 51 attaques ground-truth (65 packages malveillants documentes).
+Datasets : 529 npm + 132 PyPI packages benins, 78 samples adversariaux/holdout, 51 attaques ground-truth (65 packages malveillants documentes). **1317 tests**, 86% coverage.
 
 Voir [Evaluation Methodology](docs/EVALUATION_METHODOLOGY.md) pour le protocole experimental complet.
 
@@ -784,7 +784,7 @@ npm test
 
 ### Tests
 
-- **862 tests unitaires/integration** sur 20 fichiers modulaires - 74% coverage via [Codecov](https://codecov.io/gh/DNSZLSK/muad-dib)
+- **1317 tests unitaires/integration** sur 20 fichiers modulaires - 86% coverage via [Codecov](https://codecov.io/gh/DNSZLSK/muad-dib)
 - **56 tests de fuzzing** - YAML malforme, JSON invalide, fichiers binaires, ReDoS, unicode, inputs 10MB
 - **78 samples adversariaux/holdout** - 38 adversariaux + 40 holdouts, 78/78 taux de detection (100% ADR)
 - **Validation ground truth** - 51 attaques reelles (45/49 detectees = 91.8% TPR). 4 hors scope : browser-only (3) + risque FP (1)

package/README.md

@@ -285,7 +285,7 @@ Add to `.pre-commit-config.yaml`:
 ```yaml
 repos:
   - repo: https://github.com/DNSZLSK/muad-dib
-    rev: v2.2.21
+    rev: v2.2.24
     hooks:
       - id: muaddib-scan        # Scan all threats
       # - id: muaddib-diff      # Or: only new threats
@@ -641,7 +641,7 @@ Alerts appear in Security > Code scanning alerts.
 ## Architecture
 
 ```
-MUAD'DIB 2.2.21 Scanner
+MUAD'DIB 2.2.24 Scanner
 |
 +-- IOC Match (225,000+ packages, JSON DB)
 |   +-- OSV.dev npm dump (200K+ MAL-* entries)
@@ -751,7 +751,7 @@ Output (CLI, JSON, HTML, SARIF, Webhook, Threat Feed)
 - **ADR** (Adversarial Detection Rate): detection rate on 75 evasive malicious samples — 35 adversarial (4 red-team waves) + 40 holdout (5 batches of 10, testing obfuscation, inter-module dataflow, etc.)
 - **Holdout** (pre-tuning): detection rate on 10 unseen samples with rules frozen (measures generalization)
 
-Datasets: 529 npm + 132 PyPI benign packages, 78 adversarial/holdout samples, 51 ground-truth attacks (65 documented malware packages).
+Datasets: 529 npm + 132 PyPI benign packages, 78 adversarial/holdout samples, 51 ground-truth attacks (65 documented malware packages). **1317 tests**, 86% code coverage.
 
 See [Evaluation Methodology](docs/EVALUATION_METHODOLOGY.md) for the full experimental protocol.
 
@@ -787,7 +787,7 @@ npm test
 
 ### Testing
 
-- **862 unit/integration tests** across 20 modular test files - 74% code coverage via [Codecov](https://codecov.io/gh/DNSZLSK/muad-dib)
+- **1317 unit/integration tests** across 20 modular test files - 86% code coverage via [Codecov](https://codecov.io/gh/DNSZLSK/muad-dib)
 - **56 fuzz tests** - Malformed YAML, invalid JSON, binary files, ReDoS, unicode, 10MB inputs
 - **78 adversarial/holdout samples** - 38 adversarial + 40 holdout, 78/78 detection rate (100% ADR)
 - **Ground truth validation** - 51 real-world attacks (45/49 detected = 91.8% TPR). 4 out-of-scope: browser-only (3) + FP-risky (1)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "muaddib-scanner",
-  "version": "2.2.23",
+  "version": "2.2.25",
   "description": "Supply-chain threat detection & response for npm & PyPI/Python",
   "main": "src/index.js",
   "bin": {

package/src/daemon.js

@@ -1,178 +1,178 @@
-const fs = require('fs');
-const path = require('path');
-const { run } = require('./index.js');
-
-let webhookUrl = null;
-
-async function startDaemon(options = {}) {
-  webhookUrl = options.webhook || null;
-
-  console.log(`
-╔════════════════════════════════════════════╗
-║         MUAD'DIB Security Daemon           ║
-║      Surveillance npm install active       ║
-╚════════════════════════════════════════════╝
-  `);
-
-  console.log('[DAEMON] Demarrage...');
-  console.log(`[DAEMON] Webhook: ${webhookUrl ? 'Configure' : 'Non configure'}`);
-  console.log('[DAEMON] Ctrl+C pour arreter\n');
-
-  // Surveille le dossier courant
-  const cwd = process.cwd();
-  const watchers = watchDirectory(cwd);
-
-  // Cleanup function to close all watchers
-  function cleanup() {
-    for (const w of watchers) {
-      try { w.close(); } catch { /* ignore */ }
-    }
-  }
-
-  // Keep process alive until SIGINT
-  await new Promise((resolve) => {
-    process.once('SIGINT', () => {
-      console.log('\n[DAEMON] Arret...');
-      cleanup();
-      resolve();
-    });
-  });
-
-  process.exit(0);
-}
-
-function watchDirectory(dir) {
-  const watchers = [];
-  const nodeModulesPath = path.join(dir, 'node_modules');
-  const packageLockPath = path.join(dir, 'package-lock.json');
-  const yarnLockPath = path.join(dir, 'yarn.lock');
-
-  console.log(`[DAEMON] Surveillance de ${dir}`);
-
-  // Surveille package-lock.json
-  if (fs.existsSync(packageLockPath)) {
-    const w = watchFile(packageLockPath, dir);
-    if (w) watchers.push(w);
-  }
-
-  // Surveille yarn.lock
-  if (fs.existsSync(yarnLockPath)) {
-    const w = watchFile(yarnLockPath, dir);
-    if (w) watchers.push(w);
-  }
-
-  // Surveille node_modules
-  if (fs.existsSync(nodeModulesPath)) {
-    watchers.push(watchNodeModules(nodeModulesPath, dir));
-  }
-
-  // Surveille la creation de node_modules
-  if (process.platform === 'linux') {
-    console.log('[DAEMON] Note: recursive fs.watch may not work on Linux');
-  }
-
-  const dirWatcher = fs.watch(dir, (eventType, filename) => {
-    if (filename === 'node_modules' && eventType === 'rename') {
-      const nmPath = path.join(dir, 'node_modules');
-      if (fs.existsSync(nmPath)) {
-        console.log('[DAEMON] node_modules detecte, scan en cours...');
-        triggerScan(dir);
-      }
-    }
-    if (filename === 'package-lock.json' || filename === 'yarn.lock') {
-      console.log(`[DAEMON] ${filename} modifie, scan en cours...`);
-      triggerScan(dir);
-    }
-  });
-  dirWatcher.on('error', (err) => {
-    console.log(`[DAEMON] Watcher error on ${dir}: ${err.message}`);
-  });
-  watchers.push(dirWatcher);
-
-  return watchers;
-}
-
-function watchFile(filePath, projectDir) {
-  let lastMtime;
-  try {
-    lastMtime = fs.statSync(filePath).mtime.getTime();
-  } catch {
-    return null; // File deleted between existsSync and statSync
-  }
-
-  const watcher = fs.watch(filePath, (eventType) => {
-    if (eventType === 'change') {
-      try {
-        const currentMtime = fs.statSync(filePath).mtime.getTime();
-        if (currentMtime !== lastMtime) {
-          lastMtime = currentMtime;
-          console.log(`[DAEMON] ${path.basename(filePath)} modifie`);
-          triggerScan(projectDir);
-        }
-      } catch {
-        // File may have been deleted between watch trigger and stat
-      }
-    }
-  });
-  watcher.on('error', (err) => {
-    console.log(`[DAEMON] Watcher error on ${filePath}: ${err.message}`);
-  });
-  return watcher;
-}
-
-function watchNodeModules(nodeModulesPath, projectDir) {
-  const watcher = fs.watch(nodeModulesPath, { recursive: true }, (eventType, filename) => {
-    if (filename && filename.includes('package.json')) {
-      console.log(`[DAEMON] Nouveau package detecte: ${filename}`);
-      triggerScan(projectDir);
-    }
-  });
-  watcher.on('error', (err) => {
-    console.log(`[DAEMON] Watcher error on ${nodeModulesPath}: ${err.message}`);
-  });
-  return watcher;
-}
-
-// Per-directory scan state to prevent cross-directory scan suppression
-const scanState = new Map();
-
-function getScanState(dir) {
-  if (!scanState.has(dir)) {
-    scanState.set(dir, { timeout: null, lastScanTime: 0 });
-  }
-  return scanState.get(dir);
-}
-
-function triggerScan(dir) {
-  const now = Date.now();
-  const state = getScanState(dir);
-
-  // Debounce: attend 3 secondes avant de scanner
-  if (state.timeout) {
-    clearTimeout(state.timeout);
-  }
-
-  // Evite les scans trop frequents (minimum 10 secondes entre chaque)
-  if (now - state.lastScanTime < 10000) {
-    state.timeout = setTimeout(() => triggerScan(dir), 10000 - (now - state.lastScanTime));
-    return;
-  }
-
-  state.timeout = setTimeout(async () => {
-    state.lastScanTime = Date.now();
-    console.log(`\n[DAEMON] ========== SCAN AUTOMATIQUE ==========`);
-    console.log(`[DAEMON] Cible: ${dir}`);
-    console.log(`[DAEMON] Heure: ${new Date().toLocaleTimeString()}\n`);
-
-    try {
-      await run(dir, { webhook: webhookUrl });
-    } catch (err) {
-      console.log(`[DAEMON] Erreur scan: ${err.message}`);
-    }
-
-    console.log(`\n[DAEMON] ======================================\n`);
-    console.log('[DAEMON] En attente de modifications...');
-  }, 3000);
-}
-
-module.exports = { startDaemon };
+const fs = require('fs');
+const path = require('path');
+const { run } = require('./index.js');
+
+let webhookUrl = null;
+
+async function startDaemon(options = {}) {
+  webhookUrl = options.webhook || null;
+
+  console.log(`
+╔════════════════════════════════════════════╗
+║         MUAD'DIB Security Daemon           ║
+║      Surveillance npm install active       ║
+╚════════════════════════════════════════════╝
+  `);
+
+  console.log('[DAEMON] Demarrage...');
+  console.log(`[DAEMON] Webhook: ${webhookUrl ? 'Configure' : 'Non configure'}`);
+  console.log('[DAEMON] Ctrl+C pour arreter\n');
+
+  // Surveille le dossier courant
+  const cwd = process.cwd();
+  const watchers = watchDirectory(cwd);
+
+  // Cleanup function to close all watchers
+  function cleanup() {
+    for (const w of watchers) {
+      try { w.close(); } catch { /* ignore */ }
+    }
+  }
+
+  // Keep process alive until SIGINT
+  await new Promise((resolve) => {
+    process.once('SIGINT', () => {
+      console.log('\n[DAEMON] Arret...');
+      cleanup();
+      resolve();
+    });
+  });
+
+  process.exit(0);
+}
+
+function watchDirectory(dir) {
+  const watchers = [];
+  const nodeModulesPath = path.join(dir, 'node_modules');
+  const packageLockPath = path.join(dir, 'package-lock.json');
+  const yarnLockPath = path.join(dir, 'yarn.lock');
+
+  console.log(`[DAEMON] Surveillance de ${dir}`);
+
+  // Surveille package-lock.json
+  if (fs.existsSync(packageLockPath)) {
+    const w = watchFile(packageLockPath, dir);
+    if (w) watchers.push(w);
+  }
+
+  // Surveille yarn.lock
+  if (fs.existsSync(yarnLockPath)) {
+    const w = watchFile(yarnLockPath, dir);
+    if (w) watchers.push(w);
+  }
+
+  // Surveille node_modules
+  if (fs.existsSync(nodeModulesPath)) {
+    watchers.push(watchNodeModules(nodeModulesPath, dir));
+  }
+
+  // Surveille la creation de node_modules
+  if (process.platform === 'linux') {
+    console.log('[DAEMON] Note: recursive fs.watch may not work on Linux');
+  }
+
+  const dirWatcher = fs.watch(dir, (eventType, filename) => {
+    if (filename === 'node_modules' && eventType === 'rename') {
+      const nmPath = path.join(dir, 'node_modules');
+      if (fs.existsSync(nmPath)) {
+        console.log('[DAEMON] node_modules detecte, scan en cours...');
+        triggerScan(dir);
+      }
+    }
+    if (filename === 'package-lock.json' || filename === 'yarn.lock') {
+      console.log(`[DAEMON] ${filename} modifie, scan en cours...`);
+      triggerScan(dir);
+    }
+  });
+  dirWatcher.on('error', (err) => {
+    console.log(`[DAEMON] Watcher error on ${dir}: ${err.message}`);
+  });
+  watchers.push(dirWatcher);
+
+  return watchers;
+}
+
+function watchFile(filePath, projectDir) {
+  let lastMtime;
+  try {
+    lastMtime = fs.statSync(filePath).mtime.getTime();
+  } catch {
+    return null; // File deleted between existsSync and statSync
+  }
+
+  const watcher = fs.watch(filePath, (eventType) => {
+    if (eventType === 'change') {
+      try {
+        const currentMtime = fs.statSync(filePath).mtime.getTime();
+        if (currentMtime !== lastMtime) {
+          lastMtime = currentMtime;
+          console.log(`[DAEMON] ${path.basename(filePath)} modifie`);
+          triggerScan(projectDir);
+        }
+      } catch {
+        // File may have been deleted between watch trigger and stat
+      }
+    }
+  });
+  watcher.on('error', (err) => {
+    console.log(`[DAEMON] Watcher error on ${filePath}: ${err.message}`);
+  });
+  return watcher;
+}
+
+function watchNodeModules(nodeModulesPath, projectDir) {
+  const watcher = fs.watch(nodeModulesPath, { recursive: true }, (eventType, filename) => {
+    if (filename && filename.includes('package.json')) {
+      console.log(`[DAEMON] Nouveau package detecte: ${filename}`);
+      triggerScan(projectDir);
+    }
+  });
+  watcher.on('error', (err) => {
+    console.log(`[DAEMON] Watcher error on ${nodeModulesPath}: ${err.message}`);
+  });
+  return watcher;
+}
+
+// Per-directory scan state to prevent cross-directory scan suppression
+const scanState = new Map();
+
+function getScanState(dir) {
+  if (!scanState.has(dir)) {
+    scanState.set(dir, { timeout: null, lastScanTime: 0 });
+  }
+  return scanState.get(dir);
+}
+
+function triggerScan(dir) {
+  const now = Date.now();
+  const state = getScanState(dir);
+
+  // Debounce: attend 3 secondes avant de scanner
+  if (state.timeout) {
+    clearTimeout(state.timeout);
+  }
+
+  // Evite les scans trop frequents (minimum 10 secondes entre chaque)
+  if (now - state.lastScanTime < 10000) {
+    state.timeout = setTimeout(() => triggerScan(dir), 10000 - (now - state.lastScanTime));
+    return;
+  }
+
+  state.timeout = setTimeout(async () => {
+    state.lastScanTime = Date.now();
+    console.log(`\n[DAEMON] ========== SCAN AUTOMATIQUE ==========`);
+    console.log(`[DAEMON] Cible: ${dir}`);
+    console.log(`[DAEMON] Heure: ${new Date().toLocaleTimeString()}\n`);
+
+    try {
+      await run(dir, { webhook: webhookUrl });
+    } catch (err) {
+      console.log(`[DAEMON] Erreur scan: ${err.message}`);
+    }
+
+    console.log(`\n[DAEMON] ======================================\n`);
+    console.log('[DAEMON] En attente de modifications...');
+  }, 3000);
+}
+
+module.exports = { startDaemon, watchDirectory, watchFile, watchNodeModules, triggerScan, getScanState };

package/src/ioc/scraper.js

@@ -1089,6 +1089,8 @@ async function runScraper() {
 
   // Smart deduplication: build map of best entry per key
   // For duplicates, keep the one with highest confidence, then most recent date
+  const dedupSpinner = new Spinner();
+  dedupSpinner.start('Deduplicating ' + allPackages.length + ' npm + ' + pypiPackages.length + ' PyPI entries...');
   const dedupMap = new Map();
 
   // Seed with existing IOCs
@@ -1182,6 +1184,8 @@ async function runScraper() {
     ];
   }
 
+  dedupSpinner.succeed('Deduplicated: ' + existingIOCs.packages.length + ' npm + ' + existingIOCs.pypi_packages.length + ' PyPI packages (' + addedPackages + ' new, ' + upgradedPackages + ' upgraded)');
+
   // Update metadata
   existingIOCs.updated = new Date().toISOString();
   existingIOCs.sources = [
@@ -1278,7 +1282,13 @@ async function runScraper() {
   };
 }
 
-module.exports = { runScraper, scrapeShaiHuludDetector, scrapeDatadogIOCs };
+module.exports = {
+  runScraper, scrapeShaiHuludDetector, scrapeDatadogIOCs,
+  // Pure utility functions (exported for testing)
+  parseCSVLine, parseCSV, extractVersions, parseOSVEntry,
+  createFreshness, isAllowedRedirect, loadStaticIOCs,
+  CONFIDENCE_ORDER, ALLOWED_REDIRECT_DOMAINS
+};
 
 // Direct execution if called as CLI
 if (require.main === module) {