muaddib-scanner 1.6.5 → 1.6.7

package/docker/Dockerfile

@@ -10,9 +10,9 @@ RUN adduser -D sandboxuser
 WORKDIR /sandbox
 RUN chown sandboxuser:sandboxuser /sandbox
 
-# Script d'analyse
+# Script d'analyse (sed strips Windows CRLF line endings)
 COPY sandbox-runner.sh /sandbox/
-RUN chmod +x /sandbox/sandbox-runner.sh
+RUN sed -i 's/\r$//' /sandbox/sandbox-runner.sh && chmod +x /sandbox/sandbox-runner.sh
 
 USER sandboxuser
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "muaddib-scanner",
-  "version": "1.6.5",
+  "version": "1.6.7",
   "description": "Supply-chain threat detection & response for npm",
   "main": "src/index.js",
   "bin": {

package/src/ioc/scraper.js

@@ -7,6 +7,7 @@ const IOC_FILE = path.join(__dirname, 'data/iocs.json');
 const COMPACT_IOC_FILE = path.join(__dirname, 'data/iocs-compact.json');
 const STATIC_IOCS_FILE = path.join(__dirname, 'data/static-iocs.json');
 const { generateCompactIOCs } = require('./updater.js');
+const { Spinner } = require('../utils.js');
 
 // Allowed domains for redirections (SSRF security)
 const ALLOWED_REDIRECT_DOMAINS = [
@@ -266,7 +267,7 @@ function fetchBuffer(url, redirectCount = 0) {
 }
 
 /**
- * Download a large file with progress feedback (MB received every 5s).
+ * Download a large file with spinner progress (npm/ora style).
  * Used for bulk zip downloads (OSV npm/PyPI ~50-100MB each).
  */
 function fetchBufferWithProgress(url, label, redirectCount = 0) {
@@ -302,30 +303,27 @@ function fetchBufferWithProgress(url, label, redirectCount = 0) {
       }
 
       const totalSize = parseInt(res.headers['content-length'], 10) || 0;
+      const totalMb = totalSize > 0 ? Math.round(totalSize / 1024 / 1024) : null;
       const chunks = [];
       let received = 0;
-      let lastLog = Date.now();
+
+      const spinner = new Spinner();
+      spinner.start('Downloading ' + label + '...');
 
       res.on('data', (chunk) => {
         chunks.push(chunk);
         received += chunk.length;
-        const now = Date.now();
-        if (now - lastLog >= 5000) {
-          const mb = (received / 1024 / 1024).toFixed(1);
-          if (totalSize > 0) {
-            const totalMb = (totalSize / 1024 / 1024).toFixed(1);
-            const pct = Math.round((received / totalSize) * 100);
-            process.stdout.write(`\r[SCRAPER]   ${label}: ${mb}MB / ${totalMb}MB (${pct}%)`);
-          } else {
-            process.stdout.write(`\r[SCRAPER]   ${label}: ${mb}MB received`);
-          }
-          lastLog = now;
+        const mb = Math.round(received / 1024 / 1024);
+        if (totalMb) {
+          spinner.update('Downloading ' + label + '... ' + mb + 'MB/' + totalMb + 'MB');
+        } else {
+          spinner.update('Downloading ' + label + '... ' + mb + 'MB');
         }
       });
 
       res.on('end', () => {
-        const mb = (received / 1024 / 1024).toFixed(1);
-        process.stdout.write(`\r[SCRAPER]   ${label}: ${mb}MB — done\n`);
+        const mb = Math.round(received / 1024 / 1024);
+        spinner.succeed('Downloaded ' + label + ' (' + mb + 'MB)');
         resolve(Buffer.concat(chunks));
       });
     });
@@ -613,6 +611,12 @@ async function scrapeOSSFMaliciousPackages(knownIds) {
 
     // Step 4: Batch fetch (50 concurrent, with small delay between batches for rate limit)
     const BATCH_SIZE = 50;
+    let fetchSpinner = null;
+    if (toFetch.length > 0) {
+      fetchSpinner = new Spinner();
+      fetchSpinner.start('Fetching OSSF entries... 0/' + toFetch.length);
+    }
+
     for (let i = 0; i < toFetch.length; i += BATCH_SIZE) {
       const batch = toFetch.slice(i, i + BATCH_SIZE);
       const results = await Promise.all(batch.map(function(entry) {
@@ -628,7 +632,7 @@ async function scrapeOSSFMaliciousPackages(knownIds) {
 
       // Progress
       const progress = Math.min(i + BATCH_SIZE, toFetch.length);
-      process.stdout.write('\r[SCRAPER]   Fetched ' + progress + '/' + toFetch.length);
+      fetchSpinner.update('Fetching OSSF entries... ' + progress + '/' + toFetch.length);
 
       // Small delay between batches to respect rate limits
       if (i + BATCH_SIZE < toFetch.length) {
@@ -636,12 +640,12 @@ async function scrapeOSSFMaliciousPackages(knownIds) {
       }
     }
 
-    if (toFetch.length > 0) console.log('');
+    if (fetchSpinner) {
+      fetchSpinner.succeed('Fetched OSSF entries: ' + packages.length + ' packages');
+    }
 
     // Save tree SHA for next incremental run
     try { fs.writeFileSync(shaFile, treeSha); } catch {}
-
-    console.log('[SCRAPER]   ' + packages.length + ' packages extracted');
   } catch (e) {
     console.log('[SCRAPER]   Error: ' + e.message);
   }
@@ -654,7 +658,6 @@ async function scrapeOSSFMaliciousPackages(knownIds) {
 // Bulk zip download — primary volume source
 // ============================================
 async function scrapeOSVDataDump() {
-  console.log('[SCRAPER] OSV.dev npm data dump (all.zip)...');
   const packages = [];
   const knownIds = new Set();
 
@@ -666,35 +669,42 @@ async function scrapeOSVDataDump() {
     // Extract using adm-zip
     const zip = new AdmZip(zipBuffer);
     const entries = zip.getEntries();
+    const total = entries.length;
 
     let malCount = 0;
     let skippedCount = 0;
 
-    for (const entry of entries) {
+    const spinner = new Spinner();
+    spinner.start('Parsing npm entries... 0/' + total);
+
+    for (let i = 0; i < entries.length; i++) {
+      const entry = entries[i];
       const name = entry.entryName;
 
       // Only process MAL-*.json files (malware), skip GHSA-*, CVE-*, PYSEC-* etc.
       if (!name.startsWith('MAL-') || !name.endsWith('.json')) {
         skippedCount++;
-        continue;
+      } else {
+        try {
+          const content = entry.getData().toString('utf8');
+          const vuln = JSON.parse(content);
+          const parsed = parseOSVEntry(vuln, 'osv-malicious');
+          packages.push(...parsed);
+
+          // Track known IDs so OSSF can skip them
+          knownIds.add(vuln.id || path.basename(name, '.json'));
+          malCount++;
+        } catch {
+          // Skip unparseable entries
+        }
       }
 
-      try {
-        const content = entry.getData().toString('utf8');
-        const vuln = JSON.parse(content);
-        const parsed = parseOSVEntry(vuln, 'osv-malicious');
-        packages.push(...parsed);
-
-        // Track known IDs so OSSF can skip them
-        knownIds.add(vuln.id || path.basename(name, '.json'));
-        malCount++;
-      } catch {
-        // Skip unparseable entries
+      if ((i + 1) % 1000 === 0 || i === entries.length - 1) {
+        spinner.update('Parsing npm entries... ' + (i + 1) + '/' + total);
       }
     }
 
-    console.log('[SCRAPER]   Processed ' + malCount + ' MAL-* entries (' + skippedCount + ' non-malware skipped)');
-    console.log('[SCRAPER]   ' + packages.length + ' packages extracted');
+    spinner.succeed('Parsed npm entries: ' + malCount + ' MAL-* (' + skippedCount + ' skipped) \u2192 ' + packages.length + ' packages');
   } catch (e) {
     console.log('[SCRAPER]   Error: ' + e.message);
   }
@@ -707,7 +717,6 @@ async function scrapeOSVDataDump() {
 // Bulk zip download — PyPI malicious packages
 // ============================================
 async function scrapeOSVPyPIDataDump() {
-  console.log('[SCRAPER] OSV.dev PyPI data dump (all.zip)...');
   const packages = [];
 
   try {
@@ -716,32 +725,39 @@ async function scrapeOSVPyPIDataDump() {
 
     const zip = new AdmZip(zipBuffer);
     const entries = zip.getEntries();
+    const total = entries.length;
 
     let malCount = 0;
     let skippedCount = 0;
 
-    for (const entry of entries) {
+    const spinner = new Spinner();
+    spinner.start('Parsing PyPI entries... 0/' + total);
+
+    for (let i = 0; i < entries.length; i++) {
+      const entry = entries[i];
       const name = entry.entryName;
 
       // Only process MAL-*.json files (malware)
       if (!name.startsWith('MAL-') || !name.endsWith('.json')) {
         skippedCount++;
-        continue;
+      } else {
+        try {
+          const content = entry.getData().toString('utf8');
+          const vuln = JSON.parse(content);
+          const parsed = parseOSVEntry(vuln, 'osv-malicious-pypi', 'PyPI');
+          packages.push(...parsed);
+          malCount++;
+        } catch {
+          // Skip unparseable entries
+        }
       }
 
-      try {
-        const content = entry.getData().toString('utf8');
-        const vuln = JSON.parse(content);
-        const parsed = parseOSVEntry(vuln, 'osv-malicious-pypi', 'PyPI');
-        packages.push(...parsed);
-        malCount++;
-      } catch {
-        // Skip unparseable entries
+      if ((i + 1) % 1000 === 0 || i === entries.length - 1) {
+        spinner.update('Parsing PyPI entries... ' + (i + 1) + '/' + total);
       }
     }
 
-    console.log('[SCRAPER]   Processed ' + malCount + ' PyPI MAL-* entries (' + skippedCount + ' non-malware skipped)');
-    console.log('[SCRAPER]   ' + packages.length + ' PyPI packages extracted');
+    spinner.succeed('Parsed PyPI entries: ' + malCount + ' MAL-* (' + skippedCount + ' skipped) \u2192 ' + packages.length + ' packages');
   } catch (e) {
     console.log('[SCRAPER]   Error: ' + e.message);
   }
@@ -1106,11 +1122,15 @@ async function runScraper() {
   ];
 
   // Save enriched (full) IOCs
+  const saveSpinner = new Spinner();
+  saveSpinner.start('Saving IOCs...');
   fs.writeFileSync(IOC_FILE, JSON.stringify(existingIOCs, null, 2));
 
   // Save compact IOCs (lightweight, shipped in npm)
+  saveSpinner.update('Generating compact IOCs...');
   const compactIOCs = generateCompactIOCs(existingIOCs);
   fs.writeFileSync(COMPACT_IOC_FILE, JSON.stringify(compactIOCs));
+  saveSpinner.succeed('Saved IOCs + compact format');
 
   // Display summary
   console.log('\n' + '='.repeat(60));
@@ -1163,7 +1183,7 @@ async function runScraper() {
   };
 }
 
-module.exports = { runScraper };
+module.exports = { runScraper, scrapeShaiHuludDetector, scrapeDatadogIOCs };
 
 // Direct execution if called as CLI
 if (require.main === module) {

package/src/ioc/updater.js

@@ -8,30 +8,77 @@ const LOCAL_COMPACT_FILE = path.join(__dirname, 'data/iocs-compact.json');
 const { loadYAMLIOCs } = require('./yaml-loader.js');
 
 async function updateIOCs() {
-  console.log('[MUADDIB] Updating IOCs via live scrape (OSV + OSSF + all sources)...\n');
+  console.log('[MUADDIB] Updating IOCs (fast mode)...\n');
 
-  // Run a full scrape — this downloads directly from OSV/OSSF/etc.
-  // and writes both iocs.json and iocs-compact.json
-  const { runScraper } = require('./scraper.js');
-  const result = await runScraper();
+  // Step 1: Load compact IOCs shipped in package (~225K IOCs)
+  let baseIOCs = { packages: [], pypi_packages: [], hashes: [], markers: [], files: [] };
 
-  // Also copy results to cache for loadCachedIOCs
+  if (fs.existsSync(LOCAL_COMPACT_FILE)) {
+    try {
+      const compactData = JSON.parse(fs.readFileSync(LOCAL_COMPACT_FILE, 'utf8'));
+      baseIOCs = expandCompactIOCs(compactData);
+      console.log('[1/4] Compact IOCs: ' + baseIOCs.packages.length + ' npm + ' + (baseIOCs.pypi_packages || []).length + ' PyPI');
+    } catch (e) {
+      console.log('[1/4] Error loading compact IOCs: ' + e.message);
+    }
+  } else {
+    console.log('[1/4] Compact IOCs not found (run "muaddib scrape" first for full data)');
+  }
+
+  // Step 2: Load YAML IOCs (builtin.yaml, packages.yaml, hashes.yaml)
+  const yamlIOCs = loadYAMLIOCs();
+  const yamlStandard = {
+    packages: yamlIOCs.packages || [],
+    pypi_packages: [],
+    hashes: (yamlIOCs.hashes || []).map(function(h) { return h.sha256; }),
+    markers: (yamlIOCs.markers || []).map(function(m) { return m.pattern; }),
+    files: (yamlIOCs.files || []).map(function(f) { return f.name; })
+  };
+  mergeIOCs(baseIOCs, yamlStandard);
+  console.log('[2/4] YAML IOCs: ' + yamlStandard.packages.length + ' packages, ' + yamlStandard.hashes.length + ' hashes');
+
+  // Step 3: Download additional IOCs from GitHub (GenSecAI + DataDog — small files, fast)
+  const { scrapeShaiHuludDetector, scrapeDatadogIOCs } = require('./scraper.js');
+  console.log('[3/4] Downloading GitHub IOCs...');
+
+  const [shaiHulud, datadog] = await Promise.all([
+    scrapeShaiHuludDetector(),
+    scrapeDatadogIOCs()
+  ]);
+
+  const githubIOCs = {
+    packages: [].concat(shaiHulud.packages, datadog.packages),
+    pypi_packages: [],
+    hashes: [].concat(shaiHulud.hashes || [], datadog.hashes || []),
+    markers: [],
+    files: []
+  };
+  mergeIOCs(baseIOCs, githubIOCs);
+  console.log('     +' + shaiHulud.packages.length + ' GenSecAI, +' + datadog.packages.length + ' DataDog');
+
+  // Step 4: Merge and save to cache
   if (!fs.existsSync(CACHE_PATH)) {
     fs.mkdirSync(CACHE_PATH, { recursive: true });
   }
 
-  if (fs.existsSync(LOCAL_IOC_FILE)) {
-    fs.copyFileSync(LOCAL_IOC_FILE, CACHE_IOC_FILE);
-  }
+  baseIOCs.updated = new Date().toISOString();
+  baseIOCs.sources = ['compact', 'yaml', 'shai-hulud-detector', 'datadog'];
 
-  const compactCachePath = path.join(CACHE_PATH, 'iocs-compact.json');
-  if (fs.existsSync(LOCAL_COMPACT_FILE)) {
-    fs.copyFileSync(LOCAL_COMPACT_FILE, compactCachePath);
-  }
+  // Clean internal dedup sets before serialization
+  delete baseIOCs._pkgKeys;
+  delete baseIOCs._pypiPkgKeys;
+  delete baseIOCs._hashSet;
+  delete baseIOCs._markerSet;
+  delete baseIOCs._fileSet;
+
+  fs.writeFileSync(CACHE_IOC_FILE, JSON.stringify(baseIOCs));
 
-  console.log('\n[OK] IOCs updated: ' + result.total + ' npm + ' + result.totalPyPI + ' PyPI packages');
+  const totalNpm = baseIOCs.packages.length;
+  const totalPyPI = (baseIOCs.pypi_packages || []).length;
+  console.log('[4/4] Saved to cache: ' + CACHE_IOC_FILE);
+  console.log('\n[OK] IOCs updated: ' + totalNpm + ' npm + ' + totalPyPI + ' PyPI packages');
 
-  return result;
+  return { total: totalNpm, totalPyPI: totalPyPI };
 }
 
 /**

package/src/utils.js

@@ -156,6 +156,55 @@ function getCallName(node) {
   return '';
 }
 
+/**
+ * Minimal CLI spinner (npm/ora style, no external deps).
+ * Frames rotate every 100ms via setInterval.
+ * Uses ANSI escapes to clear/rewrite the current line.
+ */
+class Spinner {
+  constructor() {
+    this._frames = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏'];
+    this._index = 0;
+    this._interval = null;
+    this._text = '';
+  }
+
+  start(text) {
+    this._text = text;
+    this._index = 0;
+    this._render();
+    this._interval = setInterval(() => this._render(), 100);
+    return this;
+  }
+
+  update(text) {
+    this._text = text;
+  }
+
+  succeed(text) {
+    this._stop();
+    process.stdout.write('\r\x1b[K\x1b[32m\u2713\x1b[0m ' + text + '\n');
+  }
+
+  fail(text) {
+    this._stop();
+    process.stdout.write('\r\x1b[K\x1b[31m\u2717\x1b[0m ' + text + '\n');
+  }
+
+  _render() {
+    const frame = this._frames[this._index % this._frames.length];
+    process.stdout.write('\r\x1b[K' + frame + ' ' + this._text);
+    this._index++;
+  }
+
+  _stop() {
+    if (this._interval) {
+      clearInterval(this._interval);
+      this._interval = null;
+    }
+  }
+}
+
 module.exports = {
   EXCLUDED_DIRS,
   DEV_PATTERNS,
@@ -163,5 +212,6 @@ module.exports = {
   findFiles,
   findJsFiles,
   escapeHtml,
-  getCallName
+  getCallName,
+  Spinner
 };