muaddib-scanner 2.10.40 → 2.10.42

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "muaddib-scanner",
-  "version": "2.10.40",
+  "version": "2.10.42",
   "description": "Supply-chain threat detection & response for npm & PyPI/Python",
   "main": "src/index.js",
   "bin": {

package/src/integrations/canary-tokens.js

@@ -10,6 +10,46 @@ function generateBase32(length) {
   return Array.from(bytes).map(b => chars[b % 32]).join('');
 }
 
+// Minimum consecutive chars from an encoded variant to match in a DNS domain.
+// 8 chars avoids FP on short legitimate hex subdomains (e.g. commit SHAs, CDN hashes).
+const MIN_ENCODED_MATCH = 8;
+
+/**
+ * Generate hex/base64/base64url encoded variants of a token value.
+ * Attackers encode tokens before DNS exfiltration to evade raw string matching.
+ * @param {string} value - Raw token value
+ * @returns {Array<{encoding: string, encoded: string}>}
+ */
+function generateEncodedVariants(value) {
+  const buf = Buffer.from(value);
+  return [
+    { encoding: 'hex', encoded: buf.toString('hex') },
+    { encoding: 'base64', encoded: buf.toString('base64') },
+    { encoding: 'base64url', encoded: buf.toString('base64url') }
+  ];
+}
+
+/**
+ * Check if a DNS domain contains an encoded token variant.
+ * Strips dots to reassemble data chunked across DNS labels (RFC 1035: max 63 chars/label).
+ * @param {string} domain - Full DNS query domain
+ * @param {Array<{encoding: string, encoded: string}>} variants
+ * @returns {{encoding: string, match: string}|null}
+ */
+function findEncodedInDomain(domain, variants) {
+  const stripped = domain.replace(/\./g, '');
+  for (const { encoding, encoded } of variants) {
+    if (encoded.length < MIN_ENCODED_MATCH) continue;
+    for (let i = 0; i <= encoded.length - MIN_ENCODED_MATCH; i++) {
+      const chunk = encoded.substring(i, i + MIN_ENCODED_MATCH);
+      if (stripped.includes(chunk)) {
+        return { encoding, match: chunk };
+      }
+    }
+  }
+  return null;
+}
+
 /**
  * Canary token generators.
  * Each generator produces a format-valid token that matches the real service format.
@@ -173,6 +213,7 @@ function detectCanaryExfiltration(networkLogs, tokens) {
   for (const domain of (networkLogs.dns_queries || [])) {
     if (!domain) continue;
     for (const [tokenName, tokenValue] of tokenEntries) {
+      // Raw value match
       if (domain.includes(tokenValue)) {
         exfiltrations.push({
           token: tokenName,
@@ -180,6 +221,18 @@ function detectCanaryExfiltration(networkLogs, tokens) {
           foundIn: `DNS query: ${domain}`,
           severity: 'CRITICAL'
         });
+        continue;
+      }
+      // Encoded variant match (hex, base64, base64url — catches DNS label chunking)
+      const variants = generateEncodedVariants(tokenValue);
+      const encodedMatch = findEncodedInDomain(domain, variants);
+      if (encodedMatch) {
+        exfiltrations.push({
+          token: tokenName,
+          value: tokenValue,
+          foundIn: `DNS query (${encodedMatch.encoding}-encoded): ${domain}`,
+          severity: 'CRITICAL'
+        });
       }
     }
   }

package/src/monitor/daemon.js

@@ -12,6 +12,8 @@ const { processQueue, SCAN_CONCURRENCY } = require('./queue.js');
 const { startHealthcheck } = require('./healthcheck.js');
 
 const POLL_INTERVAL = 60_000;
+const PROCESS_LOOP_INTERVAL = 2_000;    // Queue check interval when empty
+const QUEUE_WARNING_THRESHOLD = 5_000;  // Warn if queue depth exceeds this
 
 function sleep(ms) {
   return new Promise((resolve) => setTimeout(resolve, ms));
@@ -171,15 +173,21 @@ async function startMonitor(options, stats, dailyAlerts, recentlyScanned, downlo
   console.log('[MONITOR] npm changes stream enabled (replicate.npmjs.com) with RSS fallback');
   console.log(`[MONITOR] Scan concurrency: ${SCAN_CONCURRENCY} (MUADDIB_SCAN_CONCURRENCY to override)`);
   console.log(`[MONITOR] Sandbox concurrency: ${SANDBOX_CONCURRENCY_MAX} (MUADDIB_SANDBOX_CONCURRENCY to override)`);
-  console.log(`[MONITOR] Polling every ${POLL_INTERVAL / 1000}s. Ctrl+C to stop.\n`);
+  console.log(`[MONITOR] Polling every ${POLL_INTERVAL / 1000}s (decoupled from processing). Ctrl+C to stop.\n`);
 
   let running = true;
+  let pollIntervalHandle = null;  // Decoupled poll timer — set after initial poll
 
   // Graceful shutdown handler (shared by SIGINT and SIGTERM)
   // Daily report is NEVER sent on shutdown — it only fires at 08:00 Paris time.
   // Counters are persisted to disk so they survive the restart.
   async function gracefulShutdown(signal) {
     console.log(`\n[MONITOR] Received ${signal} — shutting down...`);
+    running = false;
+    if (pollIntervalHandle) {
+      clearInterval(pollIntervalHandle);
+      pollIntervalHandle = null;
+    }
     healthcheck.stop();
     // Flush all pending scope groups before exit
     for (const [scope, group] of pendingGrouped) {
@@ -191,25 +199,47 @@ async function startMonitor(options, stats, dailyAlerts, recentlyScanned, downlo
     saveState(state, stats);
     reportStats(stats);
     console.log('[MONITOR] State saved. Bye!');
-    running = false;
     process.exit(0);
   }
 
   process.on('SIGINT', () => gracefulShutdown('SIGINT'));
   process.on('SIGTERM', () => gracefulShutdown('SIGTERM'));
 
-  // Initial poll + scan
+  // Initial poll + scan (sequential for first run)
   await poll(state, scanQueue, stats);
   saveState(state, stats);
   await processQueue(scanQueue, stats, dailyAlerts, recentlyScanned, downloadsCache, sandboxAvailableRef.value);
 
-  // Interval polling
+  // ─── Decoupled polling ───
+  // Poll runs on its own interval, independent of processing.
+  // This ensures new packages are ingested even while a large batch is being scanned.
+  // Without this, a 2h processing batch blocks all polling — packages published and
+  // removed during that window are never seen (e.g. axios/plain-crypto-js 2026-03-30).
+  let pollInProgress = false;
+  pollIntervalHandle = setInterval(async () => {
+    if (!running || pollInProgress) return;
+    pollInProgress = true;
+    try {
+      await poll(state, scanQueue, stats);
+      saveState(state, stats);
+      if (scanQueue.length > QUEUE_WARNING_THRESHOLD) {
+        console.log(`[MONITOR] WARNING: scan queue depth ${scanQueue.length} — processing may be lagging behind ingestion`);
+      }
+    } catch (err) {
+      console.error('[MONITOR] Poll error (interval):', err.message);
+    } finally {
+      pollInProgress = false;
+    }
+  }, POLL_INTERVAL);
+
+  // ─── Continuous processing loop ───
+  // Consumes scanQueue independently of polling. Workers inside processQueue
+  // check scanQueue.length > 0 after each item, so items added by a concurrent
+  // poll are picked up immediately by running workers.
   while (running) {
-    await sleep(POLL_INTERVAL);
-    if (!running) break;
-    await poll(state, scanQueue, stats);
-    saveState(state, stats);
-    await processQueue(scanQueue, stats, dailyAlerts, recentlyScanned, downloadsCache, sandboxAvailableRef.value);
+    if (scanQueue.length > 0) {
+      await processQueue(scanQueue, stats, dailyAlerts, recentlyScanned, downloadsCache, sandboxAvailableRef.value);
+    }
 
     // Hourly stats report + cache purge
     if (Date.now() - stats.lastReportTime >= 3600_000) {
@@ -221,6 +251,9 @@ async function startMonitor(options, stats, dailyAlerts, recentlyScanned, downlo
     if (isDailyReportDue(stats)) {
       await sendDailyReport(stats, dailyAlerts, recentlyScanned, downloadsCache);
     }
+
+    // Short pause before re-checking queue — yields event loop for poll interval
+    await sleep(PROCESS_LOOP_INTERVAL);
   }
 }
 
@@ -231,5 +264,7 @@ module.exports = {
   reportStats,
   isDailyReportDue,
   sleep,
-  POLL_INTERVAL
+  POLL_INTERVAL,
+  PROCESS_LOOP_INTERVAL,
+  QUEUE_WARNING_THRESHOLD
 };

package/src/sandbox/gvisor-parser.js

@@ -0,0 +1,348 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+
+// ══════════════════════════════════════════════════════════════
+// gVisor strace log parser
+//
+// gVisor's --strace flag logs syscalls at the kernel level in its
+// debug log files. This parser extracts security-relevant data
+// (file access, network connections, process execution) and returns
+// the SAME structure as sandbox-runner.sh's strace/tcpdump parsing,
+// so the downstream scoreFindings() analyzer needs no changes.
+//
+// gVisor strace format:
+//   D0331 12:34:56.789012  1 strace.go:587] [  PID] process E syscall(args) = ret (dur)
+//
+// Or bare (without Go log prefix):
+//   [  PID] process E syscall(args) = ret (dur)
+// ══════════════════════════════════════════════════════════════
+
+const SENSITIVE_PATTERN = /\.npmrc|\.ssh\/|\.aws\/|\.env(?:$|[^a-zA-Z])|\/etc\/passwd|\/etc\/shadow|\.gitconfig|\.bash_history/;
+
+// Processes that are sandbox infrastructure — not spawned by the package
+const SAFE_PROCESSES = new Set(['node', 'npm', 'npx', 'sh', 'git']);
+
+// ── Line-level parsers ──
+
+/**
+ * Parse a single gVisor strace line.
+ * Handles both Go-log-prefixed and bare formats.
+ *
+ * @param {string} line - Raw log line
+ * @returns {object|null} Parsed syscall info or null if not a strace line
+ */
+function parseStraceLine(line) {
+  // Strip Go log prefix if present: everything up to `] ` before the `[PID]` block
+  const bracketIdx = line.indexOf('] [');
+  const content = bracketIdx >= 0 ? line.substring(bracketIdx + 2).trim() : line.trim();
+
+  // Match: [PID] process E/X syscall(args) = return (duration)
+  const match = content.match(
+    /^\[\s*(\d+)\]\s+(\S+)\s+[EX]\s+(\w+)\((.+)\)\s*=\s*(.+?)(?:\s+\([\d.]+[µm]?s\))?$/
+  );
+  if (!match) return null;
+
+  return {
+    pid: parseInt(match[1], 10),
+    process: match[2],
+    syscall: match[3],
+    args: match[4],
+    returnValue: match[5].trim()
+  };
+}
+
+/**
+ * Extract file path and flags from openat/open args.
+ * gVisor format: AT_FDCWD, "/path", O_RDONLY|O_CLOEXEC
+ *
+ * @param {string} args - Syscall arguments string
+ * @returns {object|null} { path, flags } or null
+ */
+function extractOpenatPath(args) {
+  // Comma-separated: AT_FDCWD, "/path/to/file", O_RDONLY|O_CLOEXEC, 0o0
+  const match = args.match(/"([^"]+)"[\s,]+([A-Z_|]+)/);
+  if (match) return { path: match[1], flags: match[2] };
+
+  // Space-separated (some gVisor versions): AT_FDCWD "/path" O_RDONLY
+  const matchSpace = args.match(/"([^"]+)"\s+([A-Z_|]+)/);
+  if (matchSpace) return { path: matchSpace[1], flags: matchSpace[2] };
+
+  return null;
+}
+
+/**
+ * Extract IP and port from connect() args.
+ * gVisor format: {Family: AF_INET, Addr: 1.2.3.4, Port: 443}
+ *
+ * @param {string} args - Syscall arguments string
+ * @returns {object|null} { ip, port } or null
+ */
+function extractConnectInfo(args) {
+  const match = args.match(/\{Family:\s*AF_INET,\s*Addr:\s*([\d.]+),\s*Port:\s*(\d+)\}/);
+  if (!match) return null;
+  return { ip: match[1], port: parseInt(match[2], 10) };
+}
+
+/**
+ * Extract command path from execve() args.
+ * gVisor format: execve("/usr/bin/curl", ["curl", ...], ...)
+ *
+ * @param {string} args - Syscall arguments string
+ * @returns {string|null} Command path or null
+ */
+function extractExecveCommand(args) {
+  const match = args.match(/^"([^"]+)"/);
+  return match ? match[1] : null;
+}
+
+// ── Main parser ──
+
+/**
+ * Parse gVisor strace log content and extract security-relevant findings.
+ * Returns the SAME data structure as sandbox-runner.sh's strace parsing
+ * so scoreFindings() works identically.
+ *
+ * @param {string} content - Raw gVisor strace log content
+ * @returns {object} { sensitive_files: {read, written}, network: {http_connections}, processes: {spawned} }
+ */
+function parseGvisorStrace(content) {
+  const sensitiveReads = new Set();
+  const sensitiveWrites = new Set();
+  const connections = new Map();  // dedup by ip:port
+  const processes = new Map();    // dedup by pid:command
+
+  const lines = content.split('\n');
+
+  for (const line of lines) {
+    const parsed = parseStraceLine(line);
+    if (!parsed) continue;
+
+    // Only process successful syscalls (return >= 0)
+    if (parsed.returnValue.startsWith('-')) continue;
+
+    switch (parsed.syscall) {
+      case 'openat':
+      case 'open': {
+        const info = extractOpenatPath(parsed.args);
+        if (!info || !SENSITIVE_PATTERN.test(info.path)) break;
+
+        if (/O_WRONLY|O_RDWR|O_CREAT/.test(info.flags)) {
+          sensitiveWrites.add(info.path);
+        } else if (/O_RDONLY/.test(info.flags)) {
+          sensitiveReads.add(info.path);
+        }
+        break;
+      }
+
+      case 'connect': {
+        const conn = extractConnectInfo(parsed.args);
+        if (!conn) break;
+        if (conn.ip.startsWith('127.')) break;  // skip loopback
+        if (conn.port === 65535) break;          // skip probe port
+        const key = `${conn.ip}:${conn.port}`;
+        if (!connections.has(key)) {
+          connections.set(key, { host: conn.ip, port: conn.port, protocol: 'TCP' });
+        }
+        break;
+      }
+
+      case 'execve': {
+        const cmd = extractExecveCommand(parsed.args);
+        if (!cmd) break;
+        const basename = path.basename(cmd);
+        if (SAFE_PROCESSES.has(basename)) break;
+        const key = `${parsed.pid}:${cmd}`;
+        if (!processes.has(key)) {
+          processes.set(key, { command: cmd, pid: parsed.pid });
+        }
+        break;
+      }
+    }
+  }
+
+  return {
+    sensitive_files: {
+      read: [...sensitiveReads],
+      written: [...sensitiveWrites]
+    },
+    network: {
+      http_connections: [...connections.values()]
+    },
+    processes: {
+      spawned: [...processes.values()]
+    }
+  };
+}
+
+// ── Log discovery ──
+
+/**
+ * Find gVisor log files for a specific container.
+ * Searches the debug-log directory using multiple strategies:
+ *   1. %ID% template → directory named after full container ID
+ *   2. Truncated ID (12 chars) directory
+ *   3. Files in logDir containing the container ID
+ *
+ * @param {string} containerId - Docker container ID (full 64-char or truncated)
+ * @param {string} logDir - gVisor debug-log base directory (default: /tmp/runsc)
+ * @returns {string[]} Matching log file paths
+ */
+function findGvisorLogs(containerId, logDir) {
+  logDir = logDir || '/tmp/runsc';
+  const logFiles = [];
+
+  if (!fs.existsSync(logDir)) return logFiles;
+
+  const shortId = containerId.substring(0, 12);
+
+  // Strategy 1: directory named after full container ID (%ID% template)
+  const fullDir = path.join(logDir, containerId);
+  if (fs.existsSync(fullDir) && fs.statSync(fullDir).isDirectory()) {
+    return collectLogFiles(fullDir);
+  }
+
+  // Strategy 2: directory named after truncated ID
+  const shortDir = path.join(logDir, shortId);
+  if (fs.existsSync(shortDir) && fs.statSync(shortDir).isDirectory()) {
+    return collectLogFiles(shortDir);
+  }
+
+  // Strategy 3: flat files containing the container ID in name
+  try {
+    const files = fs.readdirSync(logDir);
+    for (const file of files) {
+      if ((file.includes(containerId) || file.includes(shortId)) &&
+          (file.endsWith('.log') || file.includes('boot'))) {
+        logFiles.push(path.join(logDir, file));
+      }
+    }
+  } catch { /* directory not readable */ }
+
+  return logFiles;
+}
+
+function collectLogFiles(dir) {
+  const files = [];
+  try {
+    for (const file of fs.readdirSync(dir)) {
+      if (file.endsWith('.log') || file.includes('boot')) {
+        files.push(path.join(dir, file));
+      }
+    }
+  } catch { /* directory not readable */ }
+  return files;
+}
+
+// ── Aggregated parser (main entry point) ──
+
+/**
+ * Parse gVisor log file and return report-compatible structure.
+ * This is the main export matching the spec:
+ *   parseGvisorLog(logPath) → same format as parseStraceOutput()
+ *
+ * @param {string} logPath - Path to a gVisor strace log file
+ * @returns {object} Report supplement with sensitive_files, network, processes
+ */
+function parseGvisorLog(logPath) {
+  try {
+    const content = fs.readFileSync(logPath, 'utf8');
+    return parseGvisorStrace(content);
+  } catch {
+    return {
+      sensitive_files: { read: [], written: [] },
+      network: { http_connections: [] },
+      processes: { spawned: [] }
+    };
+  }
+}
+
+/**
+ * Parse all gVisor logs for a container and return aggregated report supplement.
+ *
+ * @param {string} containerId - Docker container ID
+ * @param {string} logDir - gVisor debug-log base directory
+ * @returns {object} Aggregated report supplement
+ */
+function parseGvisorLogs(containerId, logDir) {
+  const emptyResult = {
+    sensitive_files: { read: [], written: [] },
+    network: { http_connections: [] },
+    processes: { spawned: [] }
+  };
+
+  const logFiles = findGvisorLogs(containerId, logDir);
+  if (logFiles.length === 0) return emptyResult;
+
+  // Aggregate across all log files (boot, gofer, etc.)
+  const allReads = new Set();
+  const allWrites = new Set();
+  const allConnections = new Map();
+  const allProcesses = new Map();
+
+  for (const logFile of logFiles) {
+    const result = parseGvisorLog(logFile);
+
+    for (const f of result.sensitive_files.read) allReads.add(f);
+    for (const f of result.sensitive_files.written) allWrites.add(f);
+    for (const c of result.network.http_connections) {
+      const key = `${c.host}:${c.port}`;
+      if (!allConnections.has(key)) allConnections.set(key, c);
+    }
+    for (const p of result.processes.spawned) {
+      const key = `${p.pid}:${p.command}`;
+      if (!allProcesses.has(key)) allProcesses.set(key, p);
+    }
+  }
+
+  return {
+    sensitive_files: { read: [...allReads], written: [...allWrites] },
+    network: { http_connections: [...allConnections.values()] },
+    processes: { spawned: [...allProcesses.values()] }
+  };
+}
+
+/**
+ * Clean up gVisor log files for a container after analysis.
+ * Prevents disk fill from accumulated logs across sandbox runs.
+ *
+ * @param {string} containerId - Docker container ID
+ * @param {string} logDir - gVisor debug-log base directory
+ */
+function cleanupGvisorLogs(containerId, logDir) {
+  logDir = logDir || '/tmp/runsc';
+  const shortId = containerId.substring(0, 12);
+
+  try {
+    // Try container-specific directory first
+    for (const dirName of [containerId, shortId]) {
+      const dir = path.join(logDir, dirName);
+      if (fs.existsSync(dir) && fs.statSync(dir).isDirectory()) {
+        fs.rmSync(dir, { recursive: true, force: true });
+        return;
+      }
+    }
+
+    // Fall back to individual files
+    const files = fs.readdirSync(logDir);
+    for (const file of files) {
+      if (file.includes(containerId) || file.includes(shortId)) {
+        fs.unlinkSync(path.join(logDir, file));
+      }
+    }
+  } catch { /* cleanup is best-effort */ }
+}
+
+module.exports = {
+  parseGvisorLog,
+  parseGvisorLogs,
+  parseGvisorStrace,
+  findGvisorLogs,
+  cleanupGvisorLogs,
+  // Exported for unit tests
+  parseStraceLine,
+  extractOpenatPath,
+  extractConnectInfo,
+  extractExecveCommand
+};