muaddib-scanner 2.10.85 → 2.10.87

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "muaddib-scanner",
-  "version": "2.10.85",
+  "version": "2.10.87",
   "description": "Supply-chain threat detection & response for npm & PyPI/Python",
   "main": "src/index.js",
   "bin": {

package/src/monitor/deferred-sandbox.js

@@ -175,7 +175,9 @@ async function processDeferredItem(stats) {
   let sandboxResult;
   try {
     const canary = isCanaryEnabled();
-    sandboxResult = await runSandbox(item.name, { canary, skipSemaphore: true });
+    // maxRuns=1: deferred items are T1b/T2, time bomb detection (3 runs) is a luxury.
+    // 90s instead of 270s per item → 3× faster deferred queue drain.
+    sandboxResult = await runSandbox(item.name, { canary, skipSemaphore: true, maxRuns: 1 });
     console.log(`[DEFERRED] SANDBOX COMPLETE: ${key} -> score=${sandboxResult.score}, severity=${sandboxResult.severity}`);
   } catch (err) {
     console.error(`[DEFERRED] SANDBOX ERROR: ${key} — ${err.message}`);

package/src/monitor/queue.js

@@ -11,7 +11,7 @@ const path = require('path');
 const os = require('os');
 const { Worker } = require('worker_threads');
 const { run } = require('../index.js');
-const { runSandbox, isDockerAvailable } = require('../sandbox/index.js');
+const { runSandbox, isDockerAvailable, tryAcquireSandboxSlot, SANDBOX_CONCURRENCY_MAX } = require('../sandbox/index.js');
 const { sendWebhook } = require('../webhook.js');
 const { downloadToFile, extractTarGz, sanitizePackageName } = require('../shared/download.js');
 const { MAX_TARBALL_SIZE } = require('../shared/constants.js');
@@ -285,6 +285,16 @@ async function scanPackage(name, version, ecosystem, tarballUrl, registryMeta, s
   const cacheTrigger = meta._cacheTrigger || null;
 
   try {
+    // Pre-download size check: reject packages known to exceed MAX_TARBALL_SIZE
+    // from registry metadata, without wasting a download + 300s timeout.
+    // unpackedSize is available from getNpmLatestTarball() after lazy resolution.
+    const metaSize = meta.unpackedSize || 0;
+    if (metaSize > MAX_TARBALL_SIZE) {
+      console.log(`[MONITOR] SIZE_REJECT: ${name}@${version} — metadata size ${(metaSize / 1024 / 1024).toFixed(1)}MB exceeds ${(MAX_TARBALL_SIZE / 1024 / 1024).toFixed(0)}MB limit (skipped without download)`);
+      stats.scanned++;
+      return;
+    }
+
     const tgzPath = path.join(tmpDir, 'package.tar.gz');
 
     // Layer 3: Check tarball cache before downloading
@@ -727,40 +737,61 @@ async function scanPackage(name, version, ecosystem, tarballUrl, registryMeta, s
         if (shouldSandbox) {
           try {
             const canary = isCanaryEnabled();
-            const reason = tier === 2 ? ' (T2, queue low)' : tier === '1b' ? ' (T1b, conditional)' : '';
-            console.log(`[MONITOR] SANDBOX${reason}: launching for ${name}@${version}${canary ? ' (canary: on)' : ''}...`);
-            sandboxResult = await runSandbox(name, { canary });
-            console.log(`[MONITOR] SANDBOX: ${name}@${version} → score: ${sandboxResult.score}, severity: ${sandboxResult.severity}`);
-
-            // Check for canary exfiltration findings and send dedicated alert
-            const canaryFindings = (sandboxResult.findings || []).filter(f => f.type === 'canary_exfiltration');
-            if (canaryFindings.length > 0) {
-              console.log(`[MONITOR] CANARY EXFILTRATION: ${name}@${version} — ${canaryFindings.length} token(s) stolen!`);
-              // Dedup: skip if this package was already alerted with canary_exfiltration
-              const canaryRuleId = 'canary_exfiltration';
-              const previousRules = alertedPackageRules.get(name);
-              const alreadyAlerted = previousRules && previousRules.has(canaryRuleId);
-              if (alreadyAlerted) {
-                console.log(`[MONITOR] DEDUP: ${name} canary exfiltration (already alerted today)`);
-              } else {
-                const url = getWebhookUrl();
-                if (url) {
-                  const exfiltrations = canaryFindings.map(f => ({
-                    token: f.detail.match(/exfiltrate (\S+)/)?.[1] || 'UNKNOWN',
-                    foundIn: f.detail
-                  }));
-                  const payload = buildCanaryExfiltrationWebhookEmbed(name, version, exfiltrations);
-                  try {
-                    await sendWebhook(url, payload, { rawPayload: true });
-                    console.log(`[MONITOR] Canary exfiltration webhook sent for ${name}@${version}`);
-                    // Track in dedup map
-                    if (previousRules) {
-                      previousRules.add(canaryRuleId);
-                    } else {
-                      alertedPackageRules.set(name, new Set([canaryRuleId]));
+            const maxRuns = tier === '1a' ? undefined : 1;
+
+            if (tier === '1a') {
+              // T1a: mandatory sandbox — block-wait (high-confidence threats MUST get sandbox)
+              console.log(`[MONITOR] SANDBOX: launching for ${name}@${version}${canary ? ' (canary: on)' : ''}...`);
+              sandboxResult = await runSandbox(name, { canary, maxRuns });
+            } else if (tryAcquireSandboxSlot()) {
+              // T1b/T2: non-blocking — slot acquired atomically, run with skipSemaphore
+              const reason = tier === 2 ? ' (T2, queue low)' : ' (T1b, conditional)';
+              console.log(`[MONITOR] SANDBOX${reason}: launching for ${name}@${version}${canary ? ' (canary: on)' : ''}...`);
+              sandboxResult = await runSandbox(name, { canary, maxRuns, skipSemaphore: true });
+            } else {
+              // T1b/T2: all sandbox slots busy — defer instead of blocking worker
+              console.log(`[MONITOR] SANDBOX DEFER (slots full): ${name}@${version} (tier=${tier}, score=${riskScore})`);
+              enqueueDeferred({
+                name, version, ecosystem, tier, riskScore, tarballUrl,
+                enqueuedAt: Date.now(),
+                staticResult: result,
+                npmRegistryMeta,
+                retries: 0
+              });
+              stats.sandboxDeferred = (stats.sandboxDeferred || 0) + 1;
+            }
+
+            if (sandboxResult) {
+              console.log(`[MONITOR] SANDBOX: ${name}@${version} → score: ${sandboxResult.score}, severity: ${sandboxResult.severity}`);
+
+              // Check for canary exfiltration findings and send dedicated alert
+              const canaryFindings = (sandboxResult.findings || []).filter(f => f.type === 'canary_exfiltration');
+              if (canaryFindings.length > 0) {
+                console.log(`[MONITOR] CANARY EXFILTRATION: ${name}@${version} — ${canaryFindings.length} token(s) stolen!`);
+                const canaryRuleId = 'canary_exfiltration';
+                const previousRules = alertedPackageRules.get(name);
+                const alreadyAlerted = previousRules && previousRules.has(canaryRuleId);
+                if (alreadyAlerted) {
+                  console.log(`[MONITOR] DEDUP: ${name} canary exfiltration (already alerted today)`);
+                } else {
+                  const url = getWebhookUrl();
+                  if (url) {
+                    const exfiltrations = canaryFindings.map(f => ({
+                      token: f.detail.match(/exfiltrate (\S+)/)?.[1] || 'UNKNOWN',
+                      foundIn: f.detail
+                    }));
+                    const payload = buildCanaryExfiltrationWebhookEmbed(name, version, exfiltrations);
+                    try {
+                      await sendWebhook(url, payload, { rawPayload: true });
+                      console.log(`[MONITOR] Canary exfiltration webhook sent for ${name}@${version}`);
+                      if (previousRules) {
+                        previousRules.add(canaryRuleId);
+                      } else {
+                        alertedPackageRules.set(name, new Set([canaryRuleId]));
+                      }
+                    } catch (webhookErr) {
+                      console.error(`[MONITOR] Canary webhook failed for ${name}@${version}: ${webhookErr.message}`);
                     }
-                  } catch (webhookErr) {
-                    console.error(`[MONITOR] Canary webhook failed for ${name}@${version}: ${webhookErr.message}`);
                   }
                 }
               }
@@ -1153,11 +1184,13 @@ async function resolveTarballAndScan(item, stats, dailyAlerts, recentlyScanned,
   let publishResult = null;
   let maintainerResult = null;
 
-  if (item.ecosystem === 'npm' && !item.fastTrack) {
+  const TEMPORAL_LOAD_SHED_THRESHOLD = 2000;
+  const skipTemporal = item.fastTrack || scanQueue.length > TEMPORAL_LOAD_SHED_THRESHOLD;
+  if (item.ecosystem === 'npm' && !skipTemporal) {
     // Run all 4 temporal checks in parallel — each is independent.
-    // With metadata cache (temporal-analysis.js), the 4 modules share 1 HTTP request.
-    // Skipped for fast-track packages (large boring packages — temporal checks make
-    // 4 HTTP requests to npm registry per package, pointless for 50MB enterprise packages).
+    // AST diff alone consumes 5 HTTP semaphore slots per package (2 tarball downloads + 3 metadata).
+    // With 16 workers that's 80 slot requests for 10 slots → workers blocked 80% of the time.
+    // Load-shed when queue > 2000: temporal analysis is a luxury during catch-up.
     const [tempRes, astRes, pubRes, maintRes] = await Promise.allSettled([
       runTemporalCheck(item.name, dailyAlerts),
       runTemporalAstCheck(item.name, dailyAlerts),
@@ -1168,6 +1201,8 @@ async function resolveTarballAndScan(item, stats, dailyAlerts, recentlyScanned,
     astResult = astRes.status === 'fulfilled' ? astRes.value : null;
     publishResult = pubRes.status === 'fulfilled' ? pubRes.value : null;
     maintainerResult = maintRes.status === 'fulfilled' ? maintRes.value : null;
+  } else if (skipTemporal && item.ecosystem === 'npm' && !item.fastTrack) {
+    console.log(`[MONITOR] TEMPORAL LOAD-SHED: ${item.name}@${item.version} (queue=${scanQueue.length} > ${TEMPORAL_LOAD_SHED_THRESHOLD})`);
   }
 
   // Abort check: if timeout fired during temporal checks, skip the expensive scan

package/src/sandbox/index.js

@@ -40,6 +40,19 @@ function acquireSandboxSlot() {
   });
 }
 
+/**
+ * Non-blocking slot acquisition. Returns true if slot acquired, false if all busy.
+ * Atomic: check + acquire in a single synchronous call — no race condition.
+ * Used by T1b/T2 to defer instead of blocking when slots are full.
+ */
+function tryAcquireSandboxSlot() {
+  if (_sandboxSemaphore.active < SANDBOX_CONCURRENCY_MAX) {
+    _sandboxSemaphore.active++;
+    return true;
+  }
+  return false;
+}
+
 function releaseSandboxSlot() {
   if (_sandboxSemaphore.queue.length > 0) {
     const next = _sandboxSemaphore.queue.shift();
@@ -641,12 +654,15 @@ async function runSandbox(packageName, options = {}) {
   try {
     const runtimeLabel = useGvisor ? 'gvisor' : 'docker';
     const slotInfo = skipSem ? 'deferred-slot' : `${_sandboxSemaphore.active}/${SANDBOX_CONCURRENCY_MAX}`;
-    console.log(`[SANDBOX] Analyzing "${displayName}" in isolated container (mode: ${mode}, runtime: ${runtimeLabel}${canaryEnabled ? ', canary: on' : ''}${local ? ', local' : ''}, runs: ${TIME_OFFSETS.length}, slots: ${slotInfo})...`);
+    // maxRuns: cap number of time-offset runs. Default: all TIME_OFFSETS (3 runs).
+    // T1b/T2 use maxRuns=1 to reduce 270s→90s — time bomb detection is a luxury under load.
+    const effectiveRuns = Math.min(options.maxRuns || TIME_OFFSETS.length, TIME_OFFSETS.length);
+    console.log(`[SANDBOX] Analyzing "${displayName}" in isolated container (mode: ${mode}, runtime: ${runtimeLabel}${canaryEnabled ? ', canary: on' : ''}${local ? ', local' : ''}, runs: ${effectiveRuns}, slots: ${slotInfo})...`);
 
     const allRuns = [];
     let bestResult = cleanResult;
 
-    for (let i = 0; i < TIME_OFFSETS.length; i++) {
+    for (let i = 0; i < effectiveRuns; i++) {
       const { offset, label } = TIME_OFFSETS[i];
       console.log(`[SANDBOX] Run ${i + 1}/${TIME_OFFSETS.length} (${label})...`);
 
@@ -1044,4 +1060,4 @@ function displayResults(result) {
   }
 }
 
-module.exports = { buildSandboxImage, runSandbox, runSingleSandbox, scoreFindings, generateNetworkReport, EXFIL_PATTERNS, SAFE_DOMAINS, getSeverity, displayResults, isDockerAvailable, imageExists, isGvisorAvailable, STATIC_CANARY_TOKENS, detectStaticCanaryExfiltration, analyzePreloadLog, TIME_OFFSETS, SAFE_SANDBOX_CMDS, SANDBOX_CONCURRENCY_MAX, acquireSandboxSlot, releaseSandboxSlot, resetSandboxLimiter, getSandboxSemaphore };
+module.exports = { buildSandboxImage, runSandbox, runSingleSandbox, scoreFindings, generateNetworkReport, EXFIL_PATTERNS, SAFE_DOMAINS, getSeverity, displayResults, isDockerAvailable, imageExists, isGvisorAvailable, STATIC_CANARY_TOKENS, detectStaticCanaryExfiltration, analyzePreloadLog, TIME_OFFSETS, SAFE_SANDBOX_CMDS, SANDBOX_CONCURRENCY_MAX, acquireSandboxSlot, tryAcquireSandboxSlot, releaseSandboxSlot, resetSandboxLimiter, getSandboxSemaphore };

package/src/shared/http-limiter.js

@@ -14,7 +14,7 @@
  * NOT covered: api.npmjs.org (different server), replicate.npmjs.com (CouchDB changes stream).
  */
 
-const REGISTRY_SEMAPHORE_MAX = 10;
+const REGISTRY_SEMAPHORE_MAX = 20;
 const RATE_LIMIT_PER_SEC = 30;
 
 // --- Concurrency semaphore ---