muaddib-scanner 2.11.132 → 2.11.135

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "muaddib-scanner",
-  "version": "2.11.132",
+  "version": "2.11.135",
   "description": "Supply-chain threat detection & response for npm & PyPI/Python",
   "main": "src/index.js",
   "bin": {

package/{self-scan-v2.11.132.json → self-scan-v2.11.135.json}

@@ -1,6 +1,6 @@
 {
   "target": "node_modules",
-  "timestamp": "2026-06-28T11:47:00.100Z",
+  "timestamp": "2026-06-28T13:42:09.849Z",
   "threats": [
     {
       "type": "string_mutation_obfuscation",

package/src/monitor/ingestion.js

@@ -606,6 +606,116 @@ const BURST_MIN_VERSIONS_PREFETCH = (() => {
   return Number.isFinite(n) && n >= 2 ? n : 10;
 })();
 
+// A prerelease publish (1.2.0-rc.1, 2.0.0-beta, ...-dev) is EXPECTED to differ
+// from dist-tags.latest: npm semver resolution (^/~/x.y ranges) never selects a
+// prerelease unless it is requested explicitly, so an account-takeover that wants
+// victims to auto-resolve to the malicious version cannot use one. Excluding
+// prereleases from the ATO *prefetch* signal drops the dominant false positive
+// (CI/dev/rc/beta churn — ~17% of npm publishes per the 2026-06-28 backtest, which
+// would otherwise balloon the bounded tarball cache) WITHOUT weakening the
+// release-version ATO catch (Leo Platform: malicious leo-sdk@6.0.19, a plain
+// release, still fires). Prefetch-only: queue.js keeps its own (more inclusive)
+// atoSignal for fast-track / anti-eviction, and this never touches scoring.
+function isPrereleaseVersion(v) {
+  // semver: the prerelease lives after '-' in the version core, before any '+'
+  // build metadata. Strip build metadata first so '1.0.0+build-7' is NOT flagged.
+  return /-/.test(String(v == null ? '' : v).split('+')[0]);
+}
+
+// Derive the capture-at-publish trigger for the fast-takedown class (Leo/Miasma)
+// from a registry packument. Shared by preResolveNpmBatch (Stage 2.1) and the
+// real-time capture lane so both compute the SAME signal: ATO = published version
+// != dist-tags.latest, excluding prereleases (isPrereleaseVersion); burst =
+// >= BURST_MIN_VERSIONS_PREFETCH versions in the recent window. Returns the cache
+// trigger object (shouldCache:true) or null. `name` is passed through so a package
+// that is ALSO an ioc/typosquat match still gets the right (higher-priority) reason.
+function evaluateBurstAtoTrigger(name, npmInfo, version) {
+  if (!npmInfo) return null;
+  const atoSignal = !!(npmInfo.latestTagVersion && version &&
+    version !== npmInfo.latestTagVersion && !isPrereleaseVersion(version));
+  const burstCount = Number.isFinite(npmInfo.recentWindowCount)
+    ? npmInfo.recentWindowCount
+    : ((Array.isArray(npmInfo.recentVersions) ? npmInfo.recentVersions.length : 0) + 1);
+  const isBurst = burstCount >= BURST_MIN_VERSIONS_PREFETCH;
+  if (!atoSignal && !isBurst) return null;
+  const trig = evaluateCacheTrigger(name, null, null, { atoSignal, isBurst });
+  return trig.shouldCache ? trig : null;
+}
+
+// Fix C — real-time capture lane. preResolveNpmBatch sheds the packument prefetch
+// when the scan queue is deep (queue > PRE_RESOLVE_SHED_QUEUE — the chronic backlog
+// state). Under shed nothing is prefetched AT ALL: schedulePrefetch needs a resolved
+// tarballUrl, which the shed never fetched — so even ioc/typosquat/first_publish items
+// (flagged name-only at ingestion) are not captured, and burst/ATO is never even
+// computed. The fast-takedown tarballs then 404 before the backlogged scan reaches
+// them (the Leo/Miasma gap Fix A meant to close). This lane resolves a BOUNDED subset
+// of the still-unresolved items REGARDLESS of the shed — prioritizing the already
+// name-flagged (known high-risk, FP-free), then filling the budget to DISCOVER
+// burst/ATO — and enriches each item (version/tarballUrl/_npmInfo/_cacheTrigger) IN
+// PLACE so (a) the schedulePrefetch pass that immediately follows captures the tarball
+// at publish, and (b) the already-enqueued item (enqueueScan pushes by reference)
+// carries the real version so scanPackage's cache-hit (keyed by name@version) finds it.
+//
+// Bounded + rate-limited + flag-gated so it can be turned on only once throughput /
+// OOM headroom allows: OFF unless MUADDIB_REALTIME_PREFETCH=1; at most
+// MUADDIB_REALTIME_PREFETCH_MAX items/cycle (default 30); registry calls go through
+// getNpmLatestTarball's shared semaphore (honors the 429 brain). npm-only for now.
+const REALTIME_PREFETCH_CONCURRENCY = 4;
+function realtimePrefetchEnabled() {
+  return process.env.MUADDIB_REALTIME_PREFETCH === '1';
+}
+function realtimePrefetchMax() {
+  const n = parseInt(process.env.MUADDIB_REALTIME_PREFETCH_MAX, 10);
+  return Number.isFinite(n) && n >= 0 ? n : 30;
+}
+async function realtimePrefetchHighRisk(items, stats) {
+  if (!realtimePrefetchEnabled()) return;
+  const cap = realtimePrefetchMax();
+  if (!cap || !Array.isArray(items) || items.length === 0) return;
+  // Act only on items the shed (or a resolve failure) left UNRESOLVED (no _npmInfo).
+  // When preResolveNpmBatch did NOT shed, every item already has _npmInfo and this
+  // selects nothing (no duplicate fetch). Flagged first (ioc/typo/first_publish —
+  // known high-risk, just needs its tarballUrl resolved to be prefetched), then
+  // unflagged (candidates for burst/ATO discovery).
+  const flagged = [];
+  const unflagged = [];
+  for (const it of items) {
+    if (!it || it.ecosystem !== 'npm' || !it.name || it._npmInfo) continue;
+    (it._cacheTrigger ? flagged : unflagged).push(it);
+  }
+  const selected = flagged.concat(unflagged).slice(0, cap);
+  if (selected.length === 0) return;
+  const overflow = (flagged.length + unflagged.length) - selected.length;
+  if (overflow > 0) {
+    // No silent caps (CLAUDE.md): a burst wider than the budget is only partially captured.
+    console.warn(`[MONITOR] REALTIME PREFETCH: ${flagged.length + unflagged.length} unresolved high-risk candidates, capped at ${cap} this cycle (${overflow} deferred to lazy scan)`);
+    if (stats) stats.realtimePrefetchCapped = (stats.realtimePrefetchCapped || 0) + overflow;
+  }
+  let idx = 0;
+  const worker = async () => {
+    while (idx < selected.length) {
+      const it = selected[idx++];
+      try {
+        const npmInfo = await getNpmLatestTarball(it.name);
+        if (!npmInfo || !npmInfo.tarball) continue;
+        // Enrich in place — the item is already enqueued by reference (shed path), so
+        // this also lets the eventual scan reuse the URL + hit the prefetch cache.
+        it.tarballUrl = it.tarballUrl || npmInfo.tarball;
+        if (!it.version) it.version = npmInfo.version || '';
+        it._npmInfo = npmInfo;
+        if (!it._cacheTrigger) {
+          const trig = evaluateBurstAtoTrigger(it.name, npmInfo, it.version);
+          if (trig) it._cacheTrigger = trig;
+        }
+        if (it._cacheTrigger && stats) stats.realtimePrefetchFlagged = (stats.realtimePrefetchFlagged || 0) + 1;
+      } catch {
+        if (stats) stats.realtimePrefetchFailed = (stats.realtimePrefetchFailed || 0) + 1;
+      }
+    }
+  };
+  await Promise.all(Array.from({ length: Math.min(REALTIME_PREFETCH_CONCURRENCY, selected.length) }, worker));
+}
+
 async function preResolveNpmBatch(items, stats, scanQueue) {
   if (!items || items.length === 0) return;
   const start = Date.now();
@@ -660,16 +770,8 @@ async function preResolveNpmBatch(items, stats, scanQueue) {
           // higher-priority (ioc/typosquat) trigger already fired. queue.js still computes
           // the same signals later for scan-queue protection + extras enqueue (idempotent).
           if (!item._cacheTrigger) {
-            const atoSignal = !!(npmInfo.latestTagVersion && item.version &&
-              item.version !== npmInfo.latestTagVersion);
-            const burstCount = Number.isFinite(npmInfo.recentWindowCount)
-              ? npmInfo.recentWindowCount
-              : ((Array.isArray(npmInfo.recentVersions) ? npmInfo.recentVersions.length : 0) + 1);
-            const isBurst = burstCount >= BURST_MIN_VERSIONS_PREFETCH;
-            if (atoSignal || isBurst) {
-              const trig = evaluateCacheTrigger(item.name, null, null, { atoSignal, isBurst });
-              if (trig.shouldCache) item._cacheTrigger = trig;
-            }
+            const trig = evaluateBurstAtoTrigger(item.name, npmInfo, item.version);
+            if (trig) item._cacheTrigger = trig;
           }
           resolved++;
         } else {
@@ -945,6 +1047,12 @@ async function pollNpmChanges(state, scanQueue, stats) {
     // resolveTarballAndScan() picks up the slack — zero scan loss.
     await preResolveNpmBatch(newItems, stats, scanQueue);
 
+    // Fix C — real-time capture lane: when preResolveNpmBatch shed under backlog,
+    // resolve+flag a bounded high-risk subset anyway so the schedulePrefetch below
+    // still captures them. No-op when not shedding or when MUADDIB_REALTIME_PREFETCH
+    // is off (default). See realtimePrefetchHighRisk.
+    await realtimePrefetchHighRisk(newItems, stats);
+
     // Capture-at-publish (Miasma / Leo, June 2026): prefetch the high-value
     // subset's tarballs into the local cache NOW — before the (often backlogged)
     // scan downloads them — so we win the race against fast-takedown unpublishes.
@@ -1586,6 +1694,9 @@ module.exports = {
   preResolveNpmBatch,
   preResolvePyPIBatch,
   preResolveShouldShed,
+  isPrereleaseVersion,
+  evaluateBurstAtoTrigger,
+  realtimePrefetchHighRisk,
 
   // RSS parsing
   parseNpmRss,

package/src/monitor/state.js

@@ -118,7 +118,7 @@ const MEMORY_SCORE_TOLERANCE = 0.15; // ±15% score tolerance
 
 // --- Tarball cache constants ---
 
-const TARBALL_CACHE_DIR = path.join(__dirname, '..', '..', 'data', 'tarball-cache');
+const TARBALL_CACHE_DIR = process.env.MUADDIB_TARBALL_CACHE_DIR || path.join(__dirname, '..', '..', 'data', 'tarball-cache');
 const TARBALL_CACHE_INDEX_FILE = path.join(TARBALL_CACHE_DIR, 'cache-index.json');
 const TARBALL_CACHE_DEFAULT_RETENTION_DAYS = 7;
 const TARBALL_CACHE_HIGH_RISK_RETENTION_DAYS = 30;
@@ -495,6 +495,34 @@ function cacheTarball(name, version, sourcePath, reason, retentionDays) {
   console.log(`[MONITOR] TARBALL CACHE: cached ${name}@${version} (${reason}, ${retentionDays}d, ${(fileSize / 1024).toFixed(0)}KB)`);
 }
 
+/**
+ * Reclaim orphaned tarball-cache files: `.tgz` present in `dir` with no entry in
+ * `entries` and an mtime older than `cutoffMs`. Returns { files, bytes } reclaimed.
+ * Pure w.r.t. the index (orphans are untracked) so callers need not re-save it.
+ * Extracted as a seam so it is unit-testable against a temp dir without touching the
+ * production cache. See purgeTarballCache Phase 3 for the leak this addresses.
+ */
+function gcOrphanTarballFiles(dir, entries, now, cutoffMs) {
+  let files = 0;
+  let bytes = 0;
+  let names;
+  try { names = fs.readdirSync(dir); } catch { return { files, bytes }; }
+  for (const file of names) {
+    if (!file.endsWith('.tgz')) continue;
+    const key = file.slice(0, -4);
+    if (entries && entries[key]) continue; // tracked -> Phase 1/2 owns it
+    const filePath = path.join(dir, file);
+    try {
+      const st = fs.statSync(filePath);
+      if (now - st.mtimeMs <= cutoffMs) continue; // too fresh -> may be in-flight / cache-hittable
+      fs.unlinkSync(filePath);
+      files++;
+      bytes += st.size || 0;
+    } catch { /* per-file stat/unlink errors non-fatal */ }
+  }
+  return { files, bytes };
+}
+
 /**
  * Purge expired entries and enforce size budget.
  * Called at startup and hourly.
@@ -538,10 +566,23 @@ function purgeTarballCache() {
     }
   }
 
-  if (purgedExpired > 0 || purgedBudget > 0) {
-    saveTarballCacheIndex();
+  // Phase 3: orphan-file GC — reclaim .tgz on disk with no index entry. The index is a
+  // process singleton rebuilt from cache-index.json; a single corrupt/truncated index
+  // (crash mid-atomicWriteFileSync, OOM) makes loadTarballCacheIndex fall back to an empty
+  // index, then saveTarballCacheIndex persists it — orphaning every .tgz already cached.
+  // Phase 1/2 are index-driven and can NEVER reclaim those files -> unbounded disk leak
+  // (measured 5.4GB / 8905 files, 2026-06-28). The mtime>retention bound keeps an in-flight
+  // cacheTarball (copyFileSync precedes the index save) and any still-cache-hittable recent
+  // file safe; only clearly-stale orphans are removed.
+  const orphan = gcOrphanTarballFiles(
+    TARBALL_CACHE_DIR, index.entries, now,
+    TARBALL_CACHE_HIGH_RISK_RETENTION_DAYS * 24 * 60 * 60 * 1000
+  );
+
+  if (purgedExpired > 0 || purgedBudget > 0 || orphan.files > 0) {
+    if (purgedExpired > 0 || purgedBudget > 0) saveTarballCacheIndex();
     const remaining = Object.keys(index.entries).length;
-    console.log(`[MONITOR] TARBALL CACHE: purged ${purgedExpired} expired + ${purgedBudget} budget entries (${remaining} remaining, ${(totalSize / 1024 / 1024).toFixed(1)}MB)`);
+    console.log(`[MONITOR] TARBALL CACHE: purged ${purgedExpired} expired + ${purgedBudget} budget entries + ${orphan.files} orphan files (${(orphan.bytes / 1024 / 1024).toFixed(1)}MB reclaimed, ${remaining} remaining, ${(totalSize / 1024 / 1024).toFixed(1)}MB)`);
   }
 }
 
@@ -1784,6 +1825,7 @@ module.exports = {
   tarballCachePath,
   cacheTarball,
   purgeTarballCache,
+  gcOrphanTarballFiles,
   appendTemporalDetection,
   loadTemporalDetections,
   loadState,