muaddib-scanner 2.11.109 → 2.11.111

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "muaddib-scanner",
-  "version": "2.11.109",
+  "version": "2.11.111",
   "description": "Supply-chain threat detection & response for npm & PyPI/Python",
   "main": "src/index.js",
   "bin": {

package/{self-scan-v2.11.109.json → self-scan-v2.11.111.json}

@@ -1,6 +1,6 @@
 {
   "target": "node_modules",
-  "timestamp": "2026-06-12T16:19:40.738Z",
+  "timestamp": "2026-06-13T09:27:21.416Z",
   "threats": [
     {
       "type": "string_mutation_obfuscation",

package/src/monitor/ingestion.js

@@ -564,6 +564,34 @@ async function getNpmLatestTarball(packageName) {
 // holds ~10KB of state; 1000 of them is a needless heap spike).
 const PRE_RESOLVE_CHUNK_SIZE = 50;
 
+// --- Load-aware pre-resolve shedding (2026-06-13) ---
+// Under catch-up (deep scan queue) or active npm throttle (elevated brain
+// level), prefetching up to CHANGES_LIMIT (1000) packuments per poll cycle
+// through the SHARED registry rate budget starves the per-scan metadata fetches
+// the workers actually need — and most prefetched items get spilled/shed before
+// any worker scans them, so the fetch is wasted budget that also keeps npm
+// 429-ing. When shedding, the batch skips the prefetch and enqueues items with
+// tarballUrl=null; resolveTarballAndScan() lazily resolves ONLY the items a
+// worker actually scans (the existing zero-scan-loss fallback path).
+const PRE_RESOLVE_SHED_QUEUE = Math.max(0, parseInt(process.env.MUADDIB_PRERESOLVE_SHED_QUEUE, 10) || 2000);
+const PRE_RESOLVE_SHED_LEVEL = Math.max(1, parseInt(process.env.MUADDIB_PRERESOLVE_SHED_LEVEL, 10) || 3);
+
+function preResolveShouldShed(scanQueue) {
+  // Kill-switch read live so it can be flipped via the systemd EnvironmentFile
+  // + restart without a code change/rebuild.
+  if (process.env.MUADDIB_PRERESOLVE_NO_SHED === '1') return false;
+  if (scanQueue && scanQueue.length > PRE_RESOLVE_SHED_QUEUE) return true;
+  try {
+    // Lazy-require so the brain accessor is stubbable in tests (a top-level
+    // destructure captures a frozen reference) and to dodge load-order cycles.
+    // require() is cached — negligible on this per-chunk check.
+    const { getBrainState, DEFAULT_HOST } = require('../shared/http-limiter.js');
+    const brain = getBrainState(DEFAULT_HOST);
+    if (brain && (brain.level || 0) >= PRE_RESOLVE_SHED_LEVEL) return true;
+  } catch { /* observability seam — must never block ingestion */ }
+  return false;
+}
+
 // If a scanQueue is provided, items are pushed onto it as soon as their chunk
 // finishes resolution — so a crash mid-batch only loses the current chunk's
 // in-flight work, not all the chunks that already completed. When scanQueue
@@ -575,8 +603,18 @@ async function preResolveNpmBatch(items, stats, scanQueue) {
   let resolved = 0;
   let alreadyResolved = 0;
   let failed = 0;
+  let shed = 0;
   for (let i = 0; i < items.length; i += PRE_RESOLVE_CHUNK_SIZE) {
     const chunk = items.slice(i, i + PRE_RESOLVE_CHUNK_SIZE);
+    if (preResolveShouldShed(scanQueue)) {
+      // Load-aware shed: skip the packument prefetch; enqueue as-is so workers
+      // lazy-resolve ONLY what they actually scan (resolveTarballAndScan handles
+      // tarballUrl=null — zero scan loss). Re-checked per chunk so prefetch
+      // resumes mid-batch the moment the queue drains below the threshold.
+      shed += chunk.length;
+      if (scanQueue) { for (const item of chunk) enqueueScan(scanQueue, item, stats); }
+      continue;
+    }
     await Promise.all(chunk.map(async (item) => {
       if (item.tarballUrl) { alreadyResolved++; return; }
       try {
@@ -626,10 +664,12 @@ async function preResolveNpmBatch(items, stats, scanQueue) {
   if (stats) {
     stats.npmPreResolved = (stats.npmPreResolved || 0) + resolved;
     stats.npmPreResolveFailed = (stats.npmPreResolveFailed || 0) + failed;
+    if (shed) stats.npmPreResolveShed = (stats.npmPreResolveShed || 0) + shed;
   }
   if (items.length >= 5) {
     const elapsed = Date.now() - start;
-    console.log(`[MONITOR] PRE-RESOLVE npm: ${resolved}/${items.length} in ${elapsed}ms (${failed} → lazy fallback${alreadyResolved ? `, ${alreadyResolved} already resolved` : ''})`);
+    const shedNote = shed ? `, ${shed} shed (load-aware)` : '';
+    console.log(`[MONITOR] PRE-RESOLVE npm: ${resolved}/${items.length} in ${elapsed}ms (${failed} → lazy fallback${alreadyResolved ? `, ${alreadyResolved} already resolved` : ''}${shedNote})`);
   }
 }
 
@@ -639,8 +679,17 @@ async function preResolvePyPIBatch(items, stats, scanQueue) {
   let resolved = 0;
   let alreadyResolved = 0;
   let failed = 0;
+  let shed = 0;
   for (let i = 0; i < items.length; i += PRE_RESOLVE_CHUNK_SIZE) {
     const chunk = items.slice(i, i + PRE_RESOLVE_CHUNK_SIZE);
+    if (preResolveShouldShed(scanQueue)) {
+      // Load-aware shed (shared gate): queue-depth dominates here; the prefetched
+      // PyPI metadata would mostly be for items shed before any worker scans them.
+      // Enqueue as-is — resolveTarballAndScan lazily resolves PyPI URLs too.
+      shed += chunk.length;
+      if (scanQueue) { for (const item of chunk) enqueueScan(scanQueue, item, stats); }
+      continue;
+    }
     await Promise.all(chunk.map(async (item) => {
       if (item.tarballUrl) { alreadyResolved++; return; }
       try {
@@ -679,10 +728,12 @@ async function preResolvePyPIBatch(items, stats, scanQueue) {
   if (stats) {
     stats.pypiPreResolved = (stats.pypiPreResolved || 0) + resolved;
     stats.pypiPreResolveFailed = (stats.pypiPreResolveFailed || 0) + failed;
+    if (shed) stats.pypiPreResolveShed = (stats.pypiPreResolveShed || 0) + shed;
   }
   if (items.length >= 5) {
     const elapsed = Date.now() - start;
-    console.log(`[MONITOR] PRE-RESOLVE pypi: ${resolved}/${items.length} in ${elapsed}ms (${failed} → lazy fallback${alreadyResolved ? `, ${alreadyResolved} already resolved` : ''})`);
+    const shedNote = shed ? `, ${shed} shed (load-aware)` : '';
+    console.log(`[MONITOR] PRE-RESOLVE pypi: ${resolved}/${items.length} in ${elapsed}ms (${failed} → lazy fallback${alreadyResolved ? `, ${alreadyResolved} already resolved` : ''}${shedNote})`);
   }
 }
 
@@ -1493,6 +1544,7 @@ module.exports = {
   getNpmLatestTarball,
   preResolveNpmBatch,
   preResolvePyPIBatch,
+  preResolveShouldShed,
 
   // RSS parsing
   parseNpmRss,

package/src/monitor/webhook.js

@@ -1235,9 +1235,18 @@ function buildDailyReportEmbed(stats, dailyAlerts, ledgerRollup) {
   const pypiPub = stats.pypiChangelogPackages || 0;
   const published = npmPub + pypiPub;
   const catchupSkipped = (stats.npmCatchupSkippedSeqs || 0) + (stats.pypiCatchupSkippedEvents || 0);
+  // Clarify the Ops headline so it isn't read as an overnight drop: it counts
+  // COMPLETED scans in the exact ledger window [last report → now], version/
+  // dedup-collapsed — intentionally lower than the in-memory counter (stats.scanned),
+  // which also tallies retries, burst extras and size-cap rejections
+  // (cf. queue.js uniqueScanAttempts). Surface the raw counter when it diverges.
+  const opsQualifier = headline ? ' (completed, deduped, 24h)' : '';
+  const rawCounter = (headline && typeof stats.scanned === 'number' && stats.scanned > hScanned)
+    ? ` · counter ${stats.scanned} (incl. retries/burst)`
+    : '';
   const opsSuffix = catchupSkipped > 0
-    ? `\nOps: ${hScanned} | Catch-up skip: ${catchupSkipped}`
-    : `\nOps: ${hScanned}`;
+    ? `\nOps: ${hScanned}${opsQualifier}${rawCounter} | Catch-up skip: ${catchupSkipped}`
+    : `\nOps: ${hScanned}${opsQualifier}${rawCounter}`;
   let coverageText;
   if (ledger && ledger.distinctPackages > 0 && ledger.distinctCoverage != null) {
     const pct = (ledger.distinctCoverage * 100).toFixed(0);
@@ -1344,9 +1353,10 @@ function buildDailyReportEmbed(stats, dailyAlerts, ledgerRollup) {
         { name: 'System', value: healthText, inline: false }
       ],
       footer: {
-        // Headline-source annotation: 'ledger' = window-exact [last report → now],
-        // 'counters' = in-memory fallback (ledger unavailable — pre-upgrade behavior).
-        text: `MUAD'DIB - Daily summary | headline: ${headline ? 'ledger (since last report)' : 'counters'} | ${readableTime}`
+        // Headline-source annotation: 'ledger' = window-exact [last report → now]
+        // (completed/deduped scans), 'counters' = in-memory fallback (ledger
+        // unavailable — pre-upgrade behavior).
+        text: `MUAD'DIB - Daily summary | headline: ${headline ? 'ledger — completed/deduped, exact 24h window' : 'counters (in-memory fallback)'} | ${readableTime}`
       },
       timestamp: now.toISOString()
     }]

package/src/shared/http-limiter.js

@@ -98,7 +98,10 @@ function computeBackoffTransition(state, event, consts = {}) {
     // Full-quiet reset (the incident is over, restart at base).
     const quietResetMs = s.lastPauseMs * 2 + base * 5;
     if (s.last429At && now - s.last429At > quietResetMs) s.level = 0;
-    s.level += 1;
+    // Cap: beyond ~12 both the pause (60s) and the rate (1/s floor) are
+    // saturated — an unbounded counter only makes operators read "level 25"
+    // as an emergency when it carries no additional behavior.
+    s.level = Math.min(s.level + 1, 12);
     const pause = Math.min(max, base * 2 ** (s.level - 1));
     s.lastPauseMs = pause;
     s.last429At = now;
@@ -159,20 +162,37 @@ function hostForUrl(url) {
   try { return new URL(url).hostname || DEFAULT_HOST; } catch { return DEFAULT_HOST; }
 }
 
-function _effectiveRate() {
+function _effectiveRate(level = 0) {
+  let rate = RATE_LIMIT_PER_SEC;
   if (BOOT_SLOWSTART_MS > 0 && Date.now() - _bootAt < BOOT_SLOWSTART_MS) {
-    return Math.max(1, Math.floor(RATE_LIMIT_PER_SEC / 4));
+    rate = Math.max(1, Math.floor(rate / 4));
   }
-  return RATE_LIMIT_PER_SEC;
+  // Rate-by-level (the partial-throttle fix, 2026-06-12 evening): the pause
+  // alone cannot converge against a registry that PERMANENTLY rejects a
+  // fraction of requests — every post-pause burst guarantees a 429, every
+  // window stays dirty, the level ratchets to the cap and throughput pins at
+  // ~10 req/min forever (observed: level 25, zero de-escalations, while
+  // tarball downloads flowed fine). Halving the SEND RATE per level (floor
+  // 1 req/s) makes a clean 30s window reachable — 30 spaced probes instead of
+  // one burst — so the AIMD de-escalation actually fires and the brain
+  // CONVERGES on the registry's real granted budget instead of oscillating
+  // burst→reject at the cap.
+  if (level > 0) rate = Math.max(1, Math.floor(rate / 2 ** Math.min(level, 5)));
+  return rate;
 }
 
 function _refillTokens(b) {
   const now = Date.now();
   if (now < b.bo.pauseUntil) return; // backoff pause: no refills, no grants
-  const rate = _effectiveRate();
+  const rate = _effectiveRate(b.bo.level);
   if (b.bo.pauseUntil > b.lastRefill) {
-    // First refill after a backoff pause: slow start at half budget.
-    b.tokens = Math.max(1, Math.floor(rate / 2));
+    // First refill after a backoff pause: PROBE OF ONE. The previous half-
+    // budget restart fired a 10-request burst the instant the pause expired —
+    // against a partially-throttling registry that burst GUARANTEED a 429 and
+    // re-armed the next pause. One spaced probe at a time is how the level's
+    // reduced rate (see _effectiveRate) gets a chance to produce the clean
+    // window that de-escalates.
+    b.tokens = 1;
     b.lastRefill = now;
     return;
   }