muaddib-scanner 2.11.110 → 2.11.112

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "muaddib-scanner",
-  "version": "2.11.110",
+  "version": "2.11.112",
   "description": "Supply-chain threat detection & response for npm & PyPI/Python",
   "main": "src/index.js",
   "bin": {

package/{self-scan-v2.11.110.json → self-scan-v2.11.112.json}

@@ -1,6 +1,6 @@
 {
   "target": "node_modules",
-  "timestamp": "2026-06-12T17:33:52.917Z",
+  "timestamp": "2026-06-13T13:45:22.808Z",
   "threats": [
     {
       "type": "string_mutation_obfuscation",

package/src/monitor/ingestion.js

@@ -9,6 +9,7 @@
 
 const https = require('https');
 const { acquireRegistrySlot, releaseRegistrySlot } = require('../shared/http-limiter.js');
+const { registryAuthHeaders } = require('../shared/registry-auth.js');
 const { loadCachedIOCs } = require('../ioc/updater.js');
 const { enqueueScan } = require('./scan-queue.js');
 const {
@@ -99,7 +100,7 @@ function httpsGet(url, timeoutMs = 30_000, deadlineMs = Math.max(timeoutMs * 2,
       clearTimeout(deadline);
       if (err) reject(err); else resolve(value);
     };
-    req = _deps.https.get(url, { timeout: timeoutMs }, (res) => {
+    req = _deps.https.get(url, { timeout: timeoutMs, headers: registryAuthHeaders(url) }, (res) => {
       if (res.statusCode === 301 || res.statusCode === 302) {
         res.resume();
         const location = res.headers.location;
@@ -564,6 +565,34 @@ async function getNpmLatestTarball(packageName) {
 // holds ~10KB of state; 1000 of them is a needless heap spike).
 const PRE_RESOLVE_CHUNK_SIZE = 50;
 
+// --- Load-aware pre-resolve shedding (2026-06-13) ---
+// Under catch-up (deep scan queue) or active npm throttle (elevated brain
+// level), prefetching up to CHANGES_LIMIT (1000) packuments per poll cycle
+// through the SHARED registry rate budget starves the per-scan metadata fetches
+// the workers actually need — and most prefetched items get spilled/shed before
+// any worker scans them, so the fetch is wasted budget that also keeps npm
+// 429-ing. When shedding, the batch skips the prefetch and enqueues items with
+// tarballUrl=null; resolveTarballAndScan() lazily resolves ONLY the items a
+// worker actually scans (the existing zero-scan-loss fallback path).
+const PRE_RESOLVE_SHED_QUEUE = Math.max(0, parseInt(process.env.MUADDIB_PRERESOLVE_SHED_QUEUE, 10) || 2000);
+const PRE_RESOLVE_SHED_LEVEL = Math.max(1, parseInt(process.env.MUADDIB_PRERESOLVE_SHED_LEVEL, 10) || 3);
+
+function preResolveShouldShed(scanQueue) {
+  // Kill-switch read live so it can be flipped via the systemd EnvironmentFile
+  // + restart without a code change/rebuild.
+  if (process.env.MUADDIB_PRERESOLVE_NO_SHED === '1') return false;
+  if (scanQueue && scanQueue.length > PRE_RESOLVE_SHED_QUEUE) return true;
+  try {
+    // Lazy-require so the brain accessor is stubbable in tests (a top-level
+    // destructure captures a frozen reference) and to dodge load-order cycles.
+    // require() is cached — negligible on this per-chunk check.
+    const { getBrainState, DEFAULT_HOST } = require('../shared/http-limiter.js');
+    const brain = getBrainState(DEFAULT_HOST);
+    if (brain && (brain.level || 0) >= PRE_RESOLVE_SHED_LEVEL) return true;
+  } catch { /* observability seam — must never block ingestion */ }
+  return false;
+}
+
 // If a scanQueue is provided, items are pushed onto it as soon as their chunk
 // finishes resolution — so a crash mid-batch only loses the current chunk's
 // in-flight work, not all the chunks that already completed. When scanQueue
@@ -575,8 +604,18 @@ async function preResolveNpmBatch(items, stats, scanQueue) {
   let resolved = 0;
   let alreadyResolved = 0;
   let failed = 0;
+  let shed = 0;
   for (let i = 0; i < items.length; i += PRE_RESOLVE_CHUNK_SIZE) {
     const chunk = items.slice(i, i + PRE_RESOLVE_CHUNK_SIZE);
+    if (preResolveShouldShed(scanQueue)) {
+      // Load-aware shed: skip the packument prefetch; enqueue as-is so workers
+      // lazy-resolve ONLY what they actually scan (resolveTarballAndScan handles
+      // tarballUrl=null — zero scan loss). Re-checked per chunk so prefetch
+      // resumes mid-batch the moment the queue drains below the threshold.
+      shed += chunk.length;
+      if (scanQueue) { for (const item of chunk) enqueueScan(scanQueue, item, stats); }
+      continue;
+    }
     await Promise.all(chunk.map(async (item) => {
       if (item.tarballUrl) { alreadyResolved++; return; }
       try {
@@ -626,10 +665,12 @@ async function preResolveNpmBatch(items, stats, scanQueue) {
   if (stats) {
     stats.npmPreResolved = (stats.npmPreResolved || 0) + resolved;
     stats.npmPreResolveFailed = (stats.npmPreResolveFailed || 0) + failed;
+    if (shed) stats.npmPreResolveShed = (stats.npmPreResolveShed || 0) + shed;
   }
   if (items.length >= 5) {
     const elapsed = Date.now() - start;
-    console.log(`[MONITOR] PRE-RESOLVE npm: ${resolved}/${items.length} in ${elapsed}ms (${failed} → lazy fallback${alreadyResolved ? `, ${alreadyResolved} already resolved` : ''})`);
+    const shedNote = shed ? `, ${shed} shed (load-aware)` : '';
+    console.log(`[MONITOR] PRE-RESOLVE npm: ${resolved}/${items.length} in ${elapsed}ms (${failed} → lazy fallback${alreadyResolved ? `, ${alreadyResolved} already resolved` : ''}${shedNote})`);
   }
 }
 
@@ -639,8 +680,17 @@ async function preResolvePyPIBatch(items, stats, scanQueue) {
   let resolved = 0;
   let alreadyResolved = 0;
   let failed = 0;
+  let shed = 0;
   for (let i = 0; i < items.length; i += PRE_RESOLVE_CHUNK_SIZE) {
     const chunk = items.slice(i, i + PRE_RESOLVE_CHUNK_SIZE);
+    if (preResolveShouldShed(scanQueue)) {
+      // Load-aware shed (shared gate): queue-depth dominates here; the prefetched
+      // PyPI metadata would mostly be for items shed before any worker scans them.
+      // Enqueue as-is — resolveTarballAndScan lazily resolves PyPI URLs too.
+      shed += chunk.length;
+      if (scanQueue) { for (const item of chunk) enqueueScan(scanQueue, item, stats); }
+      continue;
+    }
     await Promise.all(chunk.map(async (item) => {
       if (item.tarballUrl) { alreadyResolved++; return; }
       try {
@@ -679,10 +729,12 @@ async function preResolvePyPIBatch(items, stats, scanQueue) {
   if (stats) {
     stats.pypiPreResolved = (stats.pypiPreResolved || 0) + resolved;
     stats.pypiPreResolveFailed = (stats.pypiPreResolveFailed || 0) + failed;
+    if (shed) stats.pypiPreResolveShed = (stats.pypiPreResolveShed || 0) + shed;
   }
   if (items.length >= 5) {
     const elapsed = Date.now() - start;
-    console.log(`[MONITOR] PRE-RESOLVE pypi: ${resolved}/${items.length} in ${elapsed}ms (${failed} → lazy fallback${alreadyResolved ? `, ${alreadyResolved} already resolved` : ''})`);
+    const shedNote = shed ? `, ${shed} shed (load-aware)` : '';
+    console.log(`[MONITOR] PRE-RESOLVE pypi: ${resolved}/${items.length} in ${elapsed}ms (${failed} → lazy fallback${alreadyResolved ? `, ${alreadyResolved} already resolved` : ''}${shedNote})`);
   }
 }
 
@@ -1493,6 +1545,7 @@ module.exports = {
   getNpmLatestTarball,
   preResolveNpmBatch,
   preResolvePyPIBatch,
+  preResolveShouldShed,
 
   // RSS parsing
   parseNpmRss,

package/src/monitor/webhook.js

@@ -1235,9 +1235,18 @@ function buildDailyReportEmbed(stats, dailyAlerts, ledgerRollup) {
   const pypiPub = stats.pypiChangelogPackages || 0;
   const published = npmPub + pypiPub;
   const catchupSkipped = (stats.npmCatchupSkippedSeqs || 0) + (stats.pypiCatchupSkippedEvents || 0);
+  // Clarify the Ops headline so it isn't read as an overnight drop: it counts
+  // COMPLETED scans in the exact ledger window [last report → now], version/
+  // dedup-collapsed — intentionally lower than the in-memory counter (stats.scanned),
+  // which also tallies retries, burst extras and size-cap rejections
+  // (cf. queue.js uniqueScanAttempts). Surface the raw counter when it diverges.
+  const opsQualifier = headline ? ' (completed, deduped, 24h)' : '';
+  const rawCounter = (headline && typeof stats.scanned === 'number' && stats.scanned > hScanned)
+    ? ` · counter ${stats.scanned} (incl. retries/burst)`
+    : '';
   const opsSuffix = catchupSkipped > 0
-    ? `\nOps: ${hScanned} | Catch-up skip: ${catchupSkipped}`
-    : `\nOps: ${hScanned}`;
+    ? `\nOps: ${hScanned}${opsQualifier}${rawCounter} | Catch-up skip: ${catchupSkipped}`
+    : `\nOps: ${hScanned}${opsQualifier}${rawCounter}`;
   let coverageText;
   if (ledger && ledger.distinctPackages > 0 && ledger.distinctCoverage != null) {
     const pct = (ledger.distinctCoverage * 100).toFixed(0);
@@ -1344,9 +1353,10 @@ function buildDailyReportEmbed(stats, dailyAlerts, ledgerRollup) {
         { name: 'System', value: healthText, inline: false }
       ],
       footer: {
-        // Headline-source annotation: 'ledger' = window-exact [last report → now],
-        // 'counters' = in-memory fallback (ledger unavailable — pre-upgrade behavior).
-        text: `MUAD'DIB - Daily summary | headline: ${headline ? 'ledger (since last report)' : 'counters'} | ${readableTime}`
+        // Headline-source annotation: 'ledger' = window-exact [last report → now]
+        // (completed/deduped scans), 'counters' = in-memory fallback (ledger
+        // unavailable — pre-upgrade behavior).
+        text: `MUAD'DIB - Daily summary | headline: ${headline ? 'ledger — completed/deduped, exact 24h window' : 'counters (in-memory fallback)'} | ${readableTime}`
       },
       timestamp: now.toISOString()
     }]

package/src/scanner/npm-registry.js

@@ -1,6 +1,7 @@
 const { NPM_PACKAGE_REGEX } = require('../shared/constants.js');
 const { debugLog } = require('../utils.js');
 const { acquireRegistrySlot, releaseRegistrySlot, awaitRateToken, signal429, hostForUrl } = require('../shared/http-limiter.js');
+const { registryAuthHeaders } = require('../shared/registry-auth.js');
 const { computeAdvancedRegistrySignals } = require('../integrations/registry-signals.js');
 
 const REGISTRY_URL = 'https://registry.npmjs.org';
@@ -12,6 +13,16 @@ const SEARCH_URL = 'https://registry.npmjs.org/-/v1/search';
 const REQUEST_TIMEOUT = Math.max(1000, parseInt(process.env.MUADDIB_REGISTRY_TIMEOUT_MS, 10) || 10000); // 10s default
 const MAX_RETRIES = Math.max(1, parseInt(process.env.MUADDIB_REGISTRY_RETRIES, 10) || 5);
 
+// Per-maintainer cache for the /-/v1/search author-count lookup — the only
+// DYNAMIC (non-CDN), rate-limited registry endpoint we hit per scan. Without it,
+// firing the search on every scan generated the bulk of the monitor's 429s and
+// the shared brain then throttled the (healthy, CDN-served) packument reads too.
+// See getPackageMetadata. Env-tunable; defaults preserve the signal.
+const _authorCountCache = new Map(); // maintainer → { count, at }
+const AUTHOR_CACHE_TTL_MS = Math.max(0, parseInt(process.env.MUADDIB_AUTHOR_CACHE_TTL_MS, 10) || 3_600_000); // 1h
+const AUTHOR_CACHE_MAX = Math.max(100, parseInt(process.env.MUADDIB_AUTHOR_CACHE_MAX, 10) || 5000);
+const AUTHOR_SEARCH_ENABLED = process.env.MUADDIB_NPM_AUTHOR_SEARCH !== '0'; // kill-switch
+
 /**
  * Create a timeout signal, with fallback for older Node versions.
  * Returns { signal, cleanup } — call cleanup() after fetch to prevent timer leaks.
@@ -25,7 +36,7 @@ function createTimeoutSignal(ms) {
   return { signal: controller.signal, cleanup: () => clearTimeout(timer) };
 }
 
-async function fetchWithRetry(url) {
+async function fetchWithRetry(url, opts = {}) {
   for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {
     // The caller's acquireRegistrySlot paid the rate token for the FIRST
     // attempt only. Every retry is a new network request and must pay its own
@@ -43,7 +54,7 @@ async function fetchWithRetry(url) {
     let response;
     const { signal, cleanup } = createTimeoutSignal(REQUEST_TIMEOUT);
     try {
-      response = await fetch(url, { signal });
+      response = await fetch(url, { signal, headers: registryAuthHeaders(url) });
     } catch {
       cleanup();
       // REG-001: Retry on timeout/abort instead of returning null immediately.
@@ -71,7 +82,16 @@ async function fetchWithRetry(url) {
     // Retry-After (capped at 30s) with jitter so retries don't re-synchronize.
     if (response.status === 429) {
       try { await response.text(); } catch (e) { debugLog('response drain failed:', e.message); }
-      try { signal429(); } catch { /* limiter is best-effort */ }
+      // Back off the CORRECT host's bucket. This previously defaulted to
+      // registry.npmjs.org, so a 429 from api.npmjs.org/downloads (a SEPARATE,
+      // aggressively rate-limited host that 429s ~every request) poisoned the
+      // registry brain and stalled the tarball/packument fetches that were
+      // themselves healthy — the measured ~20s/scan throughput wall.
+      try { signal429(hostForUrl(url)); } catch { /* limiter is best-effort */ }
+      // Best-effort callers (the weekly-downloads reputation signal, whose
+      // endpoint 429s on essentially every request) opt out of the retry storm:
+      // retrying 5× with ~2s sleeps just burns ~10s/scan to still return null.
+      if (opts.noRetryOn429) return null;
       const retryAfter = parseInt(response.headers.get('retry-after'), 10);
       const base = Math.min(retryAfter && retryAfter > 0 ? retryAfter * 1000 : 2000, 30000);
       await new Promise(r => setTimeout(r, Math.round(base * (0.5 + Math.random() * 0.5))));
@@ -170,26 +190,45 @@ async function getPackageMetadata(packageName) {
   }
   const provenanceRegressed = !latestHasProvenance && anyPriorHadProvenance;
 
-  // 2. Weekly downloads + author search (parallel)
+  // 2. Weekly downloads + author package count (parallel).
+  // The author count comes from /-/v1/search?text=maintainer: which — unlike the
+  // CDN-served packument — is DYNAMIC, slow (~300-950ms) and the one per-scan call
+  // npm aggressively rate-limits. A TTL cache keyed on the maintainer collapses
+  // the search volume (maintainers repeat heavily: scopes / bots / monorepos)
+  // while keeping author_package_count byte-identical. MUADDIB_NPM_AUTHOR_SEARCH=0
+  // drops the call entirely — the count then stays absent, exactly as the
+  // pre-resolve fast path already leaves it.
   const downloadsUrl = DOWNLOADS_URL + '/' + encodeURIComponent(packageName);
-  const authorUrl = maintainer
+  const authorUrl = (AUTHOR_SEARCH_ENABLED && maintainer)
     ? SEARCH_URL + '?text=maintainer:' + encodeURIComponent(maintainer) + '&size=1'
     : null;
 
-  async function fetchAuthorWithSlot() {
-    if (!authorUrl) return null;
+  async function getAuthorPackageCount() {
+    if (!authorUrl) return 0;
+    const hit = _authorCountCache.get(maintainer);
+    if (hit && (Date.now() - hit.at) < AUTHOR_CACHE_TTL_MS) return hit.count;
     await acquireRegistrySlot();
-    try { return await fetchWithRetry(authorUrl); }
+    let data;
+    try { data = await fetchWithRetry(authorUrl); }
     finally { releaseRegistrySlot(); }
+    // 429-exhausted / error → fetchWithRetry returns null: reuse a stale entry if
+    // present and do NOT cache the miss (a transient 0 would poison the typosquat
+    // "author has ≤1 package" signal).
+    if (!data) return hit ? hit.count : 0;
+    const count = data.total ?? 0;
+    if (_authorCountCache.size >= AUTHOR_CACHE_MAX) {
+      _authorCountCache.delete(_authorCountCache.keys().next().value); // FIFO evict (bounded)
+    }
+    _authorCountCache.set(maintainer, { count, at: Date.now() });
+    return count;
   }
 
-  const [downloadsData, authorData] = await Promise.all([
-    fetchWithRetry(downloadsUrl),    // api.npmjs.org — no semaphore needed
-    fetchAuthorWithSlot()            // registry.npmjs.org — semaphore protected
+  const [downloadsData, authorPackageCount] = await Promise.all([
+    fetchWithRetry(downloadsUrl, { noRetryOn429: true }),    // api.npmjs.org — rate-limited; best-effort single shot (no retry storm, correct-host backoff)
+    getAuthorPackageCount()          // registry.npmjs.org search — cached + kill-switchable
   ]);
 
   const weeklyDownloads = downloadsData?.downloads ?? 0;
-  const authorPackageCount = authorData?.total ?? 0;
   const versionCount = meta.versions ? Object.keys(meta.versions).length : 0;
   const description = (typeof latestMeta?.description === 'string' ? latestMeta.description
     : (typeof meta.description === 'string' ? meta.description : ''));

package/src/shared/download.js

@@ -4,6 +4,7 @@ const path = require('path');
 const { execFileSync } = require('child_process');
 const AdmZip = require('adm-zip');
 const { MAX_TARBALL_SIZE, DOWNLOAD_TIMEOUT } = require('./constants.js');
+const { registryAuthHeaders } = require('./registry-auth.js');
 
 // Allowed redirect domains for tarball downloads (SSRF protection)
 const ALLOWED_DOWNLOAD_DOMAINS = [
@@ -159,7 +160,7 @@ function downloadToFile(url, destPath, timeoutMs = DOWNLOAD_TIMEOUT) {
         if (redirectCount >= MAX_REDIRECTS) {
           return reject(new Error(`Too many redirects (${MAX_REDIRECTS}) for ${url}`));
         }
-        const req = https.get(requestUrl, { timeout: timeoutMs }, (res) => {
+        const req = https.get(requestUrl, { timeout: timeoutMs, headers: registryAuthHeaders(requestUrl) }, (res) => {
           if (res.statusCode === 301 || res.statusCode === 302) {
             res.resume();
             const location = res.headers.location;

package/src/shared/registry-auth.js

@@ -0,0 +1,98 @@
+'use strict';
+
+/**
+ * npm registry authentication (2026-06-13).
+ *
+ * A supply-chain scanner fetches thousands of brand-new, never-CDN-cached
+ * packages; anonymous registry.npmjs.org traffic gets aggressively 429-throttled
+ * per-IP (observed: ~500/h of 429s at <1 req/s, scans stalling 20-46s waiting on
+ * metadata tokens). An authenticated token raises the per-account limit and
+ * de-anonymizes us.
+ *
+ * Token resolution (first hit wins), memoized for the process lifetime:
+ *   1. env MUADDIB_NPM_TOKEN  (canonical — set via systemd EnvironmentFile / drop-in)
+ *   2. env NPM_TOKEN          (common fallback)
+ *   3. .npmrc  //registry.npmjs.org/:_authToken=...  (npm-standard; cwd, $HOME, /home/muaddib)
+ *
+ * Auth is applied ONLY to registry.npmjs.org requests — other hosts (pypi.org,
+ * api.npmjs.org, replicate.npmjs.com) get NO header, so the token can never leak
+ * to a third-party host. With no token configured the header set is empty and
+ * behaviour is identical to the previous anonymous path.
+ */
+
+const fs = require('fs');
+const path = require('path');
+
+const AUTH_HOSTS = new Set(['registry.npmjs.org']);
+
+let _resolved = false;
+let _token = null;
+let _source = null;
+
+function _fromNpmrc() {
+  const files = [
+    process.env.MUADDIB_NPMRC,
+    path.join(process.cwd(), '.npmrc'),
+    process.env.HOME ? path.join(process.env.HOME, '.npmrc') : null,
+    '/home/muaddib/.npmrc',
+  ].filter(Boolean);
+  for (const f of files) {
+    let txt;
+    try { txt = fs.readFileSync(f, 'utf8'); } catch { continue; }
+    // npm-standard line: //registry.npmjs.org/:_authToken=<token>
+    const m = txt.match(/^\s*\/\/registry\.npmjs\.org\/:_authToken\s*=\s*(.+?)\s*$/m);
+    if (m) return { token: m[1].replace(/^["']|["']$/g, ''), source: `npmrc:${f}` };
+  }
+  return null;
+}
+
+function getNpmToken() {
+  if (_resolved) return _token;
+  _resolved = true;
+  const env = (process.env.MUADDIB_NPM_TOKEN || process.env.NPM_TOKEN || '').trim();
+  if (env) {
+    _token = env;
+    _source = process.env.MUADDIB_NPM_TOKEN ? 'env:MUADDIB_NPM_TOKEN' : 'env:NPM_TOKEN';
+    return _token;
+  }
+  const rc = _fromNpmrc();
+  if (rc) { _token = rc.token; _source = rc.source; }
+  return _token;
+}
+
+/** {enabled, source, last4} — for the one-time boot log. NEVER returns the token. */
+function npmAuthStatus() {
+  const t = getNpmToken();
+  return { enabled: !!t, source: t ? _source : null, last4: t ? String(t).slice(-4) : null };
+}
+
+let _logged = false;
+function logAuthStatusOnce(logger = console) {
+  if (_logged) return;
+  _logged = true;
+  const s = npmAuthStatus();
+  if (s.enabled) {
+    logger.log(`[REGISTRY-AUTH] npm registry auth ENABLED (source=${s.source}, token …${s.last4})`);
+  } else {
+    logger.warn('[REGISTRY-AUTH] npm registry auth DISABLED — anonymous registry.npmjs.org (set MUADDIB_NPM_TOKEN); expect heavier 429 throttling.');
+  }
+}
+
+/**
+ * Headers to merge into a registry request. Empty object for non-npm hosts or
+ * when no token is configured (→ anonymous, unchanged behaviour).
+ */
+function registryAuthHeaders(url) {
+  // First call doubles as the boot confirmation in the journal.
+  logAuthStatusOnce();
+  let host;
+  try { host = new URL(url).hostname; } catch { return {}; }
+  if (!AUTH_HOSTS.has(host)) return {};
+  const t = getNpmToken();
+  return t ? { Authorization: `Bearer ${t}` } : {};
+}
+
+// Test seam: reset memoized resolution (so a test can flip MUADDIB_NPM_TOKEN).
+function _resetForTests() { _resolved = false; _token = null; _source = null; _logged = false; }
+
+module.exports = { registryAuthHeaders, getNpmToken, npmAuthStatus, logAuthStatusOnce, _resetForTests };