muaddib-scanner 2.11.111 → 2.11.113

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "muaddib-scanner",
-  "version": "2.11.111",
+  "version": "2.11.113",
   "description": "Supply-chain threat detection & response for npm & PyPI/Python",
   "main": "src/index.js",
   "bin": {

package/{self-scan-v2.11.111.json → self-scan-v2.11.113.json}

@@ -1,6 +1,6 @@
 {
   "target": "node_modules",
-  "timestamp": "2026-06-13T09:27:21.416Z",
+  "timestamp": "2026-06-14T08:06:18.378Z",
   "threats": [
     {
       "type": "string_mutation_obfuscation",

package/src/monitor/ingestion.js

@@ -9,6 +9,7 @@
 
 const https = require('https');
 const { acquireRegistrySlot, releaseRegistrySlot } = require('../shared/http-limiter.js');
+const { registryAuthHeaders } = require('../shared/registry-auth.js');
 const { loadCachedIOCs } = require('../ioc/updater.js');
 const { enqueueScan } = require('./scan-queue.js');
 const {
@@ -99,7 +100,7 @@ function httpsGet(url, timeoutMs = 30_000, deadlineMs = Math.max(timeoutMs * 2,
       clearTimeout(deadline);
       if (err) reject(err); else resolve(value);
     };
-    req = _deps.https.get(url, { timeout: timeoutMs }, (res) => {
+    req = _deps.https.get(url, { timeout: timeoutMs, headers: registryAuthHeaders(url) }, (res) => {
       if (res.statusCode === 301 || res.statusCode === 302) {
         res.resume();
         const location = res.headers.location;

package/src/monitor/webhook.js

@@ -1252,6 +1252,13 @@ function buildDailyReportEmbed(stats, dailyAlerts, ledgerRollup) {
     const pct = (ledger.distinctCoverage * 100).toFixed(0);
     const approx = ledger.exactVanished === false ? '~' : '';
     coverageText = `${ledger.distinctScanned}/${ledger.distinctPackages} pkgs (${approx}${pct}%)`;
+    // Honest 24h coverage loss surfaced next to coverage: `vanished` = distinct names
+    // dropped and never re-scanned in the window — the real miss count. The raw
+    // `dropped` aggregate (which also folds in recoverable spill + retries, so it
+    // OVERSTATES loss) is relegated to the Ops embed's Ledger field, not the headline.
+    if (ledger.vanished > 0) {
+      coverageText += ` · ${ledger.exactVanished ? '' : '≥'}${ledger.vanished} vanished`;
+    }
     if (published > 0) coverageText += `\nRaw events: ${attempted}/${published}`;
     coverageText += opsSuffix;
   } else if (published > 0) {
@@ -1343,14 +1350,7 @@ function buildDailyReportEmbed(stats, dailyAlerts, ledgerRollup) {
         { name: 'vs Yesterday', value: trendsText, inline: false },
         { name: 'ML', value: mlText, inline: true },
         { name: 'LLM Detective', value: llmText, inline: true },
-        { name: 'Top Suspects', value: top3Text, inline: false },
-        ...((stats.sandboxDeferred || stats.deferredProcessed || stats.deferredExpired)
-          ? [{ name: 'Deferred Sandbox', value: `Enqueued: ${stats.sandboxDeferred || 0} | Processed: ${stats.deferredProcessed || 0} | Expired: ${stats.deferredExpired || 0}`, inline: false }]
-          : []),
-        { name: 'Stability', value: _stabilityFieldValue(stats), inline: false },
-        { name: 'Degradations', value: _degradationsFieldValue(), inline: false },
-        ...(ledgerField ? [ledgerField] : []),
-        { name: 'System', value: healthText, inline: false }
+        { name: 'Top Suspects', value: top3Text, inline: false }
       ],
       footer: {
         // Headline-source annotation: 'ledger' = window-exact [last report → now]
@@ -1359,6 +1359,30 @@ function buildDailyReportEmbed(stats, dailyAlerts, ledgerRollup) {
         text: `MUAD'DIB - Daily summary | headline: ${headline ? 'ledger — completed/deduped, exact 24h window' : 'counters (in-memory fallback)'} | ${readableTime}`
       },
       timestamp: now.toISOString()
+    }, {
+      // --- Embed 2: Ops / system state (kept OUT of the daily headline) ---
+      // Operator feedback: a daily that mixes 24h outcome with multi-day system state
+      // reads as failure when it isn't. Each line here carries its own clock:
+      //   • Ledger      → 24h window. Its `dropped` folds in recoverable spill + retries,
+      //                   so it OVERSTATES loss — `vanished` (in the Coverage field) is the
+      //                   honest miss count, which is why dropped sits here, not the headline.
+      //   • Stability   → cumulative since the 08:00 reset (backlog = point-in-time depth
+      //                   of the persistent spill file, the one snapshot in this field).
+      //   • Degradations / System → instantaneous snapshot (degradations have no TTL: if
+      //                   shown, the condition is active right now, not earlier in the window).
+      title: '⚙️ Ops / état système',
+      color: 0x95a5a6,
+      description: 'Ledger = fenêtre 24h (dropped inclut le spill récupérable — voir « vanished » pour la perte réelle) · Stability = cumulé depuis 08:00 (backlog = instantané) · Degradations/System = instantané',
+      fields: [
+        ...((stats.sandboxDeferred || stats.deferredProcessed || stats.deferredExpired)
+          ? [{ name: 'Deferred Sandbox', value: `Enqueued: ${stats.sandboxDeferred || 0} | Processed: ${stats.deferredProcessed || 0} | Expired: ${stats.deferredExpired || 0}`, inline: false }]
+          : []),
+        { name: 'Stability (cumulé depuis 08:00)', value: _stabilityFieldValue(stats), inline: false },
+        { name: 'Degradations (actif maintenant)', value: _degradationsFieldValue(), inline: false },
+        ...(ledgerField ? [ledgerField] : []),
+        { name: 'System', value: healthText, inline: false }
+      ],
+      timestamp: now.toISOString()
     }]
   };
 }

package/src/scanner/npm-registry.js

@@ -1,6 +1,7 @@
 const { NPM_PACKAGE_REGEX } = require('../shared/constants.js');
 const { debugLog } = require('../utils.js');
 const { acquireRegistrySlot, releaseRegistrySlot, awaitRateToken, signal429, hostForUrl } = require('../shared/http-limiter.js');
+const { registryAuthHeaders } = require('../shared/registry-auth.js');
 const { computeAdvancedRegistrySignals } = require('../integrations/registry-signals.js');
 
 const REGISTRY_URL = 'https://registry.npmjs.org';
@@ -12,6 +13,16 @@ const SEARCH_URL = 'https://registry.npmjs.org/-/v1/search';
 const REQUEST_TIMEOUT = Math.max(1000, parseInt(process.env.MUADDIB_REGISTRY_TIMEOUT_MS, 10) || 10000); // 10s default
 const MAX_RETRIES = Math.max(1, parseInt(process.env.MUADDIB_REGISTRY_RETRIES, 10) || 5);
 
+// Per-maintainer cache for the /-/v1/search author-count lookup — the only
+// DYNAMIC (non-CDN), rate-limited registry endpoint we hit per scan. Without it,
+// firing the search on every scan generated the bulk of the monitor's 429s and
+// the shared brain then throttled the (healthy, CDN-served) packument reads too.
+// See getPackageMetadata. Env-tunable; defaults preserve the signal.
+const _authorCountCache = new Map(); // maintainer → { count, at }
+const AUTHOR_CACHE_TTL_MS = Math.max(0, parseInt(process.env.MUADDIB_AUTHOR_CACHE_TTL_MS, 10) || 3_600_000); // 1h
+const AUTHOR_CACHE_MAX = Math.max(100, parseInt(process.env.MUADDIB_AUTHOR_CACHE_MAX, 10) || 5000);
+const AUTHOR_SEARCH_ENABLED = process.env.MUADDIB_NPM_AUTHOR_SEARCH !== '0'; // kill-switch
+
 /**
  * Create a timeout signal, with fallback for older Node versions.
  * Returns { signal, cleanup } — call cleanup() after fetch to prevent timer leaks.
@@ -25,7 +36,7 @@ function createTimeoutSignal(ms) {
   return { signal: controller.signal, cleanup: () => clearTimeout(timer) };
 }
 
-async function fetchWithRetry(url) {
+async function fetchWithRetry(url, opts = {}) {
   for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {
     // The caller's acquireRegistrySlot paid the rate token for the FIRST
     // attempt only. Every retry is a new network request and must pay its own
@@ -43,7 +54,7 @@ async function fetchWithRetry(url) {
     let response;
     const { signal, cleanup } = createTimeoutSignal(REQUEST_TIMEOUT);
     try {
-      response = await fetch(url, { signal });
+      response = await fetch(url, { signal, headers: registryAuthHeaders(url) });
     } catch {
       cleanup();
       // REG-001: Retry on timeout/abort instead of returning null immediately.
@@ -71,7 +82,16 @@ async function fetchWithRetry(url) {
     // Retry-After (capped at 30s) with jitter so retries don't re-synchronize.
     if (response.status === 429) {
       try { await response.text(); } catch (e) { debugLog('response drain failed:', e.message); }
-      try { signal429(); } catch { /* limiter is best-effort */ }
+      // Back off the CORRECT host's bucket. This previously defaulted to
+      // registry.npmjs.org, so a 429 from api.npmjs.org/downloads (a SEPARATE,
+      // aggressively rate-limited host that 429s ~every request) poisoned the
+      // registry brain and stalled the tarball/packument fetches that were
+      // themselves healthy — the measured ~20s/scan throughput wall.
+      try { signal429(hostForUrl(url)); } catch { /* limiter is best-effort */ }
+      // Best-effort callers (the weekly-downloads reputation signal, whose
+      // endpoint 429s on essentially every request) opt out of the retry storm:
+      // retrying 5× with ~2s sleeps just burns ~10s/scan to still return null.
+      if (opts.noRetryOn429) return null;
       const retryAfter = parseInt(response.headers.get('retry-after'), 10);
       const base = Math.min(retryAfter && retryAfter > 0 ? retryAfter * 1000 : 2000, 30000);
       await new Promise(r => setTimeout(r, Math.round(base * (0.5 + Math.random() * 0.5))));
@@ -170,26 +190,45 @@ async function getPackageMetadata(packageName) {
   }
   const provenanceRegressed = !latestHasProvenance && anyPriorHadProvenance;
 
-  // 2. Weekly downloads + author search (parallel)
+  // 2. Weekly downloads + author package count (parallel).
+  // The author count comes from /-/v1/search?text=maintainer: which — unlike the
+  // CDN-served packument — is DYNAMIC, slow (~300-950ms) and the one per-scan call
+  // npm aggressively rate-limits. A TTL cache keyed on the maintainer collapses
+  // the search volume (maintainers repeat heavily: scopes / bots / monorepos)
+  // while keeping author_package_count byte-identical. MUADDIB_NPM_AUTHOR_SEARCH=0
+  // drops the call entirely — the count then stays absent, exactly as the
+  // pre-resolve fast path already leaves it.
   const downloadsUrl = DOWNLOADS_URL + '/' + encodeURIComponent(packageName);
-  const authorUrl = maintainer
+  const authorUrl = (AUTHOR_SEARCH_ENABLED && maintainer)
     ? SEARCH_URL + '?text=maintainer:' + encodeURIComponent(maintainer) + '&size=1'
     : null;
 
-  async function fetchAuthorWithSlot() {
-    if (!authorUrl) return null;
+  async function getAuthorPackageCount() {
+    if (!authorUrl) return 0;
+    const hit = _authorCountCache.get(maintainer);
+    if (hit && (Date.now() - hit.at) < AUTHOR_CACHE_TTL_MS) return hit.count;
     await acquireRegistrySlot();
-    try { return await fetchWithRetry(authorUrl); }
+    let data;
+    try { data = await fetchWithRetry(authorUrl); }
     finally { releaseRegistrySlot(); }
+    // 429-exhausted / error → fetchWithRetry returns null: reuse a stale entry if
+    // present and do NOT cache the miss (a transient 0 would poison the typosquat
+    // "author has ≤1 package" signal).
+    if (!data) return hit ? hit.count : 0;
+    const count = data.total ?? 0;
+    if (_authorCountCache.size >= AUTHOR_CACHE_MAX) {
+      _authorCountCache.delete(_authorCountCache.keys().next().value); // FIFO evict (bounded)
+    }
+    _authorCountCache.set(maintainer, { count, at: Date.now() });
+    return count;
   }
 
-  const [downloadsData, authorData] = await Promise.all([
-    fetchWithRetry(downloadsUrl),    // api.npmjs.org — no semaphore needed
-    fetchAuthorWithSlot()            // registry.npmjs.org — semaphore protected
+  const [downloadsData, authorPackageCount] = await Promise.all([
+    fetchWithRetry(downloadsUrl, { noRetryOn429: true }),    // api.npmjs.org — rate-limited; best-effort single shot (no retry storm, correct-host backoff)
+    getAuthorPackageCount()          // registry.npmjs.org search — cached + kill-switchable
   ]);
 
   const weeklyDownloads = downloadsData?.downloads ?? 0;
-  const authorPackageCount = authorData?.total ?? 0;
   const versionCount = meta.versions ? Object.keys(meta.versions).length : 0;
   const description = (typeof latestMeta?.description === 'string' ? latestMeta.description
     : (typeof meta.description === 'string' ? meta.description : ''));

package/src/shared/download.js

@@ -4,6 +4,7 @@ const path = require('path');
 const { execFileSync } = require('child_process');
 const AdmZip = require('adm-zip');
 const { MAX_TARBALL_SIZE, DOWNLOAD_TIMEOUT } = require('./constants.js');
+const { registryAuthHeaders } = require('./registry-auth.js');
 
 // Allowed redirect domains for tarball downloads (SSRF protection)
 const ALLOWED_DOWNLOAD_DOMAINS = [
@@ -159,7 +160,7 @@ function downloadToFile(url, destPath, timeoutMs = DOWNLOAD_TIMEOUT) {
         if (redirectCount >= MAX_REDIRECTS) {
           return reject(new Error(`Too many redirects (${MAX_REDIRECTS}) for ${url}`));
         }
-        const req = https.get(requestUrl, { timeout: timeoutMs }, (res) => {
+        const req = https.get(requestUrl, { timeout: timeoutMs, headers: registryAuthHeaders(requestUrl) }, (res) => {
           if (res.statusCode === 301 || res.statusCode === 302) {
             res.resume();
             const location = res.headers.location;

package/src/shared/registry-auth.js

@@ -0,0 +1,98 @@
+'use strict';
+
+/**
+ * npm registry authentication (2026-06-13).
+ *
+ * A supply-chain scanner fetches thousands of brand-new, never-CDN-cached
+ * packages; anonymous registry.npmjs.org traffic gets aggressively 429-throttled
+ * per-IP (observed: ~500/h of 429s at <1 req/s, scans stalling 20-46s waiting on
+ * metadata tokens). An authenticated token raises the per-account limit and
+ * de-anonymizes us.
+ *
+ * Token resolution (first hit wins), memoized for the process lifetime:
+ *   1. env MUADDIB_NPM_TOKEN  (canonical — set via systemd EnvironmentFile / drop-in)
+ *   2. env NPM_TOKEN          (common fallback)
+ *   3. .npmrc  //registry.npmjs.org/:_authToken=...  (npm-standard; cwd, $HOME, /home/muaddib)
+ *
+ * Auth is applied ONLY to registry.npmjs.org requests — other hosts (pypi.org,
+ * api.npmjs.org, replicate.npmjs.com) get NO header, so the token can never leak
+ * to a third-party host. With no token configured the header set is empty and
+ * behaviour is identical to the previous anonymous path.
+ */
+
+const fs = require('fs');
+const path = require('path');
+
+const AUTH_HOSTS = new Set(['registry.npmjs.org']);
+
+let _resolved = false;
+let _token = null;
+let _source = null;
+
+function _fromNpmrc() {
+  const files = [
+    process.env.MUADDIB_NPMRC,
+    path.join(process.cwd(), '.npmrc'),
+    process.env.HOME ? path.join(process.env.HOME, '.npmrc') : null,
+    '/home/muaddib/.npmrc',
+  ].filter(Boolean);
+  for (const f of files) {
+    let txt;
+    try { txt = fs.readFileSync(f, 'utf8'); } catch { continue; }
+    // npm-standard line: //registry.npmjs.org/:_authToken=<token>
+    const m = txt.match(/^\s*\/\/registry\.npmjs\.org\/:_authToken\s*=\s*(.+?)\s*$/m);
+    if (m) return { token: m[1].replace(/^["']|["']$/g, ''), source: `npmrc:${f}` };
+  }
+  return null;
+}
+
+function getNpmToken() {
+  if (_resolved) return _token;
+  _resolved = true;
+  const env = (process.env.MUADDIB_NPM_TOKEN || process.env.NPM_TOKEN || '').trim();
+  if (env) {
+    _token = env;
+    _source = process.env.MUADDIB_NPM_TOKEN ? 'env:MUADDIB_NPM_TOKEN' : 'env:NPM_TOKEN';
+    return _token;
+  }
+  const rc = _fromNpmrc();
+  if (rc) { _token = rc.token; _source = rc.source; }
+  return _token;
+}
+
+/** {enabled, source, last4} — for the one-time boot log. NEVER returns the token. */
+function npmAuthStatus() {
+  const t = getNpmToken();
+  return { enabled: !!t, source: t ? _source : null, last4: t ? String(t).slice(-4) : null };
+}
+
+let _logged = false;
+function logAuthStatusOnce(logger = console) {
+  if (_logged) return;
+  _logged = true;
+  const s = npmAuthStatus();
+  if (s.enabled) {
+    logger.log(`[REGISTRY-AUTH] npm registry auth ENABLED (source=${s.source}, token …${s.last4})`);
+  } else {
+    logger.warn('[REGISTRY-AUTH] npm registry auth DISABLED — anonymous registry.npmjs.org (set MUADDIB_NPM_TOKEN); expect heavier 429 throttling.');
+  }
+}
+
+/**
+ * Headers to merge into a registry request. Empty object for non-npm hosts or
+ * when no token is configured (→ anonymous, unchanged behaviour).
+ */
+function registryAuthHeaders(url) {
+  // First call doubles as the boot confirmation in the journal.
+  logAuthStatusOnce();
+  let host;
+  try { host = new URL(url).hostname; } catch { return {}; }
+  if (!AUTH_HOSTS.has(host)) return {};
+  const t = getNpmToken();
+  return t ? { Authorization: `Bearer ${t}` } : {};
+}
+
+// Test seam: reset memoized resolution (so a test can flip MUADDIB_NPM_TOKEN).
+function _resetForTests() { _resolved = false; _token = null; _source = null; _logged = false; }
+
+module.exports = { registryAuthHeaders, getNpmToken, npmAuthStatus, logAuthStatusOnce, _resetForTests };