muaddib-scanner 2.11.91 → 2.11.92

package/bin/muaddib.js

@@ -659,6 +659,19 @@ if (command === 'version' || command === '--version' || command === '-v') {
     console.error('[ERROR]', err.message);
     process.exit(1);
   });
+} else if (command === 'shadow-report') {
+  const { runShadowReport } = require('../src/commands/shadow-report.js');
+  const shOpts = { json: jsonOutput };
+  for (let i = 0; i < options.length; i++) {
+    if (options[i] === '--since' && options[i + 1]) { shOpts.since = options[i + 1]; i++; }
+    else if (options[i] === '--detector' && options[i + 1]) { shOpts.detector = options[i + 1]; i++; }
+  }
+  runShadowReport(shOpts).then(() => {
+    process.exit(0);
+  }).catch(err => {
+    console.error('[ERROR]', err.message);
+    process.exit(1);
+  });
 } else if (command === 'evaluate') {
   if (wantHelp) showHelp('evaluate');
   const { evaluate } = require('../src/commands/evaluate.js');

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "muaddib-scanner",
-  "version": "2.11.91",
+  "version": "2.11.92",
   "description": "Supply-chain threat detection & response for npm & PyPI/Python",
   "main": "src/index.js",
   "bin": {

package/{self-scan-v2.11.91.json → self-scan-v2.11.92.json}

@@ -1,6 +1,6 @@
 {
   "target": "node_modules",
-  "timestamp": "2026-06-11T10:21:58.953Z",
+  "timestamp": "2026-06-11T11:05:03.615Z",
   "threats": [
     {
       "type": "string_mutation_obfuscation",

package/src/commands/shadow-report.js

@@ -0,0 +1,106 @@
+'use strict';
+
+// muaddib shadow-report — read the shadow-mode divergence log and print the
+// V1-vs-V2 adjudication split per detector. This is the read side of
+// src/shared/shadow.js: detectors compute a candidate semantics alongside the
+// live one and log disagreements; this command turns the log into the table a
+// human adjudicates before flipping the semantics.
+//
+// The log only contains DIVERGENCES (agreements are not recorded), so:
+//   old-only  = oldVerdict truthy, newVerdict falsy → alerts V2 would drop (FP killed)
+//   new-only  = newVerdict truthy, oldVerdict falsy → NEW flags (review every one)
+//   changed   = both truthy but different (e.g. severity reclassification)
+
+const { readShadowDivergences } = require('../shared/shadow.js');
+
+const DAY_MS = 24 * 60 * 60 * 1000;
+
+/** Parse `--since 7d` / `--since 12h` / ISO string → ms epoch (null = all). */
+function parseSince(s) {
+  if (!s) return null;
+  const m = /^(\d+)([dh])$/.exec(s);
+  if (m) {
+    const n = parseInt(m[1], 10);
+    return Date.now() - n * (m[2] === 'd' ? DAY_MS : 3600 * 1000);
+  }
+  const p = Date.parse(s);
+  return Number.isNaN(p) ? null : p;
+}
+
+function classify(e) {
+  const oldT = !!e.oldVerdict, newT = !!e.newVerdict;
+  if (oldT && !newT) return 'oldOnly';
+  if (!oldT && newT) return 'newOnly';
+  return 'changed';
+}
+
+async function runShadowReport(opts = {}) {
+  const sinceMs = parseSince(opts.since);
+  const entries = readShadowDivergences({
+    detector: opts.detector || undefined,
+    sinceTs: sinceMs !== null ? sinceMs : undefined
+  });
+
+  if (entries.length === 0) {
+    console.log('\n  No shadow divergences recorded' +
+      (opts.detector ? ` for detector "${opts.detector}"` : '') +
+      (opts.since ? ` since ${opts.since}` : '') +
+      '.\n  (Shadow mode logs only V1≠V2 disagreements; enable with MUADDIB_SHADOW=1.)\n');
+    return;
+  }
+
+  // Group by detector, dedup by package@version (a package rescanned N times
+  // diverges N times — the adjudication unit is the package, not the event).
+  const byDetector = new Map();
+  for (const e of entries) {
+    let d = byDetector.get(e.detector);
+    if (!d) { d = { events: 0, byKey: new Map() }; byDetector.set(e.detector, d); }
+    d.events++;
+    const key = `${e.package || '?'}@${e.version || ''}`;
+    if (!d.byKey.has(key)) d.byKey.set(key, e); // first divergence wins for the listing
+  }
+
+  if (opts.json) {
+    const out = {};
+    for (const [det, d] of byDetector) {
+      const split = { oldOnly: [], newOnly: [], changed: [] };
+      for (const [key, e] of d.byKey) split[classify(e)].push({ key, evidence: e.evidence });
+      out[det] = {
+        events: d.events, distinct: d.byKey.size,
+        oldOnly: split.oldOnly.length, newOnly: split.newOnly.length, changed: split.changed.length,
+        newOnlyList: split.newOnly, oldOnlyExamples: split.oldOnly.slice(0, 20)
+      };
+    }
+    console.log(JSON.stringify(out, null, 2));
+    return;
+  }
+
+  console.log('\n  MUAD\'DIB Shadow Divergence Report' + (opts.since ? ` (since ${opts.since})` : '') + '\n');
+  for (const [det, d] of byDetector) {
+    const split = { oldOnly: [], newOnly: [], changed: [] };
+    for (const [key, e] of d.byKey) split[classify(e)].push({ key, e });
+    console.log(`  ${det}`);
+    console.log(`    divergence events: ${d.events} | distinct pkg@version: ${d.byKey.size}`);
+    console.log(`    old-only (V2 drops the alert — FP killed): ${split.oldOnly.length}`);
+    console.log(`    new-only (V2 adds a flag — REVIEW):        ${split.newOnly.length}`);
+    if (split.changed.length) {
+      console.log(`    changed (both fire, different verdict):   ${split.changed.length}`);
+    }
+    const show = (label, list, max) => {
+      if (!list.length) return;
+      console.log(`    ${label}:`);
+      for (const { key, e } of list.slice(0, max)) {
+        const ev = e.evidence ? JSON.stringify(e.evidence) : '';
+        console.log(`      - ${key}  old=${JSON.stringify(e.oldVerdict)} new=${JSON.stringify(e.newVerdict)} ${ev.slice(0, 140)}`);
+      }
+      if (list.length > max) console.log(`      ... and ${list.length - max} more`);
+    };
+    // Every NEW flag must be human-reviewed (possible FN risk if wrong) — show all.
+    show('new-only detail', split.newOnly, 50);
+    show('old-only examples', split.oldOnly, 20);
+    show('changed detail', split.changed, 20);
+    console.log('');
+  }
+}
+
+module.exports = { runShadowReport, parseSince };

package/src/pipeline/processor.js

@@ -270,7 +270,11 @@ async function process(threats, targetPath, options, pythonDeps, warnings, scann
       debugLog('[EMAIL-DOMAIN] check failed: ' + err.message);
     }
     try {
-      const rdapThreats = await checkCompromisedDomain(_pkgMeta.npmRegistryMeta);
+      // shadowCtx identifies the package in shadow-divergence records (V2
+      // candidate semantics logged alongside V1 — zero effect on threats).
+      const rdapThreats = await checkCompromisedDomain(_pkgMeta.npmRegistryMeta, {
+        shadowCtx: { name: packageName, version: packageVersion, ecosystem: 'npm' }
+      });
       for (const t of rdapThreats) deduped.push(t);
     } catch (err) {
       debugLog('[RDAP] check failed: ' + err.message);

package/src/scanner/email-domain.js

@@ -17,6 +17,7 @@
 
 const dns = require('dns');
 const { debugLog } = require('../utils.js');
+const { isShadowEnabled, recordShadowDivergence } = require('../shared/shadow.js');
 
 const MX_TIMEOUT_MS = 3000;
 const MX_CACHE_TTL = 30 * 24 * 60 * 60 * 1000; // 30 days
@@ -236,10 +237,67 @@ function isCompromisedDomain(creationDateISO, packageCreatedAtISO) {
   return cDate > (rDate - COMPROMISE_MARGIN_MS);
 }
 
+// =============================================================================
+// V2 candidate semantics (SHADOW-ONLY until adjudicated — V1 above still emits
+// every threat). Two changes vs V1, both validated by the node-ipc takeover
+// (May 2026: domain atlantis-software.net re-registered 2026-05-07, malicious
+// 9.2.3/12.0.1 published 05-14, FIRST publish years earlier):
+//
+//  1. STRICT comparison — creation > first_publish, the 30-day pre-publish
+//     margin removed. A dev who buys their domain a few weeks before shipping
+//     v1 is the NORMAL case (the margin was the main source of the 850+ FP);
+//     a dev cannot have published with an email on a domain that did not
+//     exist yet, so creation strictly after first publish stays a hard signal.
+//     RDAP caveat that makes this work: many registries RESET the creation
+//     date on re-registration (.net/Namecheap do — node-ipc's signal).
+//  2. Public email providers excluded — gmail.com etc. can never be "taken
+//     over" by re-registration; any weird RDAP answer for them is noise.
+//     This is a domain-CLASS exclusion, not a package whitelist.
+// =============================================================================
+
+// Consumer email providers — domain takeover does not apply (the provider
+// owns the domain; accounts are compromised via other vectors, out of scope
+// for this RDAP signal).
+const PUBLIC_EMAIL_PROVIDERS = new Set([
+  'gmail.com', 'googlemail.com',
+  'outlook.com', 'hotmail.com', 'live.com', 'msn.com',
+  'yahoo.com', 'ymail.com', 'rocketmail.com',
+  'proton.me', 'protonmail.com', 'pm.me',
+  'icloud.com', 'me.com', 'mac.com',
+  'aol.com',
+  'gmx.com', 'gmx.de', 'gmx.net',
+  'mail.ru', 'inbox.ru', 'list.ru', 'bk.ru',
+  'qq.com', 'foxmail.com', '163.com', '126.com', 'yeah.net', 'sina.com',
+  'yandex.ru', 'yandex.com',
+  'zoho.com', 'fastmail.com', 'hey.com',
+  'tutanota.com', 'tuta.com', 'tuta.io',
+  'web.de', 't-online.de', 'freenet.de',
+  'free.fr', 'orange.fr', 'laposte.net', 'wanadoo.fr', 'sfr.fr',
+  'naver.com', 'daum.net', 'hanmail.net',
+  'rediffmail.com', 'seznam.cz', 'wp.pl', 'o2.pl', 'interia.pl',
+  'duck.com', 'pobox.com', 'hushmail.com', 'mailbox.org', 'posteo.de'
+]);
+
+/**
+ * V2: strict creation-after-first-publish, public providers excluded.
+ * Pure — used by the shadow hook below and by scripts/backtest-email-domain.js.
+ */
+function isCompromisedDomainV2(creationDateISO, firstPublishISO, domain) {
+  if (!creationDateISO || !firstPublishISO) return false;
+  if (domain && PUBLIC_EMAIL_PROVIDERS.has(String(domain).toLowerCase())) return false;
+  const cDate = new Date(creationDateISO).getTime();
+  const rDate = new Date(firstPublishISO).getTime();
+  if (isNaN(cDate) || isNaN(rDate)) return false;
+  return cDate > rDate;
+}
+
 /**
  * F1 entry point.
- * @param {object|null} meta - Digested metadata. Reads maintainer_emails + created_at.
+ * @param {object|null} meta - Digested metadata. Reads maintainer_emails + created_at
+ *   (= the package's FIRST publish date, both npm and PyPI sides).
  * @param {object} options - { fetchRdap } for tests to inject a mock.
+ *   { shadowCtx: {name, version, ecosystem} } identifies the scanned package in
+ *   shadow-divergence records (optional — without it divergences log package:null).
  * @returns {Promise<Array>} threats array
  */
 async function checkCompromisedDomain(meta, options = {}) {
@@ -263,6 +321,24 @@ async function checkCompromisedDomain(meta, options = {}) {
       continue;
     }
     if (!rdap || !rdap.creationDate) continue;
+    // SHADOW (zero effect on the threats emitted below): compare the live V1
+    // verdict with the V2 candidate and log only disagreements. Adjudication =
+    // scripts/backtest-email-domain.js replay + `muaddib shadow-report`.
+    try {
+      if (isShadowEnabled()) {
+        const v1 = isCompromisedDomain(rdap.creationDate, meta.created_at);
+        const v2 = isCompromisedDomainV2(rdap.creationDate, meta.created_at, domain);
+        if (v1 !== v2) {
+          const ctx = options.shadowCtx || {};
+          recordShadowDivergence({
+            detector: 'compromised_email_domain',
+            package: ctx.name, version: ctx.version, ecosystem: ctx.ecosystem,
+            oldVerdict: v1, newVerdict: v2,
+            evidence: { domain, creationDate: rdap.creationDate, firstPublish: meta.created_at, oldMarginDays: 30 }
+          });
+        }
+      }
+    } catch { /* shadow must never affect the scan */ }
     if (isCompromisedDomain(rdap.creationDate, meta.created_at)) {
       const cd = rdap.creationDate.slice(0, 10);
       const pd = meta.created_at.slice(0, 10);
@@ -297,6 +373,9 @@ module.exports = {
   checkCompromisedDomain,
   fetchRdap,
   isCompromisedDomain,
+  // V2 candidate (shadow-only until adjudicated; used by the backtest script)
+  isCompromisedDomainV2,
+  PUBLIC_EMAIL_PROVIDERS,
   _resetRdapCache,
   RDAP_TIMEOUT_MS,
   RDAP_CACHE_TTL,

package/src/scanner/pypi-maintainer.js

@@ -72,7 +72,10 @@ async function runPyPIMaintainerChecks(packageName, pypiRegistryMeta, options =
   let rdapThreats = [];
   try {
     rdapThreats = await checkCompromisedDomain(helperMeta, {
-      fetchRdap: options.fetchRdap
+      fetchRdap: options.fetchRdap,
+      // PyPI created_at is the earliest release time (pypi-registry.js) =
+      // first publish, so the V2 shadow comparison is valid on this side too.
+      shadowCtx: { name: packageName, ecosystem: 'pypi' }
     });
   } catch { /* silent */ }
   for (const t of rdapThreats) threats.push(adaptThreatToPyPI(t, declarationFile));

package/src/shared/shadow.js

@@ -0,0 +1,190 @@
+'use strict';
+
+/**
+ * Shadow-mode divergence framework.
+ *
+ * Lets a detector compute a CANDIDATE new semantics (V2) alongside its live
+ * semantics (V1) and log the cases where the two verdicts disagree — with ZERO
+ * effect on emitted threats, scores, or tiers. The divergence log is the
+ * adjudication input for flipping V1 → V2: replay historical alerts through
+ * the shadow (backtest) or let it run live as a post-merge safety net, then
+ * read the split with `muaddib shadow-report`.
+ *
+ * Contract (fail-safe by construction):
+ *  - Nothing here returns a value the scan pipeline can act on. The framework
+ *    cannot change a verdict even if misused.
+ *  - recordShadowDivergence NEVER throws — a shadow failure must never break a
+ *    scan (same posture as appendScanLedger).
+ *  - Disabled by default. The daemon opts in via MUADDIB_SHADOW=1 in its
+ *    service environment; CLI scans and tests stay inert unless they set it.
+ *  - Bounded: the JSONL file is capped at MUADDIB_SHADOW_MAX entries (default
+ *    50 000) with streaming FIFO compaction — same pattern as the scan-ledger.
+ *
+ * Concurrency: unlike the scan-ledger (main-thread-only writer), this module
+ * is called from INSIDE scan workers (pipeline/processor.js runs there), so N
+ * worker_threads may append concurrently. Each record is serialized to ONE
+ * appendFileSync call of one full line (flag 'a' = O_APPEND; small writes are
+ * serialized by the inode lock on ext4) — never two writes per line. The
+ * reader skips unparsable lines (a crash mid-write can truncate at most the
+ * final line).
+ *
+ * Env (all read at CALL time so tests can re-point after module load):
+ *   MUADDIB_SHADOW=1          enable (default off)
+ *   MUADDIB_SHADOW_FILE=path  divergence log override (tests)
+ *   MUADDIB_SHADOW_MAX=n      entry cap (default 50000)
+ */
+
+const fs = require('fs');
+const path = require('path');
+
+const DEFAULT_SHADOW_FILE = path.join(__dirname, '..', '..', 'data', 'shadow-divergence.jsonl');
+const DEFAULT_MAX_ENTRIES = 50_000;
+const EVIDENCE_MAX_BYTES = 2048;
+// Count lines (cheap streaming pass) only every N appends, not on every write.
+const COMPACT_CHECK_INTERVAL = 500;
+
+let _appendsSinceCheck = 0;
+
+function isShadowEnabled() {
+  return globalThis.process.env.MUADDIB_SHADOW === '1';
+}
+
+function _shadowFile() {
+  return globalThis.process.env.MUADDIB_SHADOW_FILE || DEFAULT_SHADOW_FILE;
+}
+
+function _maxEntries() {
+  const raw = globalThis.process.env.MUADDIB_SHADOW_MAX;
+  const n = raw ? parseInt(raw, 10) : NaN;
+  return (Number.isFinite(n) && n >= 10 && n <= 5_000_000) ? n : DEFAULT_MAX_ENTRIES;
+}
+
+/**
+ * Serialize evidence with a hard size cap. Oversized evidence is replaced by a
+ * truncated string form — the log line must stay small so the single-write
+ * append atomicity argument holds.
+ */
+function _capEvidence(evidence) {
+  if (evidence === undefined || evidence === null) return null;
+  let s;
+  try {
+    s = JSON.stringify(evidence);
+  } catch {
+    s = String(evidence);
+  }
+  if (s.length <= EVIDENCE_MAX_BYTES) {
+    try { return JSON.parse(s); } catch { return s; }
+  }
+  return { _truncated: true, head: s.slice(0, EVIDENCE_MAX_BYTES) };
+}
+
+/**
+ * Record one shadow divergence (oldVerdict !== newVerdict). Call sites are
+ * expected to compare verdicts BEFORE calling — agreements are not logged
+ * (the log captures the would-change population, not every scan).
+ * Never throws. No-op when shadow mode is disabled.
+ *
+ * @param {object} d
+ * @param {string} d.detector    e.g. 'compromised_email_domain'
+ * @param {string} [d.package]
+ * @param {string} [d.version]
+ * @param {string} [d.ecosystem]
+ * @param {*}      d.oldVerdict  live semantics result
+ * @param {*}      d.newVerdict  candidate semantics result
+ * @param {*}      [d.evidence]  capped at 2KB serialized
+ */
+function recordShadowDivergence(d) {
+  try {
+    if (!isShadowEnabled()) return;
+    if (!d || !d.detector) return;
+    const file = _shadowFile();
+    const dir = path.dirname(file);
+    if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
+    const entry = {
+      ts: new Date().toISOString(),
+      detector: String(d.detector),
+      package: d.package || null,
+      version: d.version || null,
+      ecosystem: d.ecosystem || null,
+      oldVerdict: d.oldVerdict !== undefined ? d.oldVerdict : null,
+      newVerdict: d.newVerdict !== undefined ? d.newVerdict : null,
+      evidence: _capEvidence(d.evidence)
+    };
+    // ONE write per line — see the concurrency note in the header.
+    fs.appendFileSync(file, JSON.stringify(entry) + '\n', { encoding: 'utf8', flag: 'a' });
+    _appendsSinceCheck++;
+    if (_appendsSinceCheck >= COMPACT_CHECK_INTERVAL) {
+      _appendsSinceCheck = 0;
+      _compactShadowJsonl(file);
+    }
+  } catch {
+    // Never throw, never log loudly — a shadow failure must not affect scans.
+  }
+}
+
+/**
+ * Streaming FIFO compaction: keep only the most recent max entries.
+ * Local minimal implementation (not shared with state.js) so the worker-side
+ * require graph stays free of the monitor state module.
+ */
+function _compactShadowJsonl(file) {
+  try {
+    const max = _maxEntries();
+    const lines = _readLines(file);
+    if (lines.length <= max) return;
+    const kept = lines.slice(lines.length - max);
+    const tmp = file + '.tmp';
+    fs.writeFileSync(tmp, kept.join('\n') + '\n', 'utf8');
+    fs.renameSync(tmp, file);
+  } catch {
+    // Best-effort; an oversized shadow log is preferable to a crashed scan.
+  }
+}
+
+/** Read raw lines, dropping empties. Returns [] on any error. */
+function _readLines(file) {
+  try {
+    return fs.readFileSync(file, 'utf8').split('\n').filter(l => l.trim().length > 0);
+  } catch {
+    return [];
+  }
+}
+
+/**
+ * Read divergence entries, tolerant of corrupt lines (skipped silently).
+ * @param {object} [opts]
+ * @param {string} [opts.detector] filter by detector
+ * @param {number|string} [opts.sinceTs] ms epoch or ISO — entries older are skipped
+ * @returns {Array<object>}
+ */
+function readShadowDivergences(opts = {}) {
+  let sinceMs = null;
+  if (typeof opts.sinceTs === 'number' && Number.isFinite(opts.sinceTs)) sinceMs = opts.sinceTs;
+  else if (typeof opts.sinceTs === 'string') {
+    const p = Date.parse(opts.sinceTs);
+    if (!Number.isNaN(p)) sinceMs = p;
+  }
+  const out = [];
+  for (const line of _readLines(_shadowFile())) {
+    let e;
+    try { e = JSON.parse(line); } catch { continue; } // truncated/corrupt line
+    if (!e || typeof e !== 'object' || !e.detector) continue;
+    if (opts.detector && e.detector !== opts.detector) continue;
+    if (sinceMs !== null) {
+      const t = e.ts ? Date.parse(e.ts) : NaN;
+      if (Number.isNaN(t) || t < sinceMs) continue;
+    }
+    out.push(e);
+  }
+  return out;
+}
+
+module.exports = {
+  isShadowEnabled,
+  recordShadowDivergence,
+  readShadowDivergences,
+  // test seams
+  _capEvidence,
+  _compactShadowJsonl,
+  EVIDENCE_MAX_BYTES
+};