muaddib-scanner 2.11.8 → 2.11.9

package/.env.example

@@ -21,11 +21,19 @@ OSM_API_TOKEN=
 # DISCORD_WEBHOOK_URL=
 
 # ----------------------------------------------------------------------------
-# Tuning gates (already documented in deploy/muaddib-monitor.service)
+# FPR plan gates — DEFAULT ON since v2.11.9 (no need to set these unless opting OUT)
 # ----------------------------------------------------------------------------
-
-# MUADDIB_FN_REACHABILITY=1
-# MUADDIB_DECAY=1
-# MUADDIB_MATURE_CAP=1
-# MUADDIB_METADATA_FACTOR=1
-# MUADDIB_DELTA_MODE=1
+# Measured impact on the v2.11.4 evaluation corpus (1054 packages):
+#   FPR curated 15.6% -> 9.36% (-6.24 pp), FPR random 7.0% -> 2.0% (-5.00 pp).
+#   TPR@3 / TPR@20 / ADR strictly unchanged.
+#
+# Opt-OUT individual gates (uncomment + set to 0):
+# MUADDIB_FN_REACHABILITY=0   # function-level reachability gating
+# MUADDIB_DECAY=0             # group score decay on bundled outputs
+# MUADDIB_MATURE_CAP=0        # cap mature, well-trafficked packages at MEDIUM
+# MUADDIB_METADATA_FACTOR=0   # registry signals -> reputation multiplier
+# MUADDIB_DELTA_MODE=0        # delta scoring against prior versions
+#
+# Skip the npm registry fetch ENTIRELY (disables MATURE_CAP + METADATA_FACTOR
+# + DELTA_MODE in one shot, useful for air-gap / offline CI / perf-critical):
+# MUADDIB_NO_REGISTRY_FETCH=1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "muaddib-scanner",
-  "version": "2.11.8",
+  "version": "2.11.9",
   "description": "Supply-chain threat detection & response for npm & PyPI/Python",
   "main": "src/index.js",
   "bin": {

package/src/ioc/scraper.js

@@ -12,10 +12,37 @@ const { Spinner } = require('../utils.js');
 const { NPM_PACKAGE_REGEX } = require('../shared/constants.js');
 
 // Version format validation (semver-like + wildcard)
-const VERSION_RE = /^(\*|0|[1-9]\d*(\.\d+){0,2}(-[\w.]+)?(\+[\w.]+)?)$/;
+// Permissive version validator — accepts:
+//   - npm semver (1.2.3, 1.2.3-beta.1+build42, 0.x.y)
+//   - PEP 440 (0.1.0b7, 1.2.post1, 1.2.dev0, 0.1.0rc1)
+//   - calendar versioning (2024.5.0)
+//   - 4-segment versions (99.99.99.1, 0.0.7.5)
+//   - wildcard (*)
+// Rejects only structural abuse: path traversal (..), shell metachars,
+// whitespace, slashes, length > 100. The previous regex required first
+// char in [1-9] after a '0' which broke ALL 0.x.y versions (false negative
+// spam in scraper logs ; ~600 valid PyPI/npm versions wrongly skipped per scrape).
+const VERSION_INVALID_CHARS = /[\s\\/'"`;|&$<>(){}\[\]?]/;
+function isValidVersion(version) {
+  if (!version || typeof version !== 'string') return false;
+  if (version === '*') return true;
+  if (version.length > 100) return false;
+  if (version.includes('..')) return false;
+  if (VERSION_INVALID_CHARS.test(version)) return false;
+  // Must start with a digit (or 'v' prefix), and contain only word chars / . / + / -
+  if (!/^v?\d/.test(version)) return false;
+  return /^[\w.+\-]+$/.test(version);
+}
+// Backwards compat: keep VERSION_RE as a no-op test wrapper for any legacy
+// caller that imports it. Prefer isValidVersion() in new code.
+const VERSION_RE = { test: isValidVersion };
 
-// Aggregated warning counter for noisy logs (reset per scraper run)
+// Aggregated warning counters for noisy logs (reset per scraper run).
+// Avoids spamming hundreds of WARN lines for malware feeds with non-standard
+// version strings — a single summary line is logged at the end of runScraper.
 let _noVersionSkipCount = 0;
+let _invalidVersionSkipCount = 0;
+let _invalidVersionSamples = [];  // first 3 samples for context
 
 /**
  * Validate an IOC package entry before insertion.
@@ -37,9 +64,13 @@ function validateIOCEntry(pkgName, version, ecosystem) {
       return false;
     }
   }
-  // Version validation
+  // Version validation — silent counter (aggregated log emitted by runScraper).
+  // The previous per-line WARN was spamming ~600 lines per scrape on PyPI feeds.
   if (version && !VERSION_RE.test(version)) {
-    console.warn(`[WARN] Invalid version skipped: ${version} for ${pkgName}`);
+    _invalidVersionSkipCount++;
+    if (_invalidVersionSamples.length < 3) {
+      _invalidVersionSamples.push(`${version} for ${pkgName}`);
+    }
     return false;
   }
   return true;
@@ -986,6 +1017,8 @@ async function runScraper() {
 
   // Reset aggregated warning counters
   _noVersionSkipCount = 0;
+  _invalidVersionSkipCount = 0;
+  _invalidVersionSamples = [];
 
   // Create data directory if needed
   const dataDir = path.dirname(IOC_FILE);
@@ -1054,6 +1087,12 @@ async function runScraper() {
   if (_noVersionSkipCount > 0) {
     console.log('[SCRAPER] WARN: ' + _noVersionSkipCount + ' packages skipped (no version info, wildcard fallback avoided)');
   }
+  if (_invalidVersionSkipCount > 0) {
+    const samples = _invalidVersionSamples.length > 0
+      ? ' (samples: ' + _invalidVersionSamples.join(', ') + ')'
+      : '';
+    console.log('[SCRAPER] WARN: ' + _invalidVersionSkipCount + ' entries skipped (malformed version)' + samples);
+  }
 
   // Merge all scraped packages
   const allPackages = [

package/src/pipeline/processor.js

@@ -131,11 +131,12 @@ async function process(threats, targetPath, options, pythonDeps, warnings, scann
       debugLog('[REACHABILITY] error:', e?.message);
       // Graceful fallback — treat all files as reachable
     }
-    // FPR plan C2 : function-level reachability sits behind a flag while we
-    // measure FPR delta on the corpus. Off by default so production scans stay
-    // identical until the flag is flipped. Activated only when file-level
-    // reachability succeeded (otherwise no entry-point context to seed from).
-    if (reachableFiles && globalThis.process.env.MUADDIB_FN_REACHABILITY === '1') {
+    // FPR plan C2 : function-level reachability. Default ON since v2.11.9 after
+    // measuring -2.0 pp FPR (curated 11.4% -> 9.4%) with zero TPR/ADR regression
+    // on the full evaluation corpus (1054 packages). Opt-out via
+    // MUADDIB_FN_REACHABILITY=0. Activated only when file-level reachability
+    // succeeded (otherwise no entry-point context to seed from).
+    if (reachableFiles && globalThis.process.env.MUADDIB_FN_REACHABILITY !== '0') {
       try {
         reachableFunctions = computeReachableFunctions(targetPath, reachableFiles);
       } catch (e) {
@@ -169,21 +170,25 @@ async function process(threats, targetPath, options, pythonDeps, warnings, scann
     }
   } catch { /* graceful fallback */ }
 
-  // FPR plan Chantier 4 + 5 wiring : when a package name is known and at least
-  // one of the metadata-driven gates is ON, fetch the npm registry packument
-  // and attach it as _pkgMeta.npmRegistryMeta. Without this, applyReputation-
-  // Factor and applyMatureStableCap cannot fire outside the monitor's own
-  // queue.js (which already pre-fetches the metadata bundle). getPackageMeta-
-  // data has its own in-process cache, so repeated scans of the same package
-  // hit the cache and never re-fetch. Network failure / unknown package -> the
-  // call returns null and both downstream functions degrade gracefully.
+  // FPR plan Chantier 4 + 5 wiring : fetch npm registry packument and attach
+  // it as _pkgMeta.npmRegistryMeta so applyReputationFactor, applyMatureStable-
+  // Cap, and applyDeltaMultiplier can fire. getPackageMetadata has an in-
+  // process cache, so repeated scans of the same package hit the cache and
+  // never re-fetch. Network failure / unknown package -> returns null and all
+  // downstream functions degrade gracefully.
+  //
+  // Default ON since v2.11.9. To skip the fetch entirely (air-gap, offline CI,
+  // perf-critical batch), set MUADDIB_NO_REGISTRY_FETCH=1 — this disables the
+  // 3 metadata-dependent gates (METADATA_FACTOR, MATURE_CAP, DELTA_MODE) in
+  // one shot. Individual gates can still be turned off via their own =0 flag.
   if (
     packageName &&
     _pkgMeta &&
+    globalThis.process.env.MUADDIB_NO_REGISTRY_FETCH !== '1' &&
     (
-      globalThis.process.env.MUADDIB_METADATA_FACTOR === '1' ||
-      globalThis.process.env.MUADDIB_MATURE_CAP === '1' ||
-      globalThis.process.env.MUADDIB_DELTA_MODE === '1'
+      globalThis.process.env.MUADDIB_METADATA_FACTOR !== '0' ||
+      globalThis.process.env.MUADDIB_MATURE_CAP !== '0' ||
+      globalThis.process.env.MUADDIB_DELTA_MODE !== '0'
     )
   ) {
     try {
@@ -293,13 +298,15 @@ async function process(threats, targetPath, options, pythonDeps, warnings, scann
   applyFPReductions(deduped, reachableFiles, packageName, packageDeps, reachableFunctions);
 
   // FPR plan Chantier 3 - delta-aware decay. Threats present in the last 3
-  // published versions (and not HC/IOC) decay to LOW. Off by default until
-  // the cache is warm and we've measured the FPR delta on the corpus.
+  // published versions (and not HC/IOC) decay to LOW. Default ON since v2.11.9.
+  // Opt-out: MUADDIB_DELTA_MODE=0 (or set MUADDIB_NO_REGISTRY_FETCH=1 to skip
+  // the registry fetch upstream). No-op when registry meta is absent (CLI
+  // scans on private packages, offline, or unknown package).
   let _deltaResult = null;
   if (
     packageName && packageVersion &&
     _pkgMeta && _pkgMeta.npmRegistryMeta &&
-    globalThis.process.env.MUADDIB_DELTA_MODE === '1'
+    globalThis.process.env.MUADDIB_DELTA_MODE !== '0'
   ) {
     try {
       const packument = _pkgMeta.npmRegistryMeta.packument || _pkgMeta.npmRegistryMeta;
@@ -446,9 +453,9 @@ async function process(threats, targetPath, options, pythonDeps, warnings, scann
   // FPR plan Chantier 5 : mature stable cap — caps mature, well-owned, high-
   // traffic packages at MEDIUM unless an HC type or IOC is present. Sits
   // BETWEEN the contextual caps (which it composes with) and the single-fire
-  // floor (which can override on hard signals). Gated behind
-  // MUADDIB_MATURE_CAP=1 until measured against the full evaluation corpus.
-  if (globalThis.process.env.MUADDIB_MATURE_CAP === '1') {
+  // floor (which can override on hard signals). Default ON since v2.11.9.
+  // Opt-out: MUADDIB_MATURE_CAP=0. No-op when registry meta is absent.
+  if (globalThis.process.env.MUADDIB_MATURE_CAP !== '0') {
     const matureCap = applyMatureStableCap(result, _pkgMeta && _pkgMeta.npmRegistryMeta);
     if (matureCap && matureCap.applied) {
       debugLog('[MATURE-CAP] ' + (packageName || targetPath) + ': ' +
@@ -470,12 +477,13 @@ async function process(threats, targetPath, options, pythonDeps, warnings, scann
   // Hybrid v3 Phase 4: metadata-first reputation factor — multiplies the score
   // by a factor in [0.10, 1.5] derived from npm registry signals. Applied LAST
   // so all severity logic completes first; the factor is the final, package-
-  // wide context filter. Gated behind MUADDIB_METADATA_FACTOR=1; no-op when
-  // metadata is absent (CLI scans, offline) or the gate is off.
+  // wide context filter. Default ON since v2.11.9. Opt-out:
+  // MUADDIB_METADATA_FACTOR=0. No-op when metadata is absent (CLI scans on
+  // unknown package, offline, MUADDIB_NO_REGISTRY_FETCH=1).
   // NOTE: this module's exported function is named `process`, which shadows
   // the global `process` inside its body. Use globalThis.process.env to reach
   // the real environment.
-  if (globalThis.process.env.MUADDIB_METADATA_FACTOR === '1') {
+  if (globalThis.process.env.MUADDIB_METADATA_FACTOR !== '0') {
     const repAdjust = applyReputationFactor(result, _pkgMeta && _pkgMeta.npmRegistryMeta);
     if (repAdjust) {
       debugLog('[META-FACTOR] ' + (packageName || targetPath) + ': factor=' +
@@ -501,10 +509,10 @@ async function process(threats, targetPath, options, pythonDeps, warnings, scann
   // FPR plan Chantier 3 : persist this version's signature set so future scans
   // (or future versions) can use it as a baseline for delta decay. Best-effort
   // and idempotent ; cache misses on read are silent so a missed write never
-  // blocks scoring. Only write when the user opted in to delta-mode AND we
+  // blocks scoring. Write whenever delta-mode is enabled (default ON) AND we
   // have a concrete package@version pair.
   if (
-    globalThis.process.env.MUADDIB_DELTA_MODE === '1' &&
+    globalThis.process.env.MUADDIB_DELTA_MODE !== '0' &&
     packageName && packageVersion
   ) {
     try {

package/src/scoring.js

@@ -186,7 +186,8 @@ function _isReplacedByCompound(t) {
 }
 
 function computeGroupScore(threats) {
-  if (process.env.MUADDIB_DECAY === '1') return computeGroupScoreDecay(threats);
+  // Score decay default ON since v2.11.9 (FPR plan Chantier 1). Opt-out: MUADDIB_DECAY=0.
+  if (process.env.MUADDIB_DECAY !== '0') return computeGroupScoreDecay(threats);
   let score = 0;
   let protoHookMediumPoints = 0;
   let dataflowMediumPoints = 0;