muaddib-scanner 2.10.68 → 2.10.70

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "muaddib-scanner",
-  "version": "2.10.68",
+  "version": "2.10.70",
   "description": "Supply-chain threat detection & response for npm & PyPI/Python",
   "main": "src/index.js",
   "bin": {

package/src/ml/classifier.js

@@ -126,16 +126,20 @@ function resetShadowModel() {
 }
 
 /**
- * Run shadow model prediction and log comparison with main model.
- * Never affects the actual classification decision.
+ * Run shadow model prediction and log result.
+ * NEVER affects the actual classification decision — log-only.
+ *
+ * Runs independently of the main model's guard rails so that shadow
+ * predictions are logged for ALL packages with score >= 20, not just
+ * T1 zone. This provides validation data across the full score range
+ * before the shadow model is promoted to production.
  *
  * @param {Object} result - scan result
  * @param {Object} meta - enriched metadata
- * @param {string} mainPrediction - the main model's prediction
- * @param {number} mainProbability - the main model's probability
  * @param {string} packageName - for logging
+ * @param {number} score - risk score (for log context)
  */
-function runShadowComparison(result, meta, mainPrediction, mainProbability, packageName) {
+function runShadowPrediction(result, meta, packageName, score) {
   const shadow = loadShadowModel();
   if (!shadow) return;
 
@@ -151,20 +155,21 @@ function runShadowComparison(result, meta, mainPrediction, mainProbability, pack
   }
 
   const shadowProb = sigmoid(margin);
+  const roundedP = Math.round(shadowProb * 1000) / 1000;
   const shadowPred = shadowProb >= shadow.threshold ? 'malicious' : 'clean';
 
   _shadowStats.total++;
-  if (shadowPred === mainPrediction) {
-    _shadowStats.agree++;
-  } else {
+  if (shadowPred === 'malicious') {
     _shadowStats.disagree++;
-    console.log(`[ML-SHADOW] Disagreement on ${packageName}: main=${mainPrediction}(${mainProbability}) shadow=${shadowPred}(${Math.round(shadowProb * 1000) / 1000}) [${_shadowStats.disagree}/${_shadowStats.total} disagree]`);
+    console.log(`[ML-SHADOW] ${packageName} → ${shadowPred} (p=${roundedP}, score=${score}) [${_shadowStats.disagree}/${_shadowStats.total} flagged]`);
+  } else {
+    _shadowStats.agree++;
   }
 
   // Periodic summary every 100 classifications
   if (_shadowStats.total % 100 === 0) {
-    const agreeRate = ((_shadowStats.agree / _shadowStats.total) * 100).toFixed(1);
-    console.log(`[ML-SHADOW] Stats: ${_shadowStats.total} total, ${agreeRate}% agree, ${_shadowStats.disagree} disagree`);
+    const flagRate = ((_shadowStats.disagree / _shadowStats.total) * 100).toFixed(1);
+    console.log(`[ML-SHADOW] Stats: ${_shadowStats.total} total, ${_shadowStats.disagree} flagged (${flagRate}%), ${_shadowStats.agree} clean`);
   }
 }
 
@@ -371,13 +376,6 @@ function classifyPackage(result, meta) {
 
   const roundedProb = Math.round(probability * 1000) / 1000;
 
-  // Shadow model comparison (log-only, never affects decision)
-  if (isShadowModelAvailable()) {
-    const pkgName = (result && result.summary && result.summary.packageName) ||
-                    (meta && meta.name) || 'unknown';
-    runShadowComparison(result, meta, prediction, roundedProb, pkgName);
-  }
-
   return {
     prediction,
     probability: roundedProb,
@@ -401,9 +399,10 @@ module.exports = {
   loadBundlerModel,
   predictBundler,
   buildBundlerFeatureVector,
-  // Shadow model (ML1 v2, log-only comparison)
+  // Shadow model (ML1 v2, log-only prediction)
   isShadowModelAvailable,
   resetShadowModel,
   loadShadowModel,
+  runShadowPrediction,
   getShadowStats
 };

package/src/ml/train-xgboost.py

@@ -359,23 +359,36 @@ def filter_leaky_features(X: pd.DataFrame, y: np.ndarray,
     return X_filtered, retained
 
 
-def source_discrimination_gate(X: pd.DataFrame, y: np.ndarray,
-                                active_features: list,
-                                max_accuracy: float = 0.65) -> bool:
+def source_discrimination_diagnostic(X: pd.DataFrame, y: np.ndarray,
+                                      active_features: list):
     """
-    Step 2c: Hard gate — verify that retained behavioral features cannot
-    trivially distinguish data source (monitor vs Datadog).
+    Step 2c: Source discrimination diagnostic (LOG-ONLY, non-blocking).
+
+    DESIGN NOTE: This test cannot function as a hard gate when source labels
+    are perfectly confounded with class labels (all negatives = monitor,
+    all positives = Datadog). In that case, legitimate behavioral features
+    (score, count_critical, type_*) will dominate the discriminator because
+    malware genuinely behaves differently from clean packages — this is
+    signal, not leak.
 
-    Since all negatives come from monitor and all positives from Datadog,
-    y IS the source label. A shallow classifier that achieves accuracy > 65%
-    on the retained features indicates residual source-identity leaks.
+    A true source discrimination test would require either:
+    (a) positives re-scanned through our own pipeline, or
+    (b) negatives and positives from the SAME source.
 
-    Returns: True if gate passes (accuracy <= max_accuracy), False if fails.
-    Prints SHAP top 10 of the discriminator to identify offending features.
+    This diagnostic still serves a purpose: it flags NON-BEHAVIORAL features
+    that shouldn't appear in the top discriminators. If metadata features
+    (unpacked_size_bytes, file_count_total, etc.) appear despite being
+    excluded in Step 2a, something is wrong.
+
+    The real validation happens in shadow deployment on live production data.
     """
     print("\n" + "=" * 60)
-    print(f"[Step 2c/8] Source discrimination gate (threshold={max_accuracy:.0%})...")
+    print("[Step 2c/8] Source discrimination diagnostic (log-only)...")
     print("=" * 60)
+    print("  NOTE: source=Datadog correlates 100% with label=malicious.")
+    print("  This diagnostic checks for non-behavioral features in the")
+    print("  top discriminators, NOT for overall accuracy (which will")
+    print("  always be high due to the source/label confound).")
 
     X_active = X[active_features]
 
@@ -385,7 +398,6 @@ def source_discrimination_gate(X: pd.DataFrame, y: np.ndarray,
     )
 
     # Shallow model — depth=3, 50 rounds, no class weighting
-    # (we want to detect ANY discriminability, not optimize for one class)
     params = {
         'objective': 'binary:logistic',
         'eval_metric': 'logloss',
@@ -407,35 +419,52 @@ def source_discrimination_gate(X: pd.DataFrame, y: np.ndarray,
     p = precision_score(y_te, preds, zero_division=0)
     r = recall_score(y_te, preds, zero_division=0)
 
-    print(f"  Discrimination accuracy: {accuracy:.3f} (P={p:.3f} R={r:.3f})")
+    print(f"\n  Discrimination accuracy: {accuracy:.3f} (P={p:.3f} R={r:.3f})")
+    print(f"  (Expected to be high due to source/label confound)")
 
-    # SHAP analysis to identify which features drive discrimination
+    # SHAP analysis — the diagnostic value is in WHICH features dominate
     explainer = shap.TreeExplainer(model)
     shap_values = explainer.shap_values(X_te)
     mean_abs_shap = np.abs(shap_values).mean(axis=0)
     importance = sorted(zip(active_features, mean_abs_shap),
                         key=lambda x: x[1], reverse=True)
 
-    print(f"\n  Top 10 features driving source discrimination:")
+    # Known behavioral features that SHOULD dominate (malware scores higher)
+    EXPECTED_BEHAVIORAL = {
+        'score', 'global_risk_score', 'max_file_score', 'package_score',
+        'count_total', 'count_critical', 'count_high', 'count_medium',
+        'count_low', 'distinct_threat_types', 'severity_ratio_high',
+        'max_single_points', 'points_concentration', 'file_count_with_threats',
+        'file_score_mean', 'file_score_max', 'threat_density',
+    }
+    # Features that should NOT appear (already excluded, but sanity check)
+    EXCLUDED_CHECK = {
+        'unpacked_size_bytes', 'file_count_total', 'has_tests',
+        'dep_count', 'dev_dep_count', 'reputation_factor',
+        'package_age_days', 'weekly_downloads', 'version_count',
+        'author_package_count', 'has_repository', 'readme_size',
+    }
+
+    print(f"\n  Top 10 features driving discrimination:")
+    has_leak = False
     for i, (name, val) in enumerate(importance[:10]):
-        flag = ""
-        # Flag non-behavioral features that shouldn't be discriminative
-        if name in ('unpacked_size_bytes', 'file_count_total', 'has_tests',
-                     'dep_count', 'dev_dep_count', 'reputation_factor'):
-            flag = " *** NON-BEHAVIORAL"
+        if name in EXCLUDED_CHECK:
+            flag = " *** LEAK — should have been excluded in Step 2a!"
+            has_leak = True
+        elif name in EXPECTED_BEHAVIORAL:
+            flag = " (expected — behavioral)"
+        elif name.startswith('type_') or name.startswith('has_'):
+            flag = " (behavioral signal)"
+        else:
+            flag = ""
         print(f"    {i + 1:2d}. {name:40s} {val:.6f}{flag}")
 
-    if accuracy <= max_accuracy:
-        print(f"\n  [GATE PASS] Accuracy {accuracy:.3f} <= {max_accuracy:.3f}")
-        print(f"  Behavioral features do not trivially encode source identity.")
-        return True
+    if has_leak:
+        print(f"\n  [WARNING] Non-behavioral features found in top discriminators!")
+        print(f"  Check EXCLUDED_METADATA — some metadata features leaked through.")
     else:
-        print(f"\n  [GATE FAIL] Accuracy {accuracy:.3f} > {max_accuracy:.3f}")
-        print(f"  Retained features still encode source identity.")
-        print(f"  Offending features (exclude and re-run):")
-        for name, val in importance[:5]:
-            print(f"    - {name} (SHAP={val:.6f})")
-        return False
+        print(f"\n  [OK] Top discriminators are all behavioral features.")
+        print(f"  No metadata/source-proxy leak detected.")
 
 
 def split_data(X: pd.DataFrame, y: np.ndarray) -> tuple:
@@ -836,18 +865,15 @@ def main():
     else:
         active_features = list(remaining_features)
 
-    # Step 2c: Source discrimination gate — HARD STOP if features encode source
+    # Step 2c: Source discrimination diagnostic (log-only).
+    # NOT a hard gate — source label is 100% confounded with class label
+    # (all positives = Datadog, all negatives = monitor), so behavioral
+    # features will always dominate the discriminator. The diagnostic
+    # checks that no METADATA features leaked through Step 2a.
     if not args.skip_gate:
-        gate_pass = source_discrimination_gate(X, y, active_features)
-        if not gate_pass:
-            print("\n" + "=" * 60)
-            print("ABORTED: Source discrimination gate failed.")
-            print("The retained features still encode source identity.")
-            print("Add offending features to EXCLUDED_METADATA and re-run.")
-            print("=" * 60)
-            sys.exit(1)
+        source_discrimination_diagnostic(X, y, active_features)
     else:
-        print("\n  [Step 2c] Source discrimination gate SKIPPED (--skip-gate)")
+        print("\n  [Step 2c] Source discrimination diagnostic SKIPPED (--skip-gate)")
 
     # Class imbalance weight
     n_neg = stats['n_neg']

package/src/monitor/queue.js

@@ -647,6 +647,24 @@ async function scanPackage(name, version, ecosystem, tarballUrl, registryMeta, s
           }
         }
 
+        // Shadow model: log-only prediction for ALL score >= 20 npm packages.
+        // Runs independently of classifyPackage — no effect on mlResult, webhooks,
+        // or any decisions. Collects shadow validation data for the retrained model.
+        if (riskScore >= 20 && ecosystem === 'npm') {
+          try {
+            const { isShadowModelAvailable, runShadowPrediction } = require('../ml/classifier.js');
+            if (isShadowModelAvailable()) {
+              const shadowMeta = { npmRegistryMeta, fileCountTotal, hasTests, unpackedSize: meta.unpackedSize, registryMeta: meta };
+              runShadowPrediction(result, shadowMeta, `${name}@${version}`, riskScore);
+            }
+          } catch (err) {
+            // Non-fatal: shadow failure must never block the pipeline
+            if (err.code !== 'MODULE_NOT_FOUND') {
+              console.error(`[ML-SHADOW] Error for ${name}@${version}: ${err.message}`);
+            }
+          }
+        }
+
         stats.suspect++;
 
         // Fire-and-forget tarball archiving — never blocks the pipeline