muaddib-scanner 2.10.68 → 2.10.69

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "muaddib-scanner",
-  "version": "2.10.68",
+  "version": "2.10.69",
   "description": "Supply-chain threat detection & response for npm & PyPI/Python",
   "main": "src/index.js",
   "bin": {

package/src/ml/train-xgboost.py

@@ -359,23 +359,36 @@ def filter_leaky_features(X: pd.DataFrame, y: np.ndarray,
     return X_filtered, retained
 
 
-def source_discrimination_gate(X: pd.DataFrame, y: np.ndarray,
-                                active_features: list,
-                                max_accuracy: float = 0.65) -> bool:
+def source_discrimination_diagnostic(X: pd.DataFrame, y: np.ndarray,
+                                      active_features: list):
     """
-    Step 2c: Hard gate — verify that retained behavioral features cannot
-    trivially distinguish data source (monitor vs Datadog).
+    Step 2c: Source discrimination diagnostic (LOG-ONLY, non-blocking).
+
+    DESIGN NOTE: This test cannot function as a hard gate when source labels
+    are perfectly confounded with class labels (all negatives = monitor,
+    all positives = Datadog). In that case, legitimate behavioral features
+    (score, count_critical, type_*) will dominate the discriminator because
+    malware genuinely behaves differently from clean packages — this is
+    signal, not leak.
 
-    Since all negatives come from monitor and all positives from Datadog,
-    y IS the source label. A shallow classifier that achieves accuracy > 65%
-    on the retained features indicates residual source-identity leaks.
+    A true source discrimination test would require either:
+    (a) positives re-scanned through our own pipeline, or
+    (b) negatives and positives from the SAME source.
 
-    Returns: True if gate passes (accuracy <= max_accuracy), False if fails.
-    Prints SHAP top 10 of the discriminator to identify offending features.
+    This diagnostic still serves a purpose: it flags NON-BEHAVIORAL features
+    that shouldn't appear in the top discriminators. If metadata features
+    (unpacked_size_bytes, file_count_total, etc.) appear despite being
+    excluded in Step 2a, something is wrong.
+
+    The real validation happens in shadow deployment on live production data.
     """
     print("\n" + "=" * 60)
-    print(f"[Step 2c/8] Source discrimination gate (threshold={max_accuracy:.0%})...")
+    print("[Step 2c/8] Source discrimination diagnostic (log-only)...")
     print("=" * 60)
+    print("  NOTE: source=Datadog correlates 100% with label=malicious.")
+    print("  This diagnostic checks for non-behavioral features in the")
+    print("  top discriminators, NOT for overall accuracy (which will")
+    print("  always be high due to the source/label confound).")
 
     X_active = X[active_features]
 
@@ -385,7 +398,6 @@ def source_discrimination_gate(X: pd.DataFrame, y: np.ndarray,
     )
 
     # Shallow model — depth=3, 50 rounds, no class weighting
-    # (we want to detect ANY discriminability, not optimize for one class)
     params = {
         'objective': 'binary:logistic',
         'eval_metric': 'logloss',
@@ -407,35 +419,52 @@ def source_discrimination_gate(X: pd.DataFrame, y: np.ndarray,
     p = precision_score(y_te, preds, zero_division=0)
     r = recall_score(y_te, preds, zero_division=0)
 
-    print(f"  Discrimination accuracy: {accuracy:.3f} (P={p:.3f} R={r:.3f})")
+    print(f"\n  Discrimination accuracy: {accuracy:.3f} (P={p:.3f} R={r:.3f})")
+    print(f"  (Expected to be high due to source/label confound)")
 
-    # SHAP analysis to identify which features drive discrimination
+    # SHAP analysis — the diagnostic value is in WHICH features dominate
     explainer = shap.TreeExplainer(model)
     shap_values = explainer.shap_values(X_te)
     mean_abs_shap = np.abs(shap_values).mean(axis=0)
     importance = sorted(zip(active_features, mean_abs_shap),
                         key=lambda x: x[1], reverse=True)
 
-    print(f"\n  Top 10 features driving source discrimination:")
+    # Known behavioral features that SHOULD dominate (malware scores higher)
+    EXPECTED_BEHAVIORAL = {
+        'score', 'global_risk_score', 'max_file_score', 'package_score',
+        'count_total', 'count_critical', 'count_high', 'count_medium',
+        'count_low', 'distinct_threat_types', 'severity_ratio_high',
+        'max_single_points', 'points_concentration', 'file_count_with_threats',
+        'file_score_mean', 'file_score_max', 'threat_density',
+    }
+    # Features that should NOT appear (already excluded, but sanity check)
+    EXCLUDED_CHECK = {
+        'unpacked_size_bytes', 'file_count_total', 'has_tests',
+        'dep_count', 'dev_dep_count', 'reputation_factor',
+        'package_age_days', 'weekly_downloads', 'version_count',
+        'author_package_count', 'has_repository', 'readme_size',
+    }
+
+    print(f"\n  Top 10 features driving discrimination:")
+    has_leak = False
     for i, (name, val) in enumerate(importance[:10]):
-        flag = ""
-        # Flag non-behavioral features that shouldn't be discriminative
-        if name in ('unpacked_size_bytes', 'file_count_total', 'has_tests',
-                     'dep_count', 'dev_dep_count', 'reputation_factor'):
-            flag = " *** NON-BEHAVIORAL"
+        if name in EXCLUDED_CHECK:
+            flag = " *** LEAK — should have been excluded in Step 2a!"
+            has_leak = True
+        elif name in EXPECTED_BEHAVIORAL:
+            flag = " (expected — behavioral)"
+        elif name.startswith('type_') or name.startswith('has_'):
+            flag = " (behavioral signal)"
+        else:
+            flag = ""
         print(f"    {i + 1:2d}. {name:40s} {val:.6f}{flag}")
 
-    if accuracy <= max_accuracy:
-        print(f"\n  [GATE PASS] Accuracy {accuracy:.3f} <= {max_accuracy:.3f}")
-        print(f"  Behavioral features do not trivially encode source identity.")
-        return True
+    if has_leak:
+        print(f"\n  [WARNING] Non-behavioral features found in top discriminators!")
+        print(f"  Check EXCLUDED_METADATA — some metadata features leaked through.")
     else:
-        print(f"\n  [GATE FAIL] Accuracy {accuracy:.3f} > {max_accuracy:.3f}")
-        print(f"  Retained features still encode source identity.")
-        print(f"  Offending features (exclude and re-run):")
-        for name, val in importance[:5]:
-            print(f"    - {name} (SHAP={val:.6f})")
-        return False
+        print(f"\n  [OK] Top discriminators are all behavioral features.")
+        print(f"  No metadata/source-proxy leak detected.")
 
 
 def split_data(X: pd.DataFrame, y: np.ndarray) -> tuple:
@@ -836,18 +865,15 @@ def main():
     else:
         active_features = list(remaining_features)
 
-    # Step 2c: Source discrimination gate — HARD STOP if features encode source
+    # Step 2c: Source discrimination diagnostic (log-only).
+    # NOT a hard gate — source label is 100% confounded with class label
+    # (all positives = Datadog, all negatives = monitor), so behavioral
+    # features will always dominate the discriminator. The diagnostic
+    # checks that no METADATA features leaked through Step 2a.
     if not args.skip_gate:
-        gate_pass = source_discrimination_gate(X, y, active_features)
-        if not gate_pass:
-            print("\n" + "=" * 60)
-            print("ABORTED: Source discrimination gate failed.")
-            print("The retained features still encode source identity.")
-            print("Add offending features to EXCLUDED_METADATA and re-run.")
-            print("=" * 60)
-            sys.exit(1)
+        source_discrimination_diagnostic(X, y, active_features)
     else:
-        print("\n  [Step 2c] Source discrimination gate SKIPPED (--skip-gate)")
+        print("\n  [Step 2c] Source discrimination diagnostic SKIPPED (--skip-gate)")
 
     # Class imbalance weight
     n_neg = stats['n_neg']