muaddib-scanner 2.10.49 → 2.10.51

package/README.md

@@ -282,6 +282,8 @@ repos:
 
 | Metric | Result | Details |
 |--------|--------|---------|
+| **ML FPR** | **2.85%** (239/8,393 holdout) | XGBoost retrained on 56,564 samples, 64 features, threshold=0.710 |
+| **ML TPR** | **99.93%** (2,918/2,920 holdout) | 377 confirmed_malicious via OSSF/GHSA/npm correlation |
 | **Wild TPR** (Datadog 17K) | **92.8%** (13,538/14,587 in-scope) | 17,922 packages. 3,335 skipped (no JS). By category: compromised_lib 97.8%, malicious_intent 92.1% |
 | **TPR** (Ground Truth) | **93.9%** (46/49) | 51 real attacks. 3 out-of-scope: browser-only |
 | **FPR** (Benign curated) | **10.6%** (56/529) | 529 npm packages, real source via `npm pack` |
@@ -290,7 +292,13 @@ repos:
 
 **3034 tests** across 65 files. **200 rules** (195 RULES + 5 PARANOID).
 
-> **Methodology caveats:**
+> **ML retrain methodology (v2.10.51):**
+> - Ground truth: 377 confirmed_malicious via auto-labeler (OSSF malicious-packages, GitHub Advisory Database, npm registry takedown correlation)
+> - Dataset: 56,564 samples (14,602 malicious, 41,962 clean). Stratified 80/20 split
+> - Grid search: depth=4, estimators=300, lr=0.05. AUC-ROC=0.999, F1=0.960
+> - Leaky feature filter: 23 dead/leaky features removed (source-identity proxies)
+>
+> **Static evaluation caveats:**
 > - TPR measured on 49 Node.js attack samples (3 browser-only excluded from 51 total)
 > - FPR measured on 529 curated popular npm packages (not a random sample)
 > - ADR measured with global threshold (score >= 20) as of v2.6.5

package/ml-retrain/auto-labeler/auto_labeler.py

@@ -0,0 +1,312 @@
+#!/usr/bin/env python3
+"""
+MUAD'DIB Auto-Labeling Pipeline
+
+Correlates muaddib suspects with external signals (OSSF, GHSA, npm status)
+to produce ground truth labels for ML training.
+
+Usage:
+    python auto_labeler.py --full              # Run all steps
+    python auto_labeler.py --step ossf         # Run individual step
+    python auto_labeler.py --step npm
+    python auto_labeler.py --step ghsa
+    python auto_labeler.py --step label
+    python auto_labeler.py --update            # Cron mode: re-check pending/unconfirmed
+
+Environment:
+    GITHUB_TOKEN    Optional, for higher GHSA API rate limits
+    MUADDIB_DATA    Override data directory (default: /opt/muaddib/data)
+"""
+
+import argparse
+import json
+import logging
+import os
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+
+import ossf_index
+import ghsa_checker
+import npm_checker
+import labeler
+
+# ── Paths ──
+MUADDIB_DATA = Path(os.environ.get("MUADDIB_DATA", "/opt/muaddib/data"))
+MUADDIB_ALERTS = Path(os.environ.get("MUADDIB_ALERTS", "/opt/muaddib/logs/alerts"))
+BASE_DIR = Path(__file__).parent
+CACHE_DIR = BASE_DIR / "data"
+OSSF_REPO_DIR = CACHE_DIR / "ossf-malicious-packages"
+OUTPUT_PATH = MUADDIB_DATA / "auto-labels.json"
+
+log = logging.getLogger("auto-labeler")
+
+
+def setup_logging(verbose=False):
+    level = logging.DEBUG if verbose else logging.INFO
+    fmt = "%(asctime)s [%(name)s] %(levelname)s %(message)s"
+    logging.basicConfig(level=level, format=fmt, datefmt="%Y-%m-%d %H:%M:%S")
+
+
+def load_detections():
+    """Load detections.json from muaddib data directory."""
+    path = MUADDIB_DATA / "detections.json"
+    if not path.is_file():
+        log.error("detections.json not found at %s", path)
+        sys.exit(1)
+
+    with open(path, "r", encoding="utf-8") as f:
+        data = json.load(f)
+
+    detections = data.get("detections", [])
+    npm_count = sum(1 for d in detections if d.get("ecosystem") == "npm")
+    log.info("Loaded %d detections (%d npm)", len(detections), npm_count)
+    return detections
+
+
+def load_alert_scores():
+    """Load risk scores and tiers from individual alert files.
+
+    Scans logs/alerts/ for JSON files and extracts score + tier info.
+    Returns dict keyed by "name@version".
+    """
+    scores = {}
+
+    # Try cached scores first
+    cache_path = CACHE_DIR / "alert-scores-cache.json"
+    if cache_path.is_file():
+        try:
+            with open(cache_path, "r", encoding="utf-8") as f:
+                cached = json.load(f)
+            if cached.get("count", 0) > 0:
+                log.info("Loaded %d cached alert scores", cached["count"])
+                return cached.get("scores", {})
+        except (json.JSONDecodeError, OSError):
+            pass
+
+    if not MUADDIB_ALERTS.is_dir():
+        log.warning("Alerts directory not found at %s — scores will be estimated from severity",
+                     MUADDIB_ALERTS)
+        return scores
+
+    alert_files = list(MUADDIB_ALERTS.glob("*.json"))
+    log.info("Scanning %d alert files for scores...", len(alert_files))
+
+    for filepath in alert_files:
+        try:
+            with open(filepath, "r", encoding="utf-8") as f:
+                alert = json.load(f)
+
+            target = alert.get("target", "")
+            summary = alert.get("summary", {})
+            score = summary.get("riskScore", summary.get("globalRiskScore", 0))
+
+            # Parse target: "npm/package-name@version" or "pypi/package@version"
+            if "/" in target and "@" in target:
+                eco_pkg = target.split("/", 1)
+                if len(eco_pkg) == 2:
+                    pkg_ver = eco_pkg[1]
+                    # Determine tier from priority
+                    priority = alert.get("priority", {})
+                    tier = ""
+                    p_level = priority.get("level", "")
+                    if p_level == "P1":
+                        tier = "T1a"
+                    elif p_level == "P2":
+                        tier = "T1b"
+                    elif p_level == "P3":
+                        tier = "T2"
+
+                    scores[pkg_ver] = {"score": score, "tier": tier}
+
+        except (json.JSONDecodeError, OSError):
+            continue
+
+    # Cache for next run
+    CACHE_DIR.mkdir(parents=True, exist_ok=True)
+    with open(cache_path, "w", encoding="utf-8") as f:
+        json.dump({"count": len(scores), "built_at": datetime.now(timezone.utc).isoformat(),
+                    "scores": scores}, f)
+
+    log.info("Extracted scores from %d alerts", len(scores))
+    return scores
+
+
+# ── Steps ──
+
+def step_ossf():
+    """Step 1: Index OSSF malicious-packages."""
+    log.info("=== Step 1: OSSF Index ===")
+    ossf_index.clone_or_update(OSSF_REPO_DIR)
+    index = ossf_index.build_index(OSSF_REPO_DIR, CACHE_DIR)
+    return index
+
+
+def step_ghsa():
+    """Step 3: Index GitHub Advisory Database."""
+    log.info("=== Step 3: GHSA Index ===")
+    # Try cache first
+    index = ghsa_checker.load_cached_index(CACHE_DIR)
+    if index is not None:
+        return index
+    return ghsa_checker.build_index(CACHE_DIR)
+
+
+def step_npm(detections):
+    """Step 2: Check npm status for suspects."""
+    log.info("=== Step 2: npm Status Check ===")
+    return npm_checker.check_suspects(detections, CACHE_DIR)
+
+
+def step_label(detections, o_index, g_index, npm_status, alert_scores):
+    """Step 4: Generate labels."""
+    log.info("=== Step 4: Generate Labels ===")
+
+    labels = labeler.label_suspects(detections, o_index, g_index, npm_status, alert_scores)
+    missed = labeler.find_missed(o_index, g_index, detections)
+    summary = labeler.export_labels(labels, missed, OUTPUT_PATH)
+
+    return summary
+
+
+# ── Modes ──
+
+def run_full():
+    """Run all steps sequentially."""
+    log.info("Starting full auto-labeling pipeline")
+    start = datetime.now()
+
+    detections = load_detections()
+    alert_scores = load_alert_scores()
+
+    # Steps 1+3 don't depend on detections — could be parallel but keep it simple
+    o_index = step_ossf()
+    g_index = step_ghsa()
+    npm_status = step_npm(detections)
+
+    summary = step_label(detections, o_index, g_index, npm_status, alert_scores)
+
+    elapsed = (datetime.now() - start).total_seconds()
+    log.info("Pipeline complete in %.1fs — %s", elapsed, summary)
+    return summary
+
+
+def run_update():
+    """Cron mode: re-check pending/unconfirmed labels against fresh external data."""
+    log.info("Starting update (cron mode)")
+
+    # Refresh external indices
+    ossf_index.clone_or_update(OSSF_REPO_DIR)
+    o_index = ossf_index.build_index(OSSF_REPO_DIR, CACHE_DIR)
+    g_index = ghsa_checker.build_index(CACHE_DIR)
+
+    # Load existing labels
+    if not OUTPUT_PATH.is_file():
+        log.error("No existing auto-labels.json — run --full first")
+        sys.exit(1)
+
+    with open(OUTPUT_PATH, "r", encoding="utf-8") as f:
+        existing = json.load(f)
+
+    existing_labels = existing.get("labels", {})
+    detections = load_detections()
+    alert_scores = load_alert_scores()
+
+    # Find labels that need re-evaluation
+    to_recheck = []
+    for key, entry in existing_labels.items():
+        lbl = entry.get("auto_label")
+        if lbl in ("pending", "unconfirmed", "likely_malicious"):
+            # Re-extract detection info
+            for det in detections:
+                if f"{det['package']}@{det['version']}" == key:
+                    to_recheck.append(det)
+                    break
+
+    if not to_recheck:
+        log.info("No pending/unconfirmed labels to re-check")
+        return
+
+    log.info("Re-checking %d labels (pending/unconfirmed/likely_malicious)", len(to_recheck))
+
+    # Re-check npm status for these specific packages
+    npm_status = npm_checker.check_suspects(to_recheck, CACHE_DIR)
+
+    # Re-label
+    updated = labeler.label_suspects(to_recheck, o_index, g_index, npm_status, alert_scores)
+
+    # Merge updates into existing labels
+    changes = 0
+    for key, new_entry in updated.items():
+        old = existing_labels.get(key, {})
+        if old.get("auto_label") != new_entry.get("auto_label"):
+            log.info("RELABEL: %s — %s → %s",
+                     key, old.get("auto_label"), new_entry.get("auto_label"))
+            changes += 1
+        existing_labels[key] = new_entry
+
+    # Also refresh missed detection
+    missed = labeler.find_missed(o_index, g_index, detections)
+    for name, info in missed.items():
+        mk = f"{name}@*"
+        if mk not in existing_labels:
+            existing_labels[mk] = info
+            changes += 1
+
+    # Re-export
+    labeler.export_labels(
+        {k: v for k, v in existing_labels.items() if v.get("auto_label") != "missed"},
+        {k.replace("@*", ""): v for k, v in existing_labels.items() if v.get("auto_label") == "missed"},
+        OUTPUT_PATH,
+    )
+
+    log.info("Update complete: %d labels changed", changes)
+
+
+def main():
+    parser = argparse.ArgumentParser(description="MUAD'DIB Auto-Labeling Pipeline")
+    group = parser.add_mutually_exclusive_group(required=True)
+    group.add_argument("--full", action="store_true", help="Run all steps")
+    group.add_argument("--step", choices=["ossf", "ghsa", "npm", "label"],
+                       help="Run individual step")
+    group.add_argument("--update", action="store_true",
+                       help="Cron: re-check pending/unconfirmed")
+    parser.add_argument("-v", "--verbose", action="store_true", help="Debug logging")
+    parser.add_argument("--data-dir", help="Override MUADDIB_DATA path")
+    parser.add_argument("--alerts-dir", help="Override MUADDIB_ALERTS path")
+    args = parser.parse_args()
+
+    setup_logging(args.verbose)
+
+    if args.data_dir:
+        global MUADDIB_DATA, OUTPUT_PATH
+        MUADDIB_DATA = Path(args.data_dir)
+        OUTPUT_PATH = MUADDIB_DATA / "auto-labels.json"
+    if args.alerts_dir:
+        global MUADDIB_ALERTS
+        MUADDIB_ALERTS = Path(args.alerts_dir)
+
+    if args.full:
+        run_full()
+    elif args.update:
+        run_update()
+    elif args.step == "ossf":
+        step_ossf()
+    elif args.step == "ghsa":
+        step_ghsa()
+    elif args.step == "npm":
+        step_npm(load_detections())
+    elif args.step == "label":
+        detections = load_detections()
+        alert_scores = load_alert_scores()
+        o_index = ossf_index.load_cached_index(CACHE_DIR)
+        g_index = ghsa_checker.load_cached_index(CACHE_DIR)
+        npm_status = npm_checker._load_cache(CACHE_DIR / npm_checker.CACHE_FILENAME)
+        if o_index is None or g_index is None:
+            log.error("Run --step ossf and --step ghsa first (or use --full)")
+            sys.exit(1)
+        step_label(detections, o_index, g_index, npm_status, alert_scores)
+
+
+if __name__ == "__main__":
+    main()

package/ml-retrain/auto-labeler/ghsa_checker.py

@@ -0,0 +1,169 @@
+"""
+GitHub Advisory Database checker.
+
+Fetches all npm malware advisories from the GitHub Advisory Database API.
+Supports optional GITHUB_TOKEN env var for higher rate limits.
+"""
+
+import json
+import logging
+import os
+import time
+from datetime import datetime
+from pathlib import Path
+
+import requests
+
+log = logging.getLogger("auto-labeler.ghsa")
+
+GHSA_API = "https://api.github.com/advisories"
+INDEX_FILENAME = "ghsa-index.json"
+# Cache validity: 12 hours
+CACHE_TTL_SECONDS = 12 * 3600
+
+
+def _get_headers():
+    token = os.environ.get("GITHUB_TOKEN")
+    headers = {"Accept": "application/vnd.github+json"}
+    if token:
+        headers["Authorization"] = f"Bearer {token}"
+        log.info("Using GITHUB_TOKEN for GHSA API (5000 req/h)")
+    else:
+        log.info("No GITHUB_TOKEN — GHSA API limited to 60 req/h")
+    return headers
+
+
+def fetch_malware_advisories():
+    """Fetch all npm malware advisories from GHSA. Returns list of advisories."""
+    headers = _get_headers()
+    advisories = []
+    page = 1
+    per_page = 100
+
+    while True:
+        params = {
+            "type": "malware",
+            "ecosystem": "npm",
+            "per_page": per_page,
+            "page": page,
+        }
+
+        for attempt in range(3):
+            try:
+                resp = requests.get(GHSA_API, headers=headers, params=params, timeout=30)
+
+                if resp.status_code == 403:
+                    # Rate limited
+                    retry_after = int(resp.headers.get("Retry-After", 60))
+                    log.warning("GHSA rate limited, waiting %ds", retry_after)
+                    time.sleep(retry_after)
+                    continue
+
+                resp.raise_for_status()
+                break
+            except requests.RequestException as e:
+                wait = 2 ** attempt * 5
+                log.warning("GHSA request failed (attempt %d): %s — retrying in %ds",
+                            attempt + 1, e, wait)
+                time.sleep(wait)
+        else:
+            log.error("GHSA fetch failed after 3 attempts on page %d", page)
+            break
+
+        batch = resp.json()
+        if not batch:
+            break
+
+        advisories.extend(batch)
+        log.info("GHSA page %d: %d advisories (total: %d)", page, len(batch), len(advisories))
+
+        if len(batch) < per_page:
+            break
+        page += 1
+        time.sleep(1)  # Courtesy delay
+
+    return advisories
+
+
+def build_index(cache_dir):
+    """Build GHSA index from API. Returns dict keyed by package name."""
+    advisories = fetch_malware_advisories()
+    index = {}
+
+    for adv in advisories:
+        ghsa_id = adv.get("ghsa_id", "")
+        published = adv.get("published_at", "")
+        summary = adv.get("summary", "")
+        withdrawn = adv.get("withdrawn_at")
+
+        # Skip withdrawn advisories
+        if withdrawn:
+            continue
+
+        for vuln in adv.get("vulnerabilities", []):
+            pkg = vuln.get("package", {})
+            ecosystem = pkg.get("ecosystem", "").lower()
+            name = pkg.get("name", "")
+
+            if ecosystem != "npm" or not name:
+                continue
+
+            version_range = vuln.get("vulnerable_version_range", "")
+
+            entry = {
+                "source": "ghsa",
+                "ghsa_id": ghsa_id,
+                "date": published,
+                "summary": summary[:200],
+                "version_range": version_range,
+            }
+
+            # Index by package name (version matching is approximate for GHSA)
+            if name not in index:
+                index[name] = []
+            index[name].append(entry)
+
+    log.info("GHSA index: %d packages from %d advisories", len(index), len(advisories))
+
+    # Cache to disk
+    cache_dir = Path(cache_dir)
+    cache_dir.mkdir(parents=True, exist_ok=True)
+    cache_path = cache_dir / INDEX_FILENAME
+    with open(cache_path, "w", encoding="utf-8") as f:
+        json.dump({"built_at": datetime.utcnow().isoformat() + "Z",
+                    "count": len(index),
+                    "index": index}, f)
+    log.info("GHSA index cached to %s", cache_path)
+
+    return index
+
+
+def load_cached_index(cache_dir):
+    """Load index from cache if fresh enough."""
+    cache_path = Path(cache_dir) / INDEX_FILENAME
+    if not cache_path.is_file():
+        return None
+    try:
+        stat = cache_path.stat()
+        age = time.time() - stat.st_mtime
+        if age > CACHE_TTL_SECONDS:
+            log.info("GHSA cache expired (%.1fh old)", age / 3600)
+            return None
+
+        with open(cache_path, "r", encoding="utf-8") as f:
+            data = json.load(f)
+        log.info("Loaded cached GHSA index (%d packages, built %s)",
+                 data.get("count", 0), data.get("built_at", "?"))
+        return data.get("index", {})
+    except (json.JSONDecodeError, OSError) as e:
+        log.warning("Failed to load GHSA cache: %s", e)
+        return None
+
+
+def lookup(index, name):
+    """Check if a package name is in the GHSA index.
+
+    Returns the list of advisory entries or None.
+    """
+    entries = index.get(name)
+    return entries if entries else None