muaddib-scanner 2.10.49 → 2.10.50

package/ml-retrain/auto-labeler/auto_labeler.py

@@ -0,0 +1,312 @@
+#!/usr/bin/env python3
+"""
+MUAD'DIB Auto-Labeling Pipeline
+
+Correlates muaddib suspects with external signals (OSSF, GHSA, npm status)
+to produce ground truth labels for ML training.
+
+Usage:
+    python auto_labeler.py --full              # Run all steps
+    python auto_labeler.py --step ossf         # Run individual step
+    python auto_labeler.py --step npm
+    python auto_labeler.py --step ghsa
+    python auto_labeler.py --step label
+    python auto_labeler.py --update            # Cron mode: re-check pending/unconfirmed
+
+Environment:
+    GITHUB_TOKEN    Optional, for higher GHSA API rate limits
+    MUADDIB_DATA    Override data directory (default: /opt/muaddib/data)
+"""
+
+import argparse
+import json
+import logging
+import os
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+
+import ossf_index
+import ghsa_checker
+import npm_checker
+import labeler
+
+# ── Paths ──
+MUADDIB_DATA = Path(os.environ.get("MUADDIB_DATA", "/opt/muaddib/data"))
+MUADDIB_ALERTS = Path(os.environ.get("MUADDIB_ALERTS", "/opt/muaddib/logs/alerts"))
+BASE_DIR = Path(__file__).parent
+CACHE_DIR = BASE_DIR / "data"
+OSSF_REPO_DIR = CACHE_DIR / "ossf-malicious-packages"
+OUTPUT_PATH = MUADDIB_DATA / "auto-labels.json"
+
+log = logging.getLogger("auto-labeler")
+
+
+def setup_logging(verbose=False):
+    level = logging.DEBUG if verbose else logging.INFO
+    fmt = "%(asctime)s [%(name)s] %(levelname)s %(message)s"
+    logging.basicConfig(level=level, format=fmt, datefmt="%Y-%m-%d %H:%M:%S")
+
+
+def load_detections():
+    """Load detections.json from muaddib data directory."""
+    path = MUADDIB_DATA / "detections.json"
+    if not path.is_file():
+        log.error("detections.json not found at %s", path)
+        sys.exit(1)
+
+    with open(path, "r", encoding="utf-8") as f:
+        data = json.load(f)
+
+    detections = data.get("detections", [])
+    npm_count = sum(1 for d in detections if d.get("ecosystem") == "npm")
+    log.info("Loaded %d detections (%d npm)", len(detections), npm_count)
+    return detections
+
+
+def load_alert_scores():
+    """Load risk scores and tiers from individual alert files.
+
+    Scans logs/alerts/ for JSON files and extracts score + tier info.
+    Returns dict keyed by "name@version".
+    """
+    scores = {}
+
+    # Try cached scores first
+    cache_path = CACHE_DIR / "alert-scores-cache.json"
+    if cache_path.is_file():
+        try:
+            with open(cache_path, "r", encoding="utf-8") as f:
+                cached = json.load(f)
+            if cached.get("count", 0) > 0:
+                log.info("Loaded %d cached alert scores", cached["count"])
+                return cached.get("scores", {})
+        except (json.JSONDecodeError, OSError):
+            pass
+
+    if not MUADDIB_ALERTS.is_dir():
+        log.warning("Alerts directory not found at %s — scores will be estimated from severity",
+                     MUADDIB_ALERTS)
+        return scores
+
+    alert_files = list(MUADDIB_ALERTS.glob("*.json"))
+    log.info("Scanning %d alert files for scores...", len(alert_files))
+
+    for filepath in alert_files:
+        try:
+            with open(filepath, "r", encoding="utf-8") as f:
+                alert = json.load(f)
+
+            target = alert.get("target", "")
+            summary = alert.get("summary", {})
+            score = summary.get("riskScore", summary.get("globalRiskScore", 0))
+
+            # Parse target: "npm/package-name@version" or "pypi/package@version"
+            if "/" in target and "@" in target:
+                eco_pkg = target.split("/", 1)
+                if len(eco_pkg) == 2:
+                    pkg_ver = eco_pkg[1]
+                    # Determine tier from priority
+                    priority = alert.get("priority", {})
+                    tier = ""
+                    p_level = priority.get("level", "")
+                    if p_level == "P1":
+                        tier = "T1a"
+                    elif p_level == "P2":
+                        tier = "T1b"
+                    elif p_level == "P3":
+                        tier = "T2"
+
+                    scores[pkg_ver] = {"score": score, "tier": tier}
+
+        except (json.JSONDecodeError, OSError):
+            continue
+
+    # Cache for next run
+    CACHE_DIR.mkdir(parents=True, exist_ok=True)
+    with open(cache_path, "w", encoding="utf-8") as f:
+        json.dump({"count": len(scores), "built_at": datetime.now(timezone.utc).isoformat(),
+                    "scores": scores}, f)
+
+    log.info("Extracted scores from %d alerts", len(scores))
+    return scores
+
+
+# ── Steps ──
+
+def step_ossf():
+    """Step 1: Index OSSF malicious-packages."""
+    log.info("=== Step 1: OSSF Index ===")
+    ossf_index.clone_or_update(OSSF_REPO_DIR)
+    index = ossf_index.build_index(OSSF_REPO_DIR, CACHE_DIR)
+    return index
+
+
+def step_ghsa():
+    """Step 3: Index GitHub Advisory Database."""
+    log.info("=== Step 3: GHSA Index ===")
+    # Try cache first
+    index = ghsa_checker.load_cached_index(CACHE_DIR)
+    if index is not None:
+        return index
+    return ghsa_checker.build_index(CACHE_DIR)
+
+
+def step_npm(detections):
+    """Step 2: Check npm status for suspects."""
+    log.info("=== Step 2: npm Status Check ===")
+    return npm_checker.check_suspects(detections, CACHE_DIR)
+
+
+def step_label(detections, o_index, g_index, npm_status, alert_scores):
+    """Step 4: Generate labels."""
+    log.info("=== Step 4: Generate Labels ===")
+
+    labels = labeler.label_suspects(detections, o_index, g_index, npm_status, alert_scores)
+    missed = labeler.find_missed(o_index, g_index, detections)
+    summary = labeler.export_labels(labels, missed, OUTPUT_PATH)
+
+    return summary
+
+
+# ── Modes ──
+
+def run_full():
+    """Run all steps sequentially."""
+    log.info("Starting full auto-labeling pipeline")
+    start = datetime.now()
+
+    detections = load_detections()
+    alert_scores = load_alert_scores()
+
+    # Steps 1+3 don't depend on detections — could be parallel but keep it simple
+    o_index = step_ossf()
+    g_index = step_ghsa()
+    npm_status = step_npm(detections)
+
+    summary = step_label(detections, o_index, g_index, npm_status, alert_scores)
+
+    elapsed = (datetime.now() - start).total_seconds()
+    log.info("Pipeline complete in %.1fs — %s", elapsed, summary)
+    return summary
+
+
+def run_update():
+    """Cron mode: re-check pending/unconfirmed labels against fresh external data."""
+    log.info("Starting update (cron mode)")
+
+    # Refresh external indices
+    ossf_index.clone_or_update(OSSF_REPO_DIR)
+    o_index = ossf_index.build_index(OSSF_REPO_DIR, CACHE_DIR)
+    g_index = ghsa_checker.build_index(CACHE_DIR)
+
+    # Load existing labels
+    if not OUTPUT_PATH.is_file():
+        log.error("No existing auto-labels.json — run --full first")
+        sys.exit(1)
+
+    with open(OUTPUT_PATH, "r", encoding="utf-8") as f:
+        existing = json.load(f)
+
+    existing_labels = existing.get("labels", {})
+    detections = load_detections()
+    alert_scores = load_alert_scores()
+
+    # Find labels that need re-evaluation
+    to_recheck = []
+    for key, entry in existing_labels.items():
+        lbl = entry.get("auto_label")
+        if lbl in ("pending", "unconfirmed", "likely_malicious"):
+            # Re-extract detection info
+            for det in detections:
+                if f"{det['package']}@{det['version']}" == key:
+                    to_recheck.append(det)
+                    break
+
+    if not to_recheck:
+        log.info("No pending/unconfirmed labels to re-check")
+        return
+
+    log.info("Re-checking %d labels (pending/unconfirmed/likely_malicious)", len(to_recheck))
+
+    # Re-check npm status for these specific packages
+    npm_status = npm_checker.check_suspects(to_recheck, CACHE_DIR)
+
+    # Re-label
+    updated = labeler.label_suspects(to_recheck, o_index, g_index, npm_status, alert_scores)
+
+    # Merge updates into existing labels
+    changes = 0
+    for key, new_entry in updated.items():
+        old = existing_labels.get(key, {})
+        if old.get("auto_label") != new_entry.get("auto_label"):
+            log.info("RELABEL: %s — %s → %s",
+                     key, old.get("auto_label"), new_entry.get("auto_label"))
+            changes += 1
+        existing_labels[key] = new_entry
+
+    # Also refresh missed detection
+    missed = labeler.find_missed(o_index, g_index, detections)
+    for name, info in missed.items():
+        mk = f"{name}@*"
+        if mk not in existing_labels:
+            existing_labels[mk] = info
+            changes += 1
+
+    # Re-export
+    labeler.export_labels(
+        {k: v for k, v in existing_labels.items() if v.get("auto_label") != "missed"},
+        {k.replace("@*", ""): v for k, v in existing_labels.items() if v.get("auto_label") == "missed"},
+        OUTPUT_PATH,
+    )
+
+    log.info("Update complete: %d labels changed", changes)
+
+
+def main():
+    parser = argparse.ArgumentParser(description="MUAD'DIB Auto-Labeling Pipeline")
+    group = parser.add_mutually_exclusive_group(required=True)
+    group.add_argument("--full", action="store_true", help="Run all steps")
+    group.add_argument("--step", choices=["ossf", "ghsa", "npm", "label"],
+                       help="Run individual step")
+    group.add_argument("--update", action="store_true",
+                       help="Cron: re-check pending/unconfirmed")
+    parser.add_argument("-v", "--verbose", action="store_true", help="Debug logging")
+    parser.add_argument("--data-dir", help="Override MUADDIB_DATA path")
+    parser.add_argument("--alerts-dir", help="Override MUADDIB_ALERTS path")
+    args = parser.parse_args()
+
+    setup_logging(args.verbose)
+
+    if args.data_dir:
+        global MUADDIB_DATA, OUTPUT_PATH
+        MUADDIB_DATA = Path(args.data_dir)
+        OUTPUT_PATH = MUADDIB_DATA / "auto-labels.json"
+    if args.alerts_dir:
+        global MUADDIB_ALERTS
+        MUADDIB_ALERTS = Path(args.alerts_dir)
+
+    if args.full:
+        run_full()
+    elif args.update:
+        run_update()
+    elif args.step == "ossf":
+        step_ossf()
+    elif args.step == "ghsa":
+        step_ghsa()
+    elif args.step == "npm":
+        step_npm(load_detections())
+    elif args.step == "label":
+        detections = load_detections()
+        alert_scores = load_alert_scores()
+        o_index = ossf_index.load_cached_index(CACHE_DIR)
+        g_index = ghsa_checker.load_cached_index(CACHE_DIR)
+        npm_status = npm_checker._load_cache(CACHE_DIR / npm_checker.CACHE_FILENAME)
+        if o_index is None or g_index is None:
+            log.error("Run --step ossf and --step ghsa first (or use --full)")
+            sys.exit(1)
+        step_label(detections, o_index, g_index, npm_status, alert_scores)
+
+
+if __name__ == "__main__":
+    main()

package/ml-retrain/auto-labeler/ghsa_checker.py

@@ -0,0 +1,169 @@
+"""
+GitHub Advisory Database checker.
+
+Fetches all npm malware advisories from the GitHub Advisory Database API.
+Supports optional GITHUB_TOKEN env var for higher rate limits.
+"""
+
+import json
+import logging
+import os
+import time
+from datetime import datetime
+from pathlib import Path
+
+import requests
+
+log = logging.getLogger("auto-labeler.ghsa")
+
+GHSA_API = "https://api.github.com/advisories"
+INDEX_FILENAME = "ghsa-index.json"
+# Cache validity: 12 hours
+CACHE_TTL_SECONDS = 12 * 3600
+
+
+def _get_headers():
+    token = os.environ.get("GITHUB_TOKEN")
+    headers = {"Accept": "application/vnd.github+json"}
+    if token:
+        headers["Authorization"] = f"Bearer {token}"
+        log.info("Using GITHUB_TOKEN for GHSA API (5000 req/h)")
+    else:
+        log.info("No GITHUB_TOKEN — GHSA API limited to 60 req/h")
+    return headers
+
+
+def fetch_malware_advisories():
+    """Fetch all npm malware advisories from GHSA. Returns list of advisories."""
+    headers = _get_headers()
+    advisories = []
+    page = 1
+    per_page = 100
+
+    while True:
+        params = {
+            "type": "malware",
+            "ecosystem": "npm",
+            "per_page": per_page,
+            "page": page,
+        }
+
+        for attempt in range(3):
+            try:
+                resp = requests.get(GHSA_API, headers=headers, params=params, timeout=30)
+
+                if resp.status_code == 403:
+                    # Rate limited
+                    retry_after = int(resp.headers.get("Retry-After", 60))
+                    log.warning("GHSA rate limited, waiting %ds", retry_after)
+                    time.sleep(retry_after)
+                    continue
+
+                resp.raise_for_status()
+                break
+            except requests.RequestException as e:
+                wait = 2 ** attempt * 5
+                log.warning("GHSA request failed (attempt %d): %s — retrying in %ds",
+                            attempt + 1, e, wait)
+                time.sleep(wait)
+        else:
+            log.error("GHSA fetch failed after 3 attempts on page %d", page)
+            break
+
+        batch = resp.json()
+        if not batch:
+            break
+
+        advisories.extend(batch)
+        log.info("GHSA page %d: %d advisories (total: %d)", page, len(batch), len(advisories))
+
+        if len(batch) < per_page:
+            break
+        page += 1
+        time.sleep(1)  # Courtesy delay
+
+    return advisories
+
+
+def build_index(cache_dir):
+    """Build GHSA index from API. Returns dict keyed by package name."""
+    advisories = fetch_malware_advisories()
+    index = {}
+
+    for adv in advisories:
+        ghsa_id = adv.get("ghsa_id", "")
+        published = adv.get("published_at", "")
+        summary = adv.get("summary", "")
+        withdrawn = adv.get("withdrawn_at")
+
+        # Skip withdrawn advisories
+        if withdrawn:
+            continue
+
+        for vuln in adv.get("vulnerabilities", []):
+            pkg = vuln.get("package", {})
+            ecosystem = pkg.get("ecosystem", "").lower()
+            name = pkg.get("name", "")
+
+            if ecosystem != "npm" or not name:
+                continue
+
+            version_range = vuln.get("vulnerable_version_range", "")
+
+            entry = {
+                "source": "ghsa",
+                "ghsa_id": ghsa_id,
+                "date": published,
+                "summary": summary[:200],
+                "version_range": version_range,
+            }
+
+            # Index by package name (version matching is approximate for GHSA)
+            if name not in index:
+                index[name] = []
+            index[name].append(entry)
+
+    log.info("GHSA index: %d packages from %d advisories", len(index), len(advisories))
+
+    # Cache to disk
+    cache_dir = Path(cache_dir)
+    cache_dir.mkdir(parents=True, exist_ok=True)
+    cache_path = cache_dir / INDEX_FILENAME
+    with open(cache_path, "w", encoding="utf-8") as f:
+        json.dump({"built_at": datetime.utcnow().isoformat() + "Z",
+                    "count": len(index),
+                    "index": index}, f)
+    log.info("GHSA index cached to %s", cache_path)
+
+    return index
+
+
+def load_cached_index(cache_dir):
+    """Load index from cache if fresh enough."""
+    cache_path = Path(cache_dir) / INDEX_FILENAME
+    if not cache_path.is_file():
+        return None
+    try:
+        stat = cache_path.stat()
+        age = time.time() - stat.st_mtime
+        if age > CACHE_TTL_SECONDS:
+            log.info("GHSA cache expired (%.1fh old)", age / 3600)
+            return None
+
+        with open(cache_path, "r", encoding="utf-8") as f:
+            data = json.load(f)
+        log.info("Loaded cached GHSA index (%d packages, built %s)",
+                 data.get("count", 0), data.get("built_at", "?"))
+        return data.get("index", {})
+    except (json.JSONDecodeError, OSError) as e:
+        log.warning("Failed to load GHSA cache: %s", e)
+        return None
+
+
+def lookup(index, name):
+    """Check if a package name is in the GHSA index.
+
+    Returns the list of advisory entries or None.
+    """
+    entries = index.get(name)
+    return entries if entries else None

package/ml-retrain/auto-labeler/labeler.py

@@ -0,0 +1,249 @@
+"""
+Label generation engine.
+
+Correlates signals from OSSF, GHSA, and npm status to produce labels.
+
+Label tiers (by confidence):
+- confirmed_malicious: authoritative source (ossf/ghsa) OR npm takedown pattern
+- likely_malicious:     npm_removed + high muaddib score, but no authoritative confirmation
+- unconfirmed:          suspect in muaddib, still on npm, no external signal, >7 days old
+- pending:              suspect in muaddib, still on npm, no external signal, <7 days old
+- missed:               clean in muaddib BUT flagged by ossf/ghsa (false negative)
+"""
+
+import json
+import logging
+from datetime import datetime, timezone
+from pathlib import Path
+
+from ossf_index import lookup as ossf_lookup
+from ghsa_checker import lookup as ghsa_lookup
+from npm_checker import is_quick_takedown
+
+log = logging.getLogger("auto-labeler.labeler")
+
+# Thresholds
+SCORE_THRESHOLD_CONFIRMED = 50  # Minimum muaddib score for npm_removed → confirmed
+PENDING_DAYS = 7  # Days before pending → unconfirmed
+
+
+def _parse_iso(s):
+    """Parse ISO 8601 date string to datetime."""
+    if not s:
+        return None
+    try:
+        return datetime.fromisoformat(s.replace("Z", "+00:00"))
+    except (ValueError, TypeError):
+        return None
+
+
+def _days_since(iso_str):
+    """Days elapsed since the given ISO date string."""
+    dt = _parse_iso(iso_str)
+    if not dt:
+        return None
+    delta = datetime.now(timezone.utc) - dt
+    return delta.total_seconds() / 86400
+
+
+def _severity_to_score_estimate(severity):
+    """Rough score estimate from severity when exact score is unavailable."""
+    return {"CRITICAL": 70, "HIGH": 40, "MEDIUM": 15, "LOW": 5}.get(severity, 0)
+
+
+def label_suspects(detections, ossf_index, ghsa_index, npm_status, alert_scores):
+    """Generate labels for all suspect detections.
+
+    Args:
+        detections: list of detection dicts from detections.json
+        ossf_index: dict from ossf_index.build_index()
+        ghsa_index: dict from ghsa_checker.build_index()
+        npm_status: dict from npm_checker.check_suspects()
+        alert_scores: dict keyed by "name@version" with {"score": N, "tier": "T1a"} from alerts
+
+    Returns:
+        dict keyed by "name@version" with label info
+    """
+    labels = {}
+    stats = {"confirmed_malicious": 0, "likely_malicious": 0,
+             "unconfirmed": 0, "pending": 0}
+
+    for det in detections:
+        name = det["package"]
+        version = det["version"]
+        ecosystem = det.get("ecosystem", "npm")
+        key = f"{name}@{version}"
+        detection_date = det.get("first_seen_at", "")
+        severity = det.get("severity", "UNKNOWN")
+        findings = det.get("findings", [])
+
+        # Skip non-npm for now (OSSF/GHSA npm-focused)
+        if ecosystem != "npm":
+            continue
+
+        # Gather signals
+        signals = []
+
+        # Signal 1: OSSF
+        ossf_hit = ossf_lookup(ossf_index, name, version)
+        if ossf_hit:
+            signals.append("ossf")
+
+        # Signal 2: GHSA
+        ghsa_hit = ghsa_lookup(ghsa_index, name)
+        if ghsa_hit:
+            signals.append("ghsa")
+
+        # Signal 3: npm status
+        npm_result = npm_status.get(key, {})
+        npm_removed = npm_result.get("status") == "npm_removed"
+        if npm_removed:
+            signals.append("npm_removed")
+
+        # Get score from alerts or estimate from severity
+        score_info = alert_scores.get(key, {})
+        score = score_info.get("score", _severity_to_score_estimate(severity))
+        tier = score_info.get("tier", "")
+
+        # Determine label
+        label = _classify(signals, npm_result, detection_date, score)
+        stats[label] += 1
+
+        labels[key] = {
+            "muaddib_label": "suspect",
+            "auto_label": label,
+            "signals": signals,
+            "muaddib_score": score,
+            "muaddib_tier": tier,
+            "muaddib_severity": severity,
+            "muaddib_findings": findings,
+            "detection_date": detection_date,
+            "label_date": datetime.now(timezone.utc).isoformat(),
+            "npm_status": npm_result.get("status", "unknown"),
+            "npm_publish_date": npm_result.get("publish_date"),
+        }
+
+        if ossf_hit:
+            labels[key]["ossf_id"] = ossf_hit.get("osv_id")
+        if ghsa_hit:
+            labels[key]["ghsa_id"] = ghsa_hit[0].get("ghsa_id")
+
+        log.debug("LABEL %s → %s (signals=%s, score=%d)", key, label, signals, score)
+
+    log.info("Suspect labels: %s", stats)
+    return labels
+
+
+def _classify(signals, npm_result, detection_date, score):
+    """Core classification logic."""
+    has_authoritative = "ossf" in signals or "ghsa" in signals
+    npm_removed = "npm_removed" in signals
+
+    # Tier 1: Authoritative source confirms malicious
+    if has_authoritative:
+        return "confirmed_malicious"
+
+    # Tier 2: npm takedown pattern (removed + high score + quick removal)
+    if npm_removed and score >= SCORE_THRESHOLD_CONFIRMED:
+        if is_quick_takedown(npm_result, detection_date, threshold_hours=72):
+            return "confirmed_malicious"
+
+    # Tier 3: npm removed but doesn't meet confirmation criteria
+    if npm_removed:
+        return "likely_malicious"
+
+    # Tier 4: Still on npm, no external signal
+    days = _days_since(detection_date)
+    if days is not None and days > PENDING_DAYS:
+        return "unconfirmed"
+
+    return "pending"
+
+
+def find_missed(ossf_index, ghsa_index, detections):
+    """Find packages in OSSF/GHSA that muaddib did NOT detect (false negatives).
+
+    Returns dict keyed by package name with miss details.
+    """
+    # Build set of all detected package names
+    detected_names = set()
+    for det in detections:
+        if det.get("ecosystem") == "npm":
+            detected_names.add(det["package"])
+
+    missed = {}
+
+    # Check OSSF index
+    ossf_packages = set()
+    for key in ossf_index:
+        name = key.rsplit("@", 1)[0]
+        ossf_packages.add(name)
+
+    for name in ossf_packages:
+        if name not in detected_names:
+            missed[name] = {
+                "auto_label": "missed",
+                "muaddib_label": "clean",
+                "signals": ["ossf"],
+                "source_detail": "In ossf/malicious-packages but not in muaddib detections",
+                "label_date": datetime.now(timezone.utc).isoformat(),
+            }
+
+    # Check GHSA index
+    for name, entries in ghsa_index.items():
+        if name not in detected_names:
+            existing = missed.get(name)
+            if existing:
+                existing["signals"].append("ghsa")
+            else:
+                missed[name] = {
+                    "auto_label": "missed",
+                    "muaddib_label": "clean",
+                    "signals": ["ghsa"],
+                    "ghsa_id": entries[0].get("ghsa_id") if entries else None,
+                    "source_detail": "In GHSA malware advisories but not in muaddib detections",
+                    "label_date": datetime.now(timezone.utc).isoformat(),
+                }
+
+    log.info("Missed packages (false negatives): %d", len(missed))
+    if missed:
+        # Log the first 20 as these are critical for improving the scanner
+        for name in list(missed.keys())[:20]:
+            m = missed[name]
+            log.warning("MISSED: %s (signals=%s)", name, m["signals"])
+
+    return missed
+
+
+def export_labels(labels, missed, output_path):
+    """Export all labels to auto-labels.json."""
+    output_path = Path(output_path)
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+
+    # Merge suspects and missed into one output
+    all_labels = dict(labels)
+    for name, info in missed.items():
+        all_labels[f"{name}@*"] = info
+
+    # Generate summary
+    summary = {"confirmed_malicious": 0, "likely_malicious": 0,
+               "unconfirmed": 0, "pending": 0, "missed": 0}
+    for entry in all_labels.values():
+        lbl = entry.get("auto_label", "unknown")
+        if lbl in summary:
+            summary[lbl] += 1
+
+    output = {
+        "generated_at": datetime.now(timezone.utc).isoformat(),
+        "summary": summary,
+        "total": len(all_labels),
+        "labels": all_labels,
+    }
+
+    with open(output_path, "w", encoding="utf-8") as f:
+        json.dump(output, f, indent=2)
+
+    log.info("Exported %d labels to %s", len(all_labels), output_path)
+    log.info("Summary: %s", summary)
+
+    return summary

package/ml-retrain/auto-labeler/npm_checker.py

@@ -0,0 +1,228 @@
+"""
+npm registry status checker.
+
+For each suspect package, checks if the package/version still exists on npm.
+Extracts publish timing for temporal correlation (quick takedown = strong signal).
+Rate-limited to 50 requests/minute with exponential backoff.
+Resumable: saves progress to npm-status-cache.json.
+"""
+
+import json
+import logging
+import time
+from datetime import datetime
+from pathlib import Path
+
+import requests
+
+log = logging.getLogger("auto-labeler.npm")
+
+NPM_REGISTRY = "https://registry.npmjs.org"
+RATE_LIMIT = 50  # requests per minute
+RATE_WINDOW = 60  # seconds
+CACHE_FILENAME = "npm-status-cache.json"
+# Don't re-check packages checked within this window
+RECHECK_INTERVAL_SECONDS = 24 * 3600  # 24h
+
+
+def _rate_limiter():
+    """Generator-based rate limiter. Call next() before each request."""
+    timestamps = []
+    while True:
+        now = time.time()
+        # Purge timestamps older than the window
+        timestamps = [t for t in timestamps if now - t < RATE_WINDOW]
+        if len(timestamps) >= RATE_LIMIT:
+            sleep_time = timestamps[0] + RATE_WINDOW - now + 0.1
+            log.debug("Rate limit reached, sleeping %.1fs", sleep_time)
+            time.sleep(sleep_time)
+            now = time.time()
+            timestamps = [t for t in timestamps if now - t < RATE_WINDOW]
+        timestamps.append(now)
+        yield
+
+
+def _fetch_package_info(session, name, limiter):
+    """Fetch package metadata from npm. Returns (status, info) tuple."""
+    next(limiter)
+
+    url = f"{NPM_REGISTRY}/{name}"
+    for attempt in range(3):
+        try:
+            resp = session.get(url, timeout=15)
+
+            if resp.status_code == 404:
+                return "npm_removed", {"reason": "package_404"}
+
+            if resp.status_code == 429:
+                retry_after = int(resp.headers.get("Retry-After", 30))
+                log.warning("npm 429 for %s, waiting %ds", name, retry_after)
+                time.sleep(retry_after)
+                continue
+
+            resp.raise_for_status()
+            return "npm_available", resp.json()
+
+        except requests.RequestException as e:
+            wait = 2 ** attempt * 3
+            log.warning("npm fetch failed for %s (attempt %d): %s",
+                        name, attempt + 1, e)
+            time.sleep(wait)
+
+    return "npm_error", {"reason": "fetch_failed_after_retries"}
+
+
+def check_suspects(suspects, cache_dir):
+    """Check npm status for each suspect. Returns dict of results.
+
+    Args:
+        suspects: list of dicts with 'package', 'version', 'ecosystem' keys
+        cache_dir: path to cache directory
+
+    Returns:
+        dict keyed by "name@version" with status info
+    """
+    cache_dir = Path(cache_dir)
+    cache_dir.mkdir(parents=True, exist_ok=True)
+    cache_path = cache_dir / CACHE_FILENAME
+
+    # Load existing cache for resumability
+    cache = _load_cache(cache_path)
+
+    # Deduplicate suspects by name@version, npm only
+    unique = {}
+    for s in suspects:
+        if s.get("ecosystem") != "npm":
+            continue
+        key = f"{s['package']}@{s['version']}"
+        if key not in unique:
+            unique[key] = s
+
+    # Filter out recently checked
+    now = time.time()
+    to_check = {}
+    for key, s in unique.items():
+        cached = cache.get(key)
+        if cached and (now - cached.get("checked_at", 0)) < RECHECK_INTERVAL_SECONDS:
+            continue
+        to_check[key] = s
+
+    log.info("npm check: %d unique suspects, %d already cached, %d to check",
+             len(unique), len(unique) - len(to_check), len(to_check))
+
+    if not to_check:
+        return cache
+
+    session = requests.Session()
+    session.headers.update({"Accept": "application/json"})
+    limiter = _rate_limiter()
+
+    checked = 0
+    # Group by package name to avoid redundant fetches
+    by_name = {}
+    for key, s in to_check.items():
+        name = s["package"]
+        if name not in by_name:
+            by_name[name] = []
+        by_name[name].append((key, s))
+
+    total_packages = len(by_name)
+
+    for i, (name, entries) in enumerate(by_name.items()):
+        status, info = _fetch_package_info(session, name, limiter)
+
+        if i > 0 and i % 100 == 0:
+            log.info("npm check progress: %d/%d packages (%.0f%%)",
+                     i, total_packages, i / total_packages * 100)
+            _save_cache(cache, cache_path)
+
+        for key, s in entries:
+            version = s["version"]
+            result = {
+                "status": status,
+                "checked_at": now,
+            }
+
+            if status == "npm_available" and isinstance(info, dict):
+                versions = info.get("versions", {})
+                time_info = info.get("time", {})
+
+                if version not in versions:
+                    result["status"] = "npm_removed"
+                    result["reason"] = "version_removed"
+                else:
+                    result["reason"] = "available"
+
+                # Extract timing for temporal correlation
+                publish_time = time_info.get(version)
+                if publish_time:
+                    result["publish_date"] = publish_time
+
+                # Extract latest version publish time
+                modified = time_info.get("modified")
+                if modified:
+                    result["last_modified"] = modified
+
+            elif status == "npm_removed":
+                result["reason"] = "package_404"
+
+            cache[key] = result
+            checked += 1
+
+    _save_cache(cache, cache_path)
+    log.info("npm check complete: %d packages checked, %d total cached",
+             checked, len(cache))
+
+    return cache
+
+
+def _load_cache(cache_path):
+    """Load npm status cache from disk."""
+    if not cache_path.is_file():
+        return {}
+    try:
+        with open(cache_path, "r", encoding="utf-8") as f:
+            data = json.load(f)
+        if isinstance(data, dict) and "results" in data:
+            return data["results"]
+        return {}
+    except (json.JSONDecodeError, OSError):
+        return {}
+
+
+def _save_cache(cache, cache_path):
+    """Save npm status cache to disk."""
+    try:
+        with open(cache_path, "w", encoding="utf-8") as f:
+            json.dump({
+                "saved_at": datetime.utcnow().isoformat() + "Z",
+                "count": len(cache),
+                "results": cache,
+            }, f)
+    except OSError as e:
+        log.error("Failed to save npm cache: %s", e)
+
+
+def is_quick_takedown(result, detection_date_str, threshold_hours=72):
+    """Check if a package was removed quickly after publish (npm security takedown pattern).
+
+    Returns True if the package was removed AND was published recently
+    relative to the detection date (within threshold_hours).
+    """
+    if result.get("status") != "npm_removed":
+        return False
+
+    publish_date = result.get("publish_date")
+    if not publish_date:
+        return False
+
+    try:
+        publish_dt = datetime.fromisoformat(publish_date.replace("Z", "+00:00"))
+        detection_dt = datetime.fromisoformat(detection_date_str.replace("Z", "+00:00"))
+        delta_hours = (detection_dt - publish_dt).total_seconds() / 3600
+
+        # Package was detected within threshold_hours of publish
+        # AND has since been removed → strong takedown signal
+        return 0 <= delta_hours <= threshold_hours
+    except (ValueError, TypeError):
+        return False

package/ml-retrain/auto-labeler/ossf_index.py

@@ -0,0 +1,178 @@
+"""
+OSSF malicious-packages indexer.
+
+Clones (or updates) the ossf/malicious-packages repo with sparse checkout
+limited to osv/malicious/npm/, then parses all OSV JSON files into an index.
+Skips osv/withdrawn/ (retracted false positives).
+"""
+
+import json
+import logging
+import os
+import subprocess
+from datetime import datetime
+from pathlib import Path
+
+log = logging.getLogger("auto-labeler.ossf")
+
+OSSF_REPO_URL = "https://github.com/ossf/malicious-packages.git"
+OSSF_SPARSE_PATH = "osv/malicious/npm"
+
+INDEX_FILENAME = "ossf-index.json"
+
+
+def _run_git(args, cwd=None):
+    """Run a git command, raise on failure."""
+    result = subprocess.run(
+        ["git"] + args,
+        cwd=cwd,
+        capture_output=True,
+        text=True,
+        timeout=300,
+    )
+    if result.returncode != 0:
+        raise RuntimeError(f"git {' '.join(args)} failed: {result.stderr.strip()}")
+    return result.stdout.strip()
+
+
+def clone_or_update(repo_dir):
+    """Clone with sparse checkout or git pull if already present."""
+    repo_dir = Path(repo_dir)
+
+    if (repo_dir / ".git").is_dir():
+        log.info("OSSF repo exists at %s — pulling latest", repo_dir)
+        _run_git(["pull", "--ff-only"], cwd=repo_dir)
+        return
+
+    log.info("Cloning OSSF repo (sparse, depth=1) to %s", repo_dir)
+    repo_dir.mkdir(parents=True, exist_ok=True)
+
+    _run_git(["clone", "--depth", "1", "--filter=blob:none",
+              "--sparse", OSSF_REPO_URL, str(repo_dir)])
+    _run_git(["sparse-checkout", "set", OSSF_SPARSE_PATH], cwd=repo_dir)
+    log.info("OSSF clone complete (sparse: %s)", OSSF_SPARSE_PATH)
+
+
+def _parse_osv_file(filepath):
+    """Parse a single OSV JSON file and yield (key, entry) tuples."""
+    try:
+        with open(filepath, "r", encoding="utf-8") as f:
+            data = json.load(f)
+    except (json.JSONDecodeError, OSError) as e:
+        log.warning("Skipping invalid OSV file %s: %s", filepath, e)
+        return
+
+    osv_id = data.get("id", "")
+    published = data.get("published", "")
+    summary = data.get("summary", "")
+
+    # Extract attack type from database_specific if available
+    attack_type = None
+    db_specific = data.get("database_specific", {})
+    origins = db_specific.get("malicious-packages-origins", [])
+    if origins:
+        attack_type = origins[0].get("reason", None)
+
+    for affected in data.get("affected", []):
+        pkg = affected.get("package", {})
+        ecosystem = pkg.get("ecosystem", "").lower()
+        name = pkg.get("name", "")
+
+        if ecosystem != "npm" or not name:
+            continue
+
+        # Collect explicit versions
+        versions = affected.get("versions", [])
+
+        # Also extract versions from ranges
+        for rng in affected.get("ranges", []):
+            events = rng.get("events", [])
+            for event in events:
+                if "introduced" in event and event["introduced"] != "0":
+                    versions.append(event["introduced"])
+
+        entry = {
+            "source": "ossf",
+            "osv_id": osv_id,
+            "date": published,
+            "summary": summary[:200],
+            "attack_type": attack_type,
+        }
+
+        if versions:
+            for ver in set(versions):
+                yield f"{name}@{ver}", entry
+        else:
+            # No specific versions — all versions affected
+            yield f"{name}@*", entry
+
+
+def build_index(repo_dir, cache_dir):
+    """Build OSSF index from the cloned repo. Returns the index dict."""
+    repo_dir = Path(repo_dir)
+    cache_dir = Path(cache_dir)
+    osv_dir = repo_dir / "osv" / "malicious" / "npm"
+
+    if not osv_dir.is_dir():
+        log.error("OSSF osv/malicious/npm/ not found at %s", osv_dir)
+        return {}
+
+    index = {}
+    file_count = 0
+    entry_count = 0
+
+    for root, _dirs, files in os.walk(osv_dir):
+        # Skip withdrawn reports
+        if "withdrawn" in Path(root).parts:
+            continue
+
+        for fname in files:
+            if not fname.endswith(".json"):
+                continue
+
+            filepath = os.path.join(root, fname)
+            file_count += 1
+
+            for key, entry in _parse_osv_file(filepath):
+                index[key] = entry
+                entry_count += 1
+
+    log.info("OSSF index: %d entries from %d files", entry_count, file_count)
+
+    # Cache to disk
+    cache_dir.mkdir(parents=True, exist_ok=True)
+    cache_path = cache_dir / INDEX_FILENAME
+    with open(cache_path, "w", encoding="utf-8") as f:
+        json.dump({"built_at": datetime.utcnow().isoformat() + "Z",
+                    "count": len(index),
+                    "index": index}, f)
+    log.info("OSSF index cached to %s", cache_path)
+
+    return index
+
+
+def load_cached_index(cache_dir):
+    """Load index from cache if available."""
+    cache_path = Path(cache_dir) / INDEX_FILENAME
+    if not cache_path.is_file():
+        return None
+    try:
+        with open(cache_path, "r", encoding="utf-8") as f:
+            data = json.load(f)
+        log.info("Loaded cached OSSF index (%d entries, built %s)",
+                 data.get("count", 0), data.get("built_at", "?"))
+        return data.get("index", {})
+    except (json.JSONDecodeError, OSError) as e:
+        log.warning("Failed to load OSSF cache: %s", e)
+        return None
+
+
+def lookup(index, name, version):
+    """Check if a package@version is in the OSSF index.
+
+    Returns the entry dict or None. Checks both exact version and wildcard.
+    """
+    exact = index.get(f"{name}@{version}")
+    if exact:
+        return exact
+    return index.get(f"{name}@*")

package/ml-retrain/auto-labeler/requirements.txt

@@ -0,0 +1 @@
+requests>=2.28.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "muaddib-scanner",
-  "version": "2.10.49",
+  "version": "2.10.50",
   "description": "Supply-chain threat detection & response for npm & PyPI/Python",
   "main": "src/index.js",
   "bin": {