muaddib-scanner 2.11.76 → 2.11.78

package/src/scanner/temporal-analysis.js

@@ -121,6 +121,14 @@ function _fetchPackageMetadataHttp(packageName) {
         return;
       }
 
+      if (res.statusCode === 429) {
+        res.resume();
+        // Coordinated backoff on the shared registry limiter — the temporal scanners must
+        // signal 429 like the metadata path, not hammer through a rate limit (CLAUDE.md storm).
+        try { require('../shared/http-limiter.js').signal429(); } catch { /* limiter best-effort */ }
+        reject(new Error(`Registry rate limited (HTTP 429) for ${packageName}`));
+        return;
+      }
       if (res.statusCode < 200 || res.statusCode >= 300) {
         res.resume();
         reject(new Error(`Registry returned HTTP ${res.statusCode} for ${packageName}`));

package/src/scanner/temporal-ast-diff.js

@@ -71,6 +71,11 @@ function _fetchVersionMetadataHttp(packageName, version) {
         res.resume();
         return reject(new Error(`Version ${version} not found for package ${packageName}`));
       }
+      if (res.statusCode === 429) {
+        res.resume();
+        try { require('../shared/http-limiter.js').signal429(); } catch { /* limiter best-effort */ }
+        return reject(new Error(`Registry rate limited (HTTP 429) for ${packageName}@${version}`));
+      }
       if (res.statusCode < 200 || res.statusCode >= 300) {
         res.resume();
         return reject(new Error(`Registry returned HTTP ${res.statusCode} for ${packageName}@${version}`));

package/src/utils.js

@@ -392,6 +392,62 @@ function debugLog(...args) {
   if (process.env.MUADDIB_DEBUG) console.error('[DEBUG]', ...args);
 }
 
+// eslint-disable-next-line no-control-regex -- ESC (\x1b) is required to strip ANSI color sequences before measuring visible width
+const _ANSI_RE = /\x1b\[[0-9;]*m/g;
+
+/**
+ * Draws an aligned box-drawing banner around the given content line(s).
+ * Width auto-fits the widest VISIBLE line (ANSI stripped before measuring),
+ * so the right border is always straight — replacing the hand-padded boxes
+ * that drifted out of alignment. Returns the banner as a string (no leading or
+ * trailing blank lines — callers add spacing as needed).
+ *
+ * @param {string[]|string} lines - content line(s)
+ * @returns {string}
+ */
+function banner(lines) {
+  const content = Array.isArray(lines) ? lines : [String(lines)];
+  const visibleLen = (s) => String(s).replace(_ANSI_RE, '').length;
+  const PAD = 1; // spaces of padding on each side
+  const inner = content.reduce((max, l) => Math.max(max, visibleLen(l)), 0) + PAD * 2;
+  const top = '╔' + '═'.repeat(inner) + '╗';
+  const bottom = '╚' + '═'.repeat(inner) + '╝';
+  const body = content.map((l) => {
+    const trailing = inner - PAD - visibleLen(l);
+    return '║' + ' '.repeat(PAD) + l + ' '.repeat(Math.max(0, trailing)) + '║';
+  });
+  return [top, ...body, bottom].join('\n');
+}
+
+/**
+ * True when an error represents a user-initiated prompt cancellation — Ctrl-C
+ * inside an @inquirer/prompts prompt, which throws an ExitPromptError whose
+ * message contains "force closed the prompt with SIGINT". Lets the CLI exit
+ * cleanly (code 130) instead of dumping an [ERROR] line or a stack trace.
+ *
+ * @param {*} err
+ * @returns {boolean}
+ */
+function isPromptCancellation(err) {
+  if (!err) return false;
+  if (err.name === 'ExitPromptError') return true;
+  return /SIGINT|force closed the prompt/i.test(err.message || '');
+}
+
+/**
+ * Renders a fixed-width 20-cell [██░░] bar for a 0–100 risk score. Clamps and
+ * guards against undefined / NaN / out-of-range so the CLI never throws a
+ * RangeError from String.prototype.repeat on a malformed score.
+ *
+ * @param {number} score
+ * @returns {string} a 20-character bar
+ */
+function renderScoreBar(score) {
+  const s = Number.isFinite(score) ? Math.max(0, Math.min(100, score)) : 0;
+  const filled = Math.floor(s / 5);
+  return '█'.repeat(filled) + '░'.repeat(20 - filled);
+}
+
 module.exports = {
   EXCLUDED_DIRS,
   MAX_SCAN_FILES,
@@ -409,5 +465,8 @@ module.exports = {
   getExtraExcludes,
   forEachSafeFile,
   listInstalledPackages,
-  debugLog
+  debugLog,
+  banner,
+  isPromptCancellation,
+  renderScoreBar
 };

package/.dockerignore

@@ -1,7 +0,0 @@
-node_modules
-.git
-datasets
-tests
-metrics
-.muaddib-cache
-*.md

package/.env.example

@@ -1,43 +0,0 @@
-# MUAD'DIB environment variables — template
-# Copy to .env (local dev) or /opt/muaddib/.env (VPS) and fill in real values.
-# .env files are gitignored. NEVER commit a real token.
-
-# ----------------------------------------------------------------------------
-# Threat-feed API tokens (all OPTIONAL — scrapers degrade gracefully if absent)
-# ----------------------------------------------------------------------------
-
-# OpenSourceMalware.com — community-verified threat intel
-# Free tier: 60 req/min, /query-latest gives 100 most recent threats per ecosystem.
-# Sign up + generate at: https://opensourcemalware.com/auth → profile → API Tokens
-# Format: osm_<random-32+chars>
-# Used by: src/ioc/scraper.js → scrapeOSMQueryLatest()
-OSM_API_TOKEN=
-
-# ----------------------------------------------------------------------------
-# Webhook destinations (optional — monitor alerts)
-# ----------------------------------------------------------------------------
-
-# Discord webhook for monitor alerts (P1/P2/P3 triage)
-# DISCORD_WEBHOOK_URL=
-
-# ----------------------------------------------------------------------------
-# FPR plan gates — DEFAULT ON since v2.11.9 (no need to set these unless opting OUT)
-# ----------------------------------------------------------------------------
-# Measured impact on the v2.11.4 evaluation corpus (1054 packages):
-#   FPR curated 15.6% -> 9.36% (-6.24 pp), FPR random 7.0% -> 2.0% (-5.00 pp).
-#   TPR@3 / TPR@20 / ADR strictly unchanged.
-#
-# Opt-OUT individual gates (uncomment + set to 0):
-# MUADDIB_FN_REACHABILITY=0   # function-level reachability gating
-# MUADDIB_DECAY=0             # group score decay on bundled outputs
-# MUADDIB_MATURE_CAP=0        # cap mature, well-trafficked packages at MEDIUM
-# MUADDIB_METADATA_FACTOR=0   # registry signals -> reputation multiplier
-# MUADDIB_DELTA_MODE=0        # delta scoring against prior versions
-#
-# Skip ALL network fetches (npm registry packument + GitHub Releases IOC
-# bootstrap) in one shot. Disables MATURE_CAP + METADATA_FACTOR + DELTA_MODE
-# at the per-scan level AND the first-run IOC database download. Useful for:
-#   - air-gap / offline CI environments
-#   - test runners (set automatically by tests/run-tests.js)
-#   - perf-critical batch scans where you've pre-warmed the IOC cache
-# MUADDIB_NO_REGISTRY_FETCH=1

package/ml-retrain/auto-labeler/auto_labeler.py

@@ -1,312 +0,0 @@
-#!/usr/bin/env python3
-"""
-MUAD'DIB Auto-Labeling Pipeline
-
-Correlates muaddib suspects with external signals (OSSF, GHSA, npm status)
-to produce ground truth labels for ML training.
-
-Usage:
-    python auto_labeler.py --full              # Run all steps
-    python auto_labeler.py --step ossf         # Run individual step
-    python auto_labeler.py --step npm
-    python auto_labeler.py --step ghsa
-    python auto_labeler.py --step label
-    python auto_labeler.py --update            # Cron mode: re-check pending/unconfirmed
-
-Environment:
-    GITHUB_TOKEN    Optional, for higher GHSA API rate limits
-    MUADDIB_DATA    Override data directory (default: /opt/muaddib/data)
-"""
-
-import argparse
-import json
-import logging
-import os
-import sys
-from datetime import datetime, timezone
-from pathlib import Path
-
-import ossf_index
-import ghsa_checker
-import npm_checker
-import labeler
-
-# ── Paths ──
-MUADDIB_DATA = Path(os.environ.get("MUADDIB_DATA", "/opt/muaddib/data"))
-MUADDIB_ALERTS = Path(os.environ.get("MUADDIB_ALERTS", "/opt/muaddib/logs/alerts"))
-BASE_DIR = Path(__file__).parent
-CACHE_DIR = BASE_DIR / "data"
-OSSF_REPO_DIR = CACHE_DIR / "ossf-malicious-packages"
-OUTPUT_PATH = MUADDIB_DATA / "auto-labels.json"
-
-log = logging.getLogger("auto-labeler")
-
-
-def setup_logging(verbose=False):
-    level = logging.DEBUG if verbose else logging.INFO
-    fmt = "%(asctime)s [%(name)s] %(levelname)s %(message)s"
-    logging.basicConfig(level=level, format=fmt, datefmt="%Y-%m-%d %H:%M:%S")
-
-
-def load_detections():
-    """Load detections.json from muaddib data directory."""
-    path = MUADDIB_DATA / "detections.json"
-    if not path.is_file():
-        log.error("detections.json not found at %s", path)
-        sys.exit(1)
-
-    with open(path, "r", encoding="utf-8") as f:
-        data = json.load(f)
-
-    detections = data.get("detections", [])
-    npm_count = sum(1 for d in detections if d.get("ecosystem") == "npm")
-    log.info("Loaded %d detections (%d npm)", len(detections), npm_count)
-    return detections
-
-
-def load_alert_scores():
-    """Load risk scores and tiers from individual alert files.
-
-    Scans logs/alerts/ for JSON files and extracts score + tier info.
-    Returns dict keyed by "name@version".
-    """
-    scores = {}
-
-    # Try cached scores first
-    cache_path = CACHE_DIR / "alert-scores-cache.json"
-    if cache_path.is_file():
-        try:
-            with open(cache_path, "r", encoding="utf-8") as f:
-                cached = json.load(f)
-            if cached.get("count", 0) > 0:
-                log.info("Loaded %d cached alert scores", cached["count"])
-                return cached.get("scores", {})
-        except (json.JSONDecodeError, OSError):
-            pass
-
-    if not MUADDIB_ALERTS.is_dir():
-        log.warning("Alerts directory not found at %s — scores will be estimated from severity",
-                     MUADDIB_ALERTS)
-        return scores
-
-    alert_files = list(MUADDIB_ALERTS.glob("*.json"))
-    log.info("Scanning %d alert files for scores...", len(alert_files))
-
-    for filepath in alert_files:
-        try:
-            with open(filepath, "r", encoding="utf-8") as f:
-                alert = json.load(f)
-
-            target = alert.get("target", "")
-            summary = alert.get("summary", {})
-            score = summary.get("riskScore", summary.get("globalRiskScore", 0))
-
-            # Parse target: "npm/package-name@version" or "pypi/package@version"
-            if "/" in target and "@" in target:
-                eco_pkg = target.split("/", 1)
-                if len(eco_pkg) == 2:
-                    pkg_ver = eco_pkg[1]
-                    # Determine tier from priority
-                    priority = alert.get("priority", {})
-                    tier = ""
-                    p_level = priority.get("level", "")
-                    if p_level == "P1":
-                        tier = "T1a"
-                    elif p_level == "P2":
-                        tier = "T1b"
-                    elif p_level == "P3":
-                        tier = "T2"
-
-                    scores[pkg_ver] = {"score": score, "tier": tier}
-
-        except (json.JSONDecodeError, OSError):
-            continue
-
-    # Cache for next run
-    CACHE_DIR.mkdir(parents=True, exist_ok=True)
-    with open(cache_path, "w", encoding="utf-8") as f:
-        json.dump({"count": len(scores), "built_at": datetime.now(timezone.utc).isoformat(),
-                    "scores": scores}, f)
-
-    log.info("Extracted scores from %d alerts", len(scores))
-    return scores
-
-
-# ── Steps ──
-
-def step_ossf():
-    """Step 1: Index OSSF malicious-packages."""
-    log.info("=== Step 1: OSSF Index ===")
-    ossf_index.clone_or_update(OSSF_REPO_DIR)
-    index = ossf_index.build_index(OSSF_REPO_DIR, CACHE_DIR)
-    return index
-
-
-def step_ghsa():
-    """Step 3: Index GitHub Advisory Database."""
-    log.info("=== Step 3: GHSA Index ===")
-    # Try cache first
-    index = ghsa_checker.load_cached_index(CACHE_DIR)
-    if index is not None:
-        return index
-    return ghsa_checker.build_index(CACHE_DIR)
-
-
-def step_npm(detections):
-    """Step 2: Check npm status for suspects."""
-    log.info("=== Step 2: npm Status Check ===")
-    return npm_checker.check_suspects(detections, CACHE_DIR)
-
-
-def step_label(detections, o_index, g_index, npm_status, alert_scores):
-    """Step 4: Generate labels."""
-    log.info("=== Step 4: Generate Labels ===")
-
-    labels = labeler.label_suspects(detections, o_index, g_index, npm_status, alert_scores)
-    missed = labeler.find_missed(o_index, g_index, detections)
-    summary = labeler.export_labels(labels, missed, OUTPUT_PATH)
-
-    return summary
-
-
-# ── Modes ──
-
-def run_full():
-    """Run all steps sequentially."""
-    log.info("Starting full auto-labeling pipeline")
-    start = datetime.now()
-
-    detections = load_detections()
-    alert_scores = load_alert_scores()
-
-    # Steps 1+3 don't depend on detections — could be parallel but keep it simple
-    o_index = step_ossf()
-    g_index = step_ghsa()
-    npm_status = step_npm(detections)
-
-    summary = step_label(detections, o_index, g_index, npm_status, alert_scores)
-
-    elapsed = (datetime.now() - start).total_seconds()
-    log.info("Pipeline complete in %.1fs — %s", elapsed, summary)
-    return summary
-
-
-def run_update():
-    """Cron mode: re-check pending/unconfirmed labels against fresh external data."""
-    log.info("Starting update (cron mode)")
-
-    # Refresh external indices
-    ossf_index.clone_or_update(OSSF_REPO_DIR)
-    o_index = ossf_index.build_index(OSSF_REPO_DIR, CACHE_DIR)
-    g_index = ghsa_checker.build_index(CACHE_DIR)
-
-    # Load existing labels
-    if not OUTPUT_PATH.is_file():
-        log.error("No existing auto-labels.json — run --full first")
-        sys.exit(1)
-
-    with open(OUTPUT_PATH, "r", encoding="utf-8") as f:
-        existing = json.load(f)
-
-    existing_labels = existing.get("labels", {})
-    detections = load_detections()
-    alert_scores = load_alert_scores()
-
-    # Find labels that need re-evaluation
-    to_recheck = []
-    for key, entry in existing_labels.items():
-        lbl = entry.get("auto_label")
-        if lbl in ("pending", "unconfirmed", "likely_malicious"):
-            # Re-extract detection info
-            for det in detections:
-                if f"{det['package']}@{det['version']}" == key:
-                    to_recheck.append(det)
-                    break
-
-    if not to_recheck:
-        log.info("No pending/unconfirmed labels to re-check")
-        return
-
-    log.info("Re-checking %d labels (pending/unconfirmed/likely_malicious)", len(to_recheck))
-
-    # Re-check npm status for these specific packages
-    npm_status = npm_checker.check_suspects(to_recheck, CACHE_DIR)
-
-    # Re-label
-    updated = labeler.label_suspects(to_recheck, o_index, g_index, npm_status, alert_scores)
-
-    # Merge updates into existing labels
-    changes = 0
-    for key, new_entry in updated.items():
-        old = existing_labels.get(key, {})
-        if old.get("auto_label") != new_entry.get("auto_label"):
-            log.info("RELABEL: %s — %s → %s",
-                     key, old.get("auto_label"), new_entry.get("auto_label"))
-            changes += 1
-        existing_labels[key] = new_entry
-
-    # Also refresh missed detection
-    missed = labeler.find_missed(o_index, g_index, detections)
-    for name, info in missed.items():
-        mk = f"{name}@*"
-        if mk not in existing_labels:
-            existing_labels[mk] = info
-            changes += 1
-
-    # Re-export
-    labeler.export_labels(
-        {k: v for k, v in existing_labels.items() if v.get("auto_label") != "missed"},
-        {k.replace("@*", ""): v for k, v in existing_labels.items() if v.get("auto_label") == "missed"},
-        OUTPUT_PATH,
-    )
-
-    log.info("Update complete: %d labels changed", changes)
-
-
-def main():
-    parser = argparse.ArgumentParser(description="MUAD'DIB Auto-Labeling Pipeline")
-    group = parser.add_mutually_exclusive_group(required=True)
-    group.add_argument("--full", action="store_true", help="Run all steps")
-    group.add_argument("--step", choices=["ossf", "ghsa", "npm", "label"],
-                       help="Run individual step")
-    group.add_argument("--update", action="store_true",
-                       help="Cron: re-check pending/unconfirmed")
-    parser.add_argument("-v", "--verbose", action="store_true", help="Debug logging")
-    parser.add_argument("--data-dir", help="Override MUADDIB_DATA path")
-    parser.add_argument("--alerts-dir", help="Override MUADDIB_ALERTS path")
-    args = parser.parse_args()
-
-    setup_logging(args.verbose)
-
-    if args.data_dir:
-        global MUADDIB_DATA, OUTPUT_PATH
-        MUADDIB_DATA = Path(args.data_dir)
-        OUTPUT_PATH = MUADDIB_DATA / "auto-labels.json"
-    if args.alerts_dir:
-        global MUADDIB_ALERTS
-        MUADDIB_ALERTS = Path(args.alerts_dir)
-
-    if args.full:
-        run_full()
-    elif args.update:
-        run_update()
-    elif args.step == "ossf":
-        step_ossf()
-    elif args.step == "ghsa":
-        step_ghsa()
-    elif args.step == "npm":
-        step_npm(load_detections())
-    elif args.step == "label":
-        detections = load_detections()
-        alert_scores = load_alert_scores()
-        o_index = ossf_index.load_cached_index(CACHE_DIR)
-        g_index = ghsa_checker.load_cached_index(CACHE_DIR)
-        npm_status = npm_checker._load_cache(CACHE_DIR / npm_checker.CACHE_FILENAME)
-        if o_index is None or g_index is None:
-            log.error("Run --step ossf and --step ghsa first (or use --full)")
-            sys.exit(1)
-        step_label(detections, o_index, g_index, npm_status, alert_scores)
-
-
-if __name__ == "__main__":
-    main()

package/ml-retrain/auto-labeler/ghsa_checker.py

@@ -1,169 +0,0 @@
-"""
-GitHub Advisory Database checker.
-
-Fetches all npm malware advisories from the GitHub Advisory Database API.
-Supports optional GITHUB_TOKEN env var for higher rate limits.
-"""
-
-import json
-import logging
-import os
-import time
-from datetime import datetime
-from pathlib import Path
-
-import requests
-
-log = logging.getLogger("auto-labeler.ghsa")
-
-GHSA_API = "https://api.github.com/advisories"
-INDEX_FILENAME = "ghsa-index.json"
-# Cache validity: 12 hours
-CACHE_TTL_SECONDS = 12 * 3600
-
-
-def _get_headers():
-    token = os.environ.get("GITHUB_TOKEN")
-    headers = {"Accept": "application/vnd.github+json"}
-    if token:
-        headers["Authorization"] = f"Bearer {token}"
-        log.info("Using GITHUB_TOKEN for GHSA API (5000 req/h)")
-    else:
-        log.info("No GITHUB_TOKEN — GHSA API limited to 60 req/h")
-    return headers
-
-
-def fetch_malware_advisories():
-    """Fetch all npm malware advisories from GHSA. Returns list of advisories."""
-    headers = _get_headers()
-    advisories = []
-    page = 1
-    per_page = 100
-
-    while True:
-        params = {
-            "type": "malware",
-            "ecosystem": "npm",
-            "per_page": per_page,
-            "page": page,
-        }
-
-        for attempt in range(3):
-            try:
-                resp = requests.get(GHSA_API, headers=headers, params=params, timeout=30)
-
-                if resp.status_code == 403:
-                    # Rate limited
-                    retry_after = int(resp.headers.get("Retry-After", 60))
-                    log.warning("GHSA rate limited, waiting %ds", retry_after)
-                    time.sleep(retry_after)
-                    continue
-
-                resp.raise_for_status()
-                break
-            except requests.RequestException as e:
-                wait = 2 ** attempt * 5
-                log.warning("GHSA request failed (attempt %d): %s — retrying in %ds",
-                            attempt + 1, e, wait)
-                time.sleep(wait)
-        else:
-            log.error("GHSA fetch failed after 3 attempts on page %d", page)
-            break
-
-        batch = resp.json()
-        if not batch:
-            break
-
-        advisories.extend(batch)
-        log.info("GHSA page %d: %d advisories (total: %d)", page, len(batch), len(advisories))
-
-        if len(batch) < per_page:
-            break
-        page += 1
-        time.sleep(1)  # Courtesy delay
-
-    return advisories
-
-
-def build_index(cache_dir):
-    """Build GHSA index from API. Returns dict keyed by package name."""
-    advisories = fetch_malware_advisories()
-    index = {}
-
-    for adv in advisories:
-        ghsa_id = adv.get("ghsa_id", "")
-        published = adv.get("published_at", "")
-        summary = adv.get("summary", "")
-        withdrawn = adv.get("withdrawn_at")
-
-        # Skip withdrawn advisories
-        if withdrawn:
-            continue
-
-        for vuln in adv.get("vulnerabilities", []):
-            pkg = vuln.get("package", {})
-            ecosystem = pkg.get("ecosystem", "").lower()
-            name = pkg.get("name", "")
-
-            if ecosystem != "npm" or not name:
-                continue
-
-            version_range = vuln.get("vulnerable_version_range", "")
-
-            entry = {
-                "source": "ghsa",
-                "ghsa_id": ghsa_id,
-                "date": published,
-                "summary": summary[:200],
-                "version_range": version_range,
-            }
-
-            # Index by package name (version matching is approximate for GHSA)
-            if name not in index:
-                index[name] = []
-            index[name].append(entry)
-
-    log.info("GHSA index: %d packages from %d advisories", len(index), len(advisories))
-
-    # Cache to disk
-    cache_dir = Path(cache_dir)
-    cache_dir.mkdir(parents=True, exist_ok=True)
-    cache_path = cache_dir / INDEX_FILENAME
-    with open(cache_path, "w", encoding="utf-8") as f:
-        json.dump({"built_at": datetime.utcnow().isoformat() + "Z",
-                    "count": len(index),
-                    "index": index}, f)
-    log.info("GHSA index cached to %s", cache_path)
-
-    return index
-
-
-def load_cached_index(cache_dir):
-    """Load index from cache if fresh enough."""
-    cache_path = Path(cache_dir) / INDEX_FILENAME
-    if not cache_path.is_file():
-        return None
-    try:
-        stat = cache_path.stat()
-        age = time.time() - stat.st_mtime
-        if age > CACHE_TTL_SECONDS:
-            log.info("GHSA cache expired (%.1fh old)", age / 3600)
-            return None
-
-        with open(cache_path, "r", encoding="utf-8") as f:
-            data = json.load(f)
-        log.info("Loaded cached GHSA index (%d packages, built %s)",
-                 data.get("count", 0), data.get("built_at", "?"))
-        return data.get("index", {})
-    except (json.JSONDecodeError, OSError) as e:
-        log.warning("Failed to load GHSA cache: %s", e)
-        return None
-
-
-def lookup(index, name):
-    """Check if a package name is in the GHSA index.
-
-    Returns the list of advisory entries or None.
-    """
-    entries = index.get(name)
-    return entries if entries else None