@murk-security/runner-linux-arm64 0.0.0-stage → 0.7.0

package/runners/profile.py

@@ -0,0 +1,307 @@
+#!/usr/bin/env python3
+"""Preliminary-scan ("profile") pre-pass -- Phase 0.
+
+A DETERMINISTIC step that runs BEFORE the agent rounds and reuses prod_env's passive
+machinery to produce a profile.json: fingerprint + security-posture FACTS + candidate
+SIGNALS. It is a seed for the orchestrator, NOT a verdict engine -- every signal is an
+observation to confirm, never a finding (see docs/preliminary-scan-plan.md).
+
+Gating: this module does nothing unless a driver invokes it. prod_env consumes its
+output ONLY when BLACKGLASS_PROFILE is enabled and BLACKGLASS_PROFILE_PATH points at the
+file -- so with the feature off, the adapter is byte-identical to before.
+
+Phase 0 scope: section B of the catalog (security headers / cookie flags / transport),
+plus a light fingerprint. Passive, scope-bound (it only touches the origin, anonymously,
+via prod_env._http). Later phases add exposed-path sweep, version->CVE, the notable-CVE
+feed, safe-active signals, and the source/static track.
+
+    python3 profile.py --spec input-spec.json [--out profile.json] [--tier passive]
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import re
+import sys
+import time
+from pathlib import Path
+
+HERE = Path(__file__).resolve().parent
+sys.path.insert(0, str(HERE))
+import prod_env  # reuse Target + _http (scope-bound, WAF-friendly fetch)
+
+# Notable/recent-CVE feed (area: known-CVE / version->CVE track). A separate,
+# refreshable JSON data file -- NOT code -- because flagship threats (PHP2Shell etc.)
+# are always post-model-cutoff; point BLACKGLASS_CVE_FEED at a newer copy to update.
+DEFAULT_CVE_FEED = HERE / "cve_feed.json"
+# Matches "product/1.2.3" tokens in a Server / X-Powered-By banner (>=2 version parts).
+_FP_RE = re.compile(r"([A-Za-z][A-Za-z0-9_+.-]*?)/(\d+(?:\.\d+)+)")
+
+# Exposed-path sweep (catalog area C). A CAPPED list of high-signal paths, each with a
+# content SIGNATURE (`must`) the 200 body has to match -- that gate is what gives
+# precision: a soft-404 / SPA catch-all / decoy that returns 200 with unrelated content
+# does NOT flag. Signatures distilled from the nuclei exposures/misconfiguration families
+# (MIT) -- as knowledge, not by running the YAML DSL. Categories drive severity/confidence.
+MAX_SWEEP_PATHS = 40
+EXPOSED_PATHS = [
+    {"path": "/.git/config", "cat": "vcs", "must": r"\[core\]|repositoryformatversion|\[remote "},
+    {"path": "/.git/HEAD", "cat": "vcs", "must": r"ref:\s*refs/"},
+    {"path": "/.svn/entries", "cat": "vcs", "must": r"svn|dir\b|\d+\n"},
+    {"path": "/.hg/requires", "cat": "vcs", "must": r"revlog|dotencode|store"},
+    {"path": "/.env", "cat": "secrets", "must": r"(?m)^[A-Z0-9_]{2,}="},
+    {"path": "/.env.local", "cat": "secrets", "must": r"(?m)^[A-Z0-9_]{2,}="},
+    {"path": "/.env.production", "cat": "secrets", "must": r"(?m)^[A-Z0-9_]{2,}="},
+    {"path": "/.aws/credentials", "cat": "secrets", "must": r"aws_access_key_id|aws_secret_access_key"},
+    {"path": "/config.php.bak", "cat": "secrets", "must": r"<\?php|define\(|DB_"},
+    {"path": "/wp-config.php.bak", "cat": "secrets", "must": r"DB_PASSWORD|DB_NAME|AUTH_KEY"},
+    {"path": "/backup.sql", "cat": "backup", "must": r"INSERT INTO|CREATE TABLE|-- dump|DROP TABLE"},
+    {"path": "/dump.sql", "cat": "backup", "must": r"INSERT INTO|CREATE TABLE|-- dump|DROP TABLE"},
+    {"path": "/database.sql", "cat": "backup", "must": r"INSERT INTO|CREATE TABLE|-- dump"},
+    {"path": "/server-status", "cat": "debug", "must": r"Apache Server Status|Server uptime|Total Traffic"},
+    {"path": "/server-info", "cat": "debug", "must": r"Apache Server Information|Server Settings"},
+    {"path": "/phpinfo.php", "cat": "debug", "must": r"phpinfo\(\)|PHP Version|php\.ini"},
+    {"path": "/info.php", "cat": "debug", "must": r"phpinfo\(\)|PHP Version"},
+    {"path": "/actuator", "cat": "debug", "must": r"_links|/actuator"},
+    {"path": "/actuator/env", "cat": "debug", "must": r"propertySources|activeProfiles"},
+    {"path": "/actuator/heapdump", "cat": "debug", "must": r"JAVA PROFILE|HPROF"},
+    {"path": "/swagger-ui.html", "cat": "api-doc", "must": r"Swagger UI|swagger-ui"},
+    {"path": "/swagger.json", "cat": "api-doc", "must": r"\"swagger\"|\"openapi\""},
+    {"path": "/openapi.json", "cat": "api-doc", "must": r"\"openapi\"|\"swagger\""},
+    {"path": "/v2/api-docs", "cat": "api-doc", "must": r"\"swagger\"|\"openapi\""},
+    {"path": "/api-docs", "cat": "api-doc", "must": r"\"swagger\"|\"openapi\"|\"paths\""},
+    {"path": "/graphql", "cat": "api", "must": r"__schema|\"data\"|\"errors\"|GraphQL"},
+    {"path": "/phpmyadmin/", "cat": "admin", "must": r"phpMyAdmin|pma_"},
+    {"path": "/adminer.php", "cat": "admin", "must": r"Adminer|Login - Adminer"},
+    {"path": "/.gitlab-ci.yml", "cat": "ci", "must": r"(?m)^stages:|^script:|^image:"},
+    {"path": "/Jenkinsfile", "cat": "ci", "must": r"pipeline\s*\{|stage\(|node\s*\{"},
+    {"path": "/WEB-INF/web.xml", "cat": "framework", "must": r"<web-app|<servlet"},
+    {"path": "/.well-known/security.txt", "cat": "hygiene-positive", "must": r"Contact:|Policy:|Expires:"},
+]
+
+# Security response headers whose ABSENCE is a posture gap (a neutral FACT, not a vuln).
+SECURITY_HEADERS = {
+    "strict-transport-security": "HSTS (transport downgrade protection)",
+    "content-security-policy": "CSP (XSS/injection mitigation)",
+    "x-frame-options": "clickjacking protection (or frame-ancestors in CSP)",
+    "x-content-type-options": "MIME-sniffing protection (nosniff)",
+    "referrer-policy": "Referrer-Policy",
+    "permissions-policy": "Permissions-Policy",
+}
+
+
+def _lower_keys(headers: dict) -> dict:
+    return {str(k).lower(): str(v) for k, v in (headers or {}).items()}
+
+
+def header_posture(headers: dict) -> tuple[dict, list[dict]]:
+    """Return (posture, gaps). posture records each security header present/absent (+
+    value); gaps is one signal-shaped dict per MISSING header. Pure parsing -- the same
+    input always yields the same output, so a benign (well-secured) response reports no
+    gap for a header it actually sets (precision)."""
+    low = _lower_keys(headers)
+    posture, gaps = {}, []
+    for h, desc in SECURITY_HEADERS.items():
+        present = h in low
+        posture[h] = {"present": present, "value": low.get(h, "")}
+        if not present:
+            gaps.append({"area": "security-header", "title": f"missing {h}", "detail": desc,
+                         "confidence": "high", "tier_to_confirm": "passive"})
+    # CORS posture (fact only): a reflected-Origin + credentials combo is the dangerous one.
+    acao = low.get("access-control-allow-origin")
+    acac = low.get("access-control-allow-credentials")
+    if acao is not None:
+        posture["cors"] = {"allow_origin": acao, "allow_credentials": acac or ""}
+    return posture, gaps
+
+
+def cookie_posture(set_cookie: str) -> list[dict]:
+    """Best-effort flags per Set-Cookie value. urllib may join multiple cookies; we scan
+    the raw string for the flags, which is enough for a Phase-0 posture note."""
+    out = []
+    if not set_cookie:
+        return out
+    for chunk in set_cookie.split(","):
+        name = chunk.strip().split("=", 1)[0].strip()
+        if not name or " " in name:
+            continue
+        low = chunk.lower()
+        out.append({"name": name, "secure": "secure" in low, "httponly": "httponly" in low,
+                    "samesite": ("samesite" in low)})
+    return out
+
+
+def exposed_sweep(t) -> tuple[list[dict], list[dict]]:
+    """GET each capped path; flag a signal only on 200 AND a content-signature match
+    (the precision gate). hygiene-positive hits (security.txt) go to posture, not signals."""
+    signals, positives = [], []
+    for spec in EXPOSED_PATHS[:MAX_SWEEP_PATHS]:
+        r = prod_env._http(t, {"path": spec["path"]})
+        if not r.get("ok") or r.get("status") != 200:
+            continue
+        body = r.get("body") or ""
+        if not re.search(spec["must"], body, re.I | re.M):
+            continue  # 200 but content does not match -> soft-404/decoy; skip (precision)
+        if spec["cat"] == "hygiene-positive":
+            positives.append({"path": spec["path"], "note": "security.txt present"})
+            continue
+        signals.append({
+            "area": "exposed-path", "category": spec["cat"],
+            "title": f"{spec['path']} readable ({spec['cat']})", "path": spec["path"], "status": 200,
+            "confidence": "high" if spec["cat"] in ("vcs", "secrets", "backup") else "med",
+            "tier_to_confirm": "passive", "snippet": body[:160],
+        })
+    return signals, positives
+
+
+def load_cve_feed(path: str | None = None) -> list[dict]:
+    """Load the version->CVE feed (list of entries). Missing/bad feed => []."""
+    p = Path(path or os.environ.get("BLACKGLASS_CVE_FEED", "") or DEFAULT_CVE_FEED)
+    try:
+        data = json.loads(p.read_text())
+    except Exception:
+        return []
+    return data.get("entries", []) if isinstance(data, dict) else (data or [])
+
+
+def _vtuple(v) -> tuple:
+    """A dotted version -> tuple of ints ('8.1.2-beta+5' -> (8,1,2)). No digits -> ()."""
+    head = re.split(r"[-+ ]", str(v).strip(), 1)[0]
+    parts = re.findall(r"\d+", head)
+    return tuple(int(x) for x in parts) if parts else ()
+
+
+def _vcmp(a: tuple, b: tuple) -> int:
+    """Compare version tuples with zero-padding so (1,30) == (1,30,0)."""
+    n = max(len(a), len(b))
+    a = a + (0,) * (n - len(a))
+    b = b + (0,) * (n - len(b))
+    return (a > b) - (a < b)
+
+
+def _in_range(v: tuple, rng: dict) -> bool:
+    if "ge" in rng and _vcmp(v, _vtuple(rng["ge"])) < 0:
+        return False
+    if "gt" in rng and _vcmp(v, _vtuple(rng["gt"])) <= 0:
+        return False
+    if "lt" in rng and _vcmp(v, _vtuple(rng["lt"])) >= 0:
+        return False
+    if "le" in rng and _vcmp(v, _vtuple(rng["le"])) > 0:
+        return False
+    return True
+
+
+def fingerprint_products(headers: dict) -> list[dict]:
+    """Extract (product, version) pairs from the Server / X-Powered-By banners."""
+    low = _lower_keys(headers)
+    out, seen = [], set()
+    for src in ("server", "x-powered-by"):
+        for m in _FP_RE.finditer(low.get(src, "")):
+            prod, ver = m.group(1).lower(), m.group(2)
+            if (prod, ver) in seen:
+                continue
+            seen.add((prod, ver))
+            out.append({"product": prod, "version": ver, "source": src})
+    return out
+
+
+def match_cves(products: list[dict], feed: list[dict] | None = None) -> list[dict]:
+    """A SIGNAL (not a verdict) per fingerprinted product whose version falls in a
+    known-vulnerable range. A version banner is suggestive only (it can be spoofed or
+    backport-patched), so confidence is 'med' and confirmation is a separate tier."""
+    feed = feed if feed is not None else load_cve_feed()
+    sigs = []
+    for p in products:
+        pv = _vtuple(p.get("version"))
+        if not pv:
+            continue
+        prod = str(p.get("product", "")).lower()
+        for e in feed:
+            names = [str(x).lower() for x in (e.get("match") or [e.get("product", "")])]
+            if prod not in names:
+                continue
+            if not any(_in_range(pv, r) for r in (e.get("affected") or [])):
+                continue
+            sigs.append({
+                "area": "known-cve", "product": p.get("product"), "version": p.get("version"),
+                "source": p.get("source"), "cve": e.get("id"), "aka": e.get("aka", ""),
+                "cvss": e.get("cvss"), "title": e.get("title", ""), "published": e.get("published", ""),
+                "confidence": "med",  # version-fingerprint match, not a confirmed exploit
+                "tier_to_confirm": e.get("confirm", "exploit"),
+                "detail": (f"{p.get('product')}/{p.get('version')} is within the affected range. "
+                           + e.get("note", "")
+                           + " Detection by version fingerprint only; confirmation/exploitation is a separate tiered step."),
+            })
+    return sigs
+
+
+def build_profile(spec: dict, tier: str = "passive") -> dict:
+    """Build the profile by reusing prod_env's passive fetch against the origin only."""
+    t = prod_env.Target(spec, [])
+    out = {
+        "generated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+        "tier": tier, "scope": {"origin": t.origin, "root_domain": t.root_domain},
+        "target": {}, "posture": {}, "signals": [], "coverage": [],
+        "provenance": {"engine": "profile.py", "phase": 0},
+    }
+    r = prod_env._http(t, {"path": "/", "raw_headers": True})
+    if r.get("ok"):
+        hdrs = r.get("raw_headers") or r.get("headers") or {}
+        low = _lower_keys(hdrs)
+        out["target"]["server"] = low.get("server", "")
+        out["target"]["x_powered_by"] = low.get("x-powered-by", "")
+        out["target"]["status"] = r.get("status")
+        posture, gaps = header_posture(hdrs)
+        out["posture"]["headers"] = posture
+        out["posture"]["cookies"] = cookie_posture(low.get("set-cookie", ""))
+        out["signals"].extend(gaps)
+        out["coverage"].extend(["security-headers", "cookie-flags", "fingerprint"])
+        # Version->CVE track (incl. the notable-CVE feed, e.g. PHP2Shell): fingerprint
+        # products from the banners and flag any whose version is in a known range.
+        products = fingerprint_products(hdrs)
+        if products:
+            out["target"]["fingerprints"] = products
+        out["signals"].extend(match_cves(products))
+        out["coverage"].append("version-cve")
+        # Allowed methods (area A): OPTIONS on the root, best-effort (OPTIONS is passive).
+        opt = prod_env._http(t, {"path": "/", "method": "OPTIONS", "raw_headers": True})
+        allow = _lower_keys(opt.get("raw_headers") or {}).get("allow") if opt.get("ok") else None
+        if allow:
+            out["target"]["allow_methods"] = allow
+            out["coverage"].append("http-methods")
+    else:
+        out["target"]["error"] = r.get("error")
+    # Exposed-path sweep (area C): capped GETs with a content-signature gate for precision.
+    sweep_sigs, sweep_pos = exposed_sweep(t)
+    out["signals"].extend(sweep_sigs)
+    if sweep_pos:
+        out["posture"]["hygiene_positives"] = sweep_pos
+    out["coverage"].append("exposed-paths")
+    https = t.origin.lower().startswith("https")
+    out["posture"]["transport"] = {"https": https}
+    out["coverage"].append("transport")
+    if not https:
+        out["signals"].append({"area": "transport", "title": "origin served over plaintext HTTP",
+                               "detail": "no TLS on the origin", "confidence": "high",
+                               "tier_to_confirm": "passive"})
+    return out
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--spec", required=True)
+    ap.add_argument("--out", default="")
+    ap.add_argument("--tier", default="passive")
+    a = ap.parse_args()
+    spec = json.loads(Path(a.spec).read_text())
+    prof = build_profile(spec, a.tier)
+    text = json.dumps(prof, indent=2)
+    if a.out:
+        Path(a.out).write_text(text)
+    print(text)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())