@murk-security/runner-darwin-arm64 0.0.0-stage → 0.7.1

package/runners/recon.py

@@ -0,0 +1,396 @@
+#!/usr/bin/env python3
+"""Recon / EASM ops for the Blackglass basic-harness product env (Phase 2).
+
+WAF-INDEPENDENT external attack-surface discovery. These ops never hit the
+target's protected origin, so a CDN/WAF in front of the app cannot block them.
+They query a FIXED allowlist of external data sources (DoH resolvers, crt.sh,
+cloud-storage endpoints, public registries, GitHub) about the customer's
+VERIFIED root domain and org, and are strictly READ-ONLY.
+
+Imported and dispatched by prod_env.py: the harness sees ONE env process whose
+`describe` advertises these ops alongside `http`/`list_identities`. No Go-core
+change -- recon is new vocabulary on the existing `env` tool.
+
+Design constraints inherited from the Go env wiring (env.go):
+  * Each op MUST answer well under the harness's fixed 30s per-call timeout, or
+    the whole env process is killed and restarted. Every network call carries a
+    short timeout; an op returns a partial/timeout result instead of hanging.
+  * One op at a time per process; the prober fans out via `steps` batches and
+    across cells, not inside one call.
+
+Scope carve-out (approved by the operator): dns/crt/tls operate ONLY on the
+verified root domain and its subdomains. storage/registries/code_search query
+the fixed data-source allowlist about org-derived names. code_search is OPT-IN
+and needs BLACKGLASS_GITHUB_TOKEN; it is disabled (noop) otherwise.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import socket
+import ssl
+import urllib.error
+import urllib.request
+from urllib.parse import quote, urlencode, urlparse
+
+# Per-request network timeouts, sized so an op always answers inside the harness's
+# fixed 30s guillotine (env.go step() uses a 30s Do()); a timeout kills+restarts the
+# whole env process, so stay well under it. dns fans out over several DoH calls, so
+# its per-call budget is small; crt.sh is a single slow source, so it gets more.
+NET_TIMEOUT = 8       # single-call ops: tls, storage, registries, code_search, internetdb
+DOH_TIMEOUT = 4       # dns: up to ~8 sequential calls -> worst case ~32s only if DoH is down
+CRT_TIMEOUT = 12      # crt.sh per attempt; up to 2 attempts (crt.sh 502s a lot) -> <30s
+ARCHIVE_TIMEOUT = 12  # Wayback CDX can be slow; stay under the 30s guillotine
+
+# The FIXED external data sources this module may reach. A confined/prod egress
+# jail must allowlist exactly these hosts for recon to work; nothing else here
+# ever makes an outbound call. The prober cannot point recon at arbitrary hosts
+# -- it supplies a subject (domain/bucket/namespace) and we query these.
+DATA_SOURCES = (
+    "dns.google", "crt.sh", "s3.amazonaws.com", "storage.googleapis.com",
+    "hub.docker.com", "registry.npmjs.org", "api.github.com",
+    "internetdb.shodan.io", "web.archive.org",
+)
+
+_UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
+       "(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36")
+
+# Data-source base URLs, overridable via env for a proxied/air-gapped egress path
+# or for a hermetic test harness that mocks them. Defaults are the public services.
+DOH_URL = os.environ.get("RECON_DOH_URL", "https://dns.google/resolve")
+CRT_URL = os.environ.get("RECON_CRT_URL", "https://crt.sh/")
+S3_BASE = os.environ.get("RECON_S3_BASE", "https://s3.amazonaws.com")
+GCS_BASE = os.environ.get("RECON_GCS_BASE", "https://storage.googleapis.com")
+DOCKER_BASE = os.environ.get("RECON_DOCKER_BASE", "https://hub.docker.com")
+NPM_BASE = os.environ.get("RECON_NPM_BASE", "https://registry.npmjs.org")
+INTERNETDB_URL = os.environ.get("RECON_INTERNETDB_URL", "https://internetdb.shodan.io")
+WAYBACK_URL = os.environ.get("RECON_WAYBACK_URL", "https://web.archive.org/cdx/search/cdx")
+
+# CNAME targets that commonly indicate a dangling / takeover-prone record when
+# the fronting resource no longer exists. Presence is a CANDIDATE signal only;
+# the prober should corroborate (e.g. an http fetch showing the provider's
+# "no such bucket/app" page) before reporting a confirmed takeover.
+TAKEOVER_SUFFIXES = (
+    ".s3.amazonaws.com", ".cloudfront.net", ".github.io", ".herokudns.com",
+    ".herokuapp.com", ".azurewebsites.net", ".cloudapp.net", ".trafficmanager.net",
+    ".fastly.net", ".netlify.app", ".ghost.io", ".surge.sh", ".pantheonsite.io",
+    ".readthedocs.io", ".wpengine.com", ".zendesk.com", ".statuspage.io",
+)
+
+
+def _get(url: str, headers: dict | None = None, timeout: int = NET_TIMEOUT,
+         max_bytes: int = 1_000_000):
+    """GET a URL from a fixed data source. Returns (status, body_text, err).
+
+    Reads at most max_bytes to bound memory; crt.sh (big CT JSON) passes a larger cap.
+    """
+    h = {"User-Agent": _UA, "Accept": "application/json, text/plain, */*"}
+    if headers:
+        h.update(headers)
+    req = urllib.request.Request(url, headers=h, method="GET")
+    try:
+        with urllib.request.urlopen(req, timeout=timeout) as resp:
+            return resp.status, resp.read(max_bytes).decode("utf-8", "replace"), None
+    except urllib.error.HTTPError as e:
+        body = e.read(200_000).decode("utf-8", "replace") if hasattr(e, "read") else ""
+        return e.code, body, None
+    except Exception as e:  # noqa: BLE001 - surface to the agent, never crash the env
+        return None, "", str(e)
+
+
+def _get_json(url: str, headers: dict | None = None, timeout: int = NET_TIMEOUT,
+              max_bytes: int = 1_000_000):
+    status, body, err = _get(url, headers, timeout, max_bytes)
+    if err:
+        return None, err
+    try:
+        return json.loads(body), None
+    except json.JSONDecodeError:
+        return None, f"non-JSON response (status {status})"
+
+
+# --- scope ------------------------------------------------------------------
+
+def registrable(host: str) -> str:
+    """Best-effort registrable domain: the last two labels.
+
+    NOTE: this does not consult the Public Suffix List, so multi-label TLDs
+    (co.uk, com.au) resolve to the wrong root. Acceptable for a first cut;
+    revisit with a PSL if customers on such TLDs appear.
+    """
+    host = (host or "").strip(".").lower()
+    labels = host.split(".")
+    return ".".join(labels[-2:]) if len(labels) >= 2 else host
+
+
+def in_domain_scope(name: str, root: str) -> bool:
+    name = (name or "").strip(".").lower()
+    root = (root or "").strip(".").lower()
+    return bool(root) and (name == root or name.endswith("." + root))
+
+
+# --- dns --------------------------------------------------------------------
+
+def _doh(name: str, rtype: str):
+    """Resolve one record type via Google DoH JSON. Returns (status_code, [data])."""
+    url = DOH_URL + "?" + urlencode({"name": name, "type": rtype})
+    data, err = _get_json(url, timeout=DOH_TIMEOUT)
+    if err or not isinstance(data, dict):
+        return None, []
+    answers = [a.get("data", "").strip('"') for a in (data.get("Answer") or [])
+               if a.get("type") is not None]
+    return data.get("Status"), answers
+
+
+def op_dns(args: dict, root: str) -> dict:
+    name = str(args.get("name") or root)
+    if not in_domain_scope(name, root):
+        return {"ok": False, "error": f"dns is scoped to the verified root domain {root!r} and its subdomains; {name!r} is out of scope"}
+    records: dict = {}
+    for rtype in ("A", "AAAA", "CNAME", "MX", "TXT", "NS"):
+        status, answers = _doh(name, rtype)
+        if answers:
+            records[rtype] = answers
+
+    # Dangling / takeover heuristic: a CNAME whose target no longer resolves
+    # (NXDOMAIN) or points at a known takeover-prone provider.
+    dangling = False
+    takeover_candidate = None
+    for cname in records.get("CNAME", []):
+        target = cname.strip(".").lower()
+        a_status, a_ans = _doh(target, "A")
+        if a_status == 3 and not a_ans:  # NXDOMAIN, nothing resolves
+            dangling = True
+        for suffix in TAKEOVER_SUFFIXES:
+            if target.endswith(suffix):
+                takeover_candidate = {"cname": target, "provider": suffix.lstrip(".")}
+    out = {"ok": True, "name": name, "records": records}
+    if dangling:
+        out["dangling"] = True
+    if takeover_candidate:
+        out["takeover_candidate"] = takeover_candidate
+    return out
+
+
+# --- certificate transparency ----------------------------------------------
+
+def op_crt(args: dict, root: str) -> dict:
+    domain = str(args.get("domain") or root)
+    if not in_domain_scope(domain, root):
+        return {"ok": False, "error": f"crt is scoped to the verified root domain {root!r}; {domain!r} is out of scope"}
+    url = CRT_URL + "?" + urlencode({"q": "%." + domain, "output": "json"})
+    # crt.sh frequently returns 502/503 or times out; retry once (budget stays <30s).
+    data = last_err = None
+    for _ in range(2):
+        data, last_err = _get_json(url, timeout=CRT_TIMEOUT, max_bytes=20_000_000)
+        if isinstance(data, list):
+            break
+    if not isinstance(data, list):
+        return {"ok": False, "error": f"crt.sh unavailable (transient; retry later): {last_err}"}
+    hosts: set[str] = set()
+    for row in data:
+        for h in str(row.get("name_value", "")).splitlines():
+            h = h.strip().lower().lstrip("*.")
+            if h and in_domain_scope(h, root):
+                hosts.add(h)
+    ordered = sorted(hosts)[:200]
+    return {"ok": True, "domain": domain, "count": len(ordered), "hostnames": ordered}
+
+
+# --- live TLS certificate ---------------------------------------------------
+
+def op_tls(args: dict, root: str) -> dict:
+    host = str(args.get("host") or root)
+    if not in_domain_scope(host, root):
+        return {"ok": False, "error": f"tls is scoped to the verified root domain {root!r}; {host!r} is out of scope"}
+    port = int(args.get("port") or 443)
+    ctx = ssl.create_default_context()
+    try:
+        with socket.create_connection((host, port), timeout=NET_TIMEOUT) as sock:
+            with ctx.wrap_socket(sock, server_hostname=host) as ss:
+                cert = ss.getpeercert() or {}
+        sans = [v for (k, v) in cert.get("subjectAltName", []) if k.lower() == "dns"]
+        return {"ok": True, "host": host, "verified": True,
+                "subject": dict(x[0] for x in cert.get("subject", [])),
+                "notAfter": cert.get("notAfter"), "sans": sans}
+    except ssl.SSLCertVerificationError as e:
+        return {"ok": True, "host": host, "verified": False, "error": str(e),
+                "note": "cert did not verify against system roots; SANs unavailable without it -- use crt for historical SANs"}
+    except Exception as e:  # noqa: BLE001
+        return {"ok": False, "error": f"tls connect failed: {e}"}
+
+
+# --- cloud storage ----------------------------------------------------------
+
+def op_storage(args: dict, root: str) -> dict:
+    name = str(args.get("name") or "").strip().lower()
+    if not name or "/" in name or " " in name:
+        return {"ok": False, "error": "storage needs a bucket 'name' (derive candidates from the org/domain, e.g. acme, acme-backups, acme-assets)"}
+    results = []
+    # S3: path-style GET lists keys when the bucket allows public read.
+    s3_status, s3_body, s3_err = _get(f"{S3_BASE}/{quote(name)}/?max-keys=5")
+    if s3_err:
+        results.append({"provider": "s3", "error": s3_err})
+    else:
+        public = s3_status == 200 and "<ListBucketResult" in s3_body
+        exists = s3_status in (200, 403)
+        results.append({"provider": "s3", "status": s3_status, "exists": exists,
+                        "public_read": public})
+    # GCS: XML API lists objects when public.
+    g_status, g_body, g_err = _get(f"{GCS_BASE}/{quote(name)}?max-keys=5")
+    if g_err:
+        results.append({"provider": "gcs", "error": g_err})
+    else:
+        public = g_status == 200 and "<ListBucketResult" in g_body
+        exists = g_status in (200, 403)
+        results.append({"provider": "gcs", "status": g_status, "exists": exists,
+                        "public_read": public})
+    any_public = any(r.get("public_read") for r in results)
+    return {"ok": True, "name": name, "results": results, "public_read": any_public}
+
+
+# --- public registries ------------------------------------------------------
+
+def op_registries(args: dict, root: str) -> dict:
+    namespace = str(args.get("namespace") or "").strip().lower()
+    if not namespace:
+        return {"ok": False, "error": "registries needs a 'namespace' (org/user handle derived from the target, e.g. 'acme')"}
+    out: dict = {"ok": True, "namespace": namespace}
+    # Docker Hub public repositories under the namespace.
+    data, err = _get_json(f"{DOCKER_BASE}/v2/repositories/{quote(namespace)}/?page_size=25")
+    if err:
+        out["docker"] = {"error": err}
+    elif isinstance(data, dict):
+        repos = [{"name": r.get("name"), "pulls": r.get("pull_count"), "private": r.get("is_private")}
+                 for r in (data.get("results") or [])]
+        out["docker"] = {"count": data.get("count", len(repos)), "repos": repos}
+    # npm search for packages tied to the org/name.
+    npm, nerr = _get_json(f"{NPM_BASE}/-/v1/search?" + urlencode({"text": namespace, "size": 20}))
+    if nerr:
+        out["npm"] = {"error": nerr}
+    elif isinstance(npm, dict):
+        pkgs = [{"name": o.get("package", {}).get("name"), "version": o.get("package", {}).get("version")}
+                for o in (npm.get("objects") or [])]
+        out["npm"] = {"total": npm.get("total", len(pkgs)), "packages": pkgs[:20]}
+    return out
+
+
+# --- opt-in public-code secret search (token-gated) -------------------------
+
+def op_code_search(args: dict, root: str) -> dict:
+    token = os.environ.get("BLACKGLASS_GITHUB_TOKEN")
+    if not token:
+        return {"ok": True, "enabled": False,
+                "note": "code_search is opt-in and disabled: set BLACKGLASS_GITHUB_TOKEN to search public code for leaked org secrets."}
+    query = str(args.get("query") or root)
+    url = "https://api.github.com/search/code?" + urlencode({"q": query, "per_page": 20})
+    data, err = _get_json(url, headers={"Authorization": f"Bearer {token}",
+                                        "Accept": "application/vnd.github+json"})
+    if err:
+        return {"ok": False, "enabled": True, "error": f"github code search failed: {err}"}
+    items = [{"repo": i.get("repository", {}).get("full_name"), "path": i.get("path"),
+              "url": i.get("html_url")} for i in (data.get("items") or [])] if isinstance(data, dict) else []
+    return {"ok": True, "enabled": True, "query": query,
+            "total": (data or {}).get("total_count", len(items)), "matches": items}
+
+
+# --- passive internet-scan data (Shodan InternetDB) -------------------------
+
+def _looks_ipv4(s: str) -> bool:
+    parts = s.split(".")
+    return len(parts) == 4 and all(p.isdigit() and 0 <= int(p) <= 255 for p in parts)
+
+
+def op_internetdb(args: dict, root: str) -> dict:
+    """Passive lookup of Shodan's free keyless InternetDB for an IP (or a host under
+    the root domain, resolved first). Returns ports/cpes/vulns/tags/hostnames. These
+    are LEADS from third-party scan data -- never target traffic -- to be corroborated,
+    not reported as confirmed."""
+    ip = str(args.get("ip") or "").strip()
+    host = str(args.get("host") or "").strip().lower()
+    if host:
+        if not in_domain_scope(host, root):
+            return {"ok": False, "error": f"internetdb host is scoped to {root!r} and its subdomains; {host!r} is out of scope"}
+        _, answers = _doh(host, "A")
+        ips = [a for a in answers if _looks_ipv4(a)]
+        if not ips:
+            return {"ok": True, "host": host, "found": False, "note": "host has no A record to look up"}
+        ip = ips[0]
+    if not _looks_ipv4(ip):
+        return {"ok": False, "error": "internetdb needs an 'ip' (IPv4) or a 'host' under the root domain"}
+    data, err = _get_json(f"{INTERNETDB_URL}/{quote(ip)}")
+    if err:
+        return {"ok": False, "error": f"internetdb lookup failed: {err}"}
+    if not isinstance(data, dict) or "ports" not in data:
+        return {"ok": True, "ip": ip, "found": False, "note": "no InternetDB data for this IP"}
+    return {"ok": True, "ip": ip, "found": True,
+            "ports": data.get("ports", []), "hostnames": data.get("hostnames", []),
+            "cpes": data.get("cpes", []), "tags": data.get("tags", []),
+            "vulns": data.get("vulns", []),
+            "note": ("ports/cpes/vulns are LEADS from Shodan's internet-wide scan data, NOT "
+                     "confirmed facts about the live target -- corroborate (e.g. fingerprint the "
+                     "running version via http) before reporting; never report a vuln from this alone.")}
+
+
+# --- passive historical URLs (Wayback Machine CDX) --------------------------
+
+def op_archive(args: dict, root: str) -> dict:
+    """Passive pull of a host's historical URLs from the Wayback Machine CDX index
+    (queries the archive, not the target). Surfaces old/dead endpoints, params, and
+    admin paths to PROBE with http -- leads, not findings."""
+    host = str(args.get("host") or args.get("domain") or root).strip().lower()
+    if not in_domain_scope(host, root):
+        return {"ok": False, "error": f"archive is scoped to {root!r} and its subdomains; {host!r} is out of scope"}
+    url = WAYBACK_URL + "?" + urlencode({
+        "url": f"{host}/*", "output": "json",
+        "fl": "original,statuscode,mimetype", "collapse": "urlkey", "limit": "1000"})
+    data, err = _get_json(url, timeout=ARCHIVE_TIMEOUT)
+    if err:
+        return {"ok": False, "error": f"wayback lookup failed: {err}"}
+    rows = data[1:] if isinstance(data, list) and data else []  # row 0 is the header
+    paths: set[str] = set()
+    for r in rows:
+        orig = r[0] if isinstance(r, list) and r else ""
+        pr = urlparse(orig)
+        p = pr.path + (("?" + pr.query) if pr.query else "")
+        if p and p.startswith("/"):
+            paths.add(p)
+    ordered = sorted(paths)[:300]
+    return {"ok": True, "host": host, "count": len(ordered), "paths": ordered,
+            "note": ("historical URLs from the Wayback Machine -- LEADS to probe with http; many "
+                     "may be dead. They are not findings on their own.")}
+
+
+# --- dispatch + describe ----------------------------------------------------
+
+OPS = {
+    "dns": op_dns, "crt": op_crt, "tls": op_tls,
+    "storage": op_storage, "registries": op_registries, "code_search": op_code_search,
+    "internetdb": op_internetdb, "archive": op_archive,
+}
+
+
+def handle(op: str, args: dict, root: str):
+    """Return a result dict for a recon op, or None if op is not a recon op."""
+    fn = OPS.get(op)
+    if fn is None:
+        return None
+    try:
+        return fn(args, root)
+    except Exception as e:  # noqa: BLE001 - never crash the env on a recon op
+        return {"ok": False, "error": f"recon op {op!r} failed: {e}"}
+
+
+def describe_ops(root: str) -> dict:
+    """The recon ops to advertise in the env's describe, keyed by name."""
+    return {
+        "dns": f'{{"name"?: "sub.{root}"}} resolve A/AAAA/CNAME/MX/TXT/NS for a host under {root}; flags dangling CNAMEs and takeover candidates. Defaults to the root.',
+        "crt": f'{{"domain"?: "{root}"}} mine certificate-transparency logs (crt.sh) for subdomains/hostnames of {root} -- surfaces staging/admin/internal hosts leaked in certs.',
+        "tls": f'{{"host": "sub.{root}", "port"?: 443}} read a live TLS cert for a host under {root}: SANs, subject, expiry.',
+        "storage": '{"name": "bucket-name"} probe S3 + GCS for a public cloud-storage bucket (read-only). Derive candidate names from the org/domain.',
+        "registries": '{"namespace": "org"} list public Docker Hub repos + npm packages for an org/handle derived from the target.',
+        "code_search": '{"query"?: "..."} OPT-IN (needs a token): search public code for the org\'s leaked secrets. Returns a disabled note when no token is set.',
+        "internetdb": f'{{"host"?: "sub.{root}", "ip"?: "1.2.3.4"}} passive Shodan InternetDB lookup for a host under {root} (resolved to its IP) or a raw IP: open ports, CPEs, tags, and candidate CVEs. LEADS to corroborate, not confirmed findings.',
+        "archive": f'{{"host"?: "sub.{root}"}} passive Wayback Machine history for a host under {root}: old/dead URLs, params, and admin paths to then probe with http. LEADS, not findings.',
+    }