agent-bypass 2.16.0 → 2.18.0

package/scripts/origin_trace.py

@@ -0,0 +1,535 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""origin_trace — 源站溯源（绕 CDN/WAF）：DoH 真 IP + 证书透明度 + 漏质子域 + 直连指纹比对
+
+为什么需要：开了 TUN / fake-IP（iSH 常见 198.18.0.0/15）时，系统 DNS 拿不到真实 IP，
+历史解析、子域、CT 查询都会被污染；本工具统一走 DoH 拿真 IP，再逐个直连比对指纹。
+
+只读：仅做 DNS / HTTPS GET / CT 查询，不修改目标任何状态。授权范围内使用。
+
+用法:
+  python3 origin_trace.py example.com
+  python3 origin_trace.py example.com --json
+  python3 origin_trace.py example.com --max-candidates 60 --timeout 8
+  python3 origin_trace.py example.com --no-ct --max-candidates 8 --budget 60   # iSH/fake-IP：跳过 CT、限时防卡
+  python3 origin_trace.py example.com --ct-sources crt.sh,certspotter,otx     # 指定/兜底 CT 源（all=全部）
+  python3 origin_trace.py example.com --max-ct 120 --ct-budget 20             # CT 主机上限/多源总预算
+  DOH_PROVIDERS="https://1.1.1.1/dns-query" python3 origin_trace.py example.com
+
+输出: 候选源站表（ip / 分数 / 来源 / 标题 / Server / CDN 头），--json 输出机器可读结构。
+退出码: 0 正常（有候选）/ 1 无候选或全部失败 / 2 用法错误。
+"""
+import concurrent.futures
+import hashlib
+import json
+import os
+import re
+import socket
+import ssl
+import sys
+import time
+import urllib.parse
+import urllib.request
+
+try:
+    import netdns
+except Exception:                      # 单文件拷贝场景
+    netdns = None
+
+UA = "Mozilla/5.0 (compatible; origin-trace/1.0; +agent-bypass)"
+SUBS = [
+    "direct", "origin", "origin-www", "mail", "mx", "smtp", "pop", "imap",
+    "webmail", "autodiscover", "autoconfig", "cpanel", "whm", "ftp",
+    "dev", "test", "staging", "stage", "pre", "prod", "beta", "old",
+    "legacy", "backup", "admin", "api", "gateway", "vpn", "oa", "portal",
+    "m", "wap", "static", "img", "cdn", "assets", "uploads",
+]
+ISH = os.path.isdir("/var/minis") or bool(os.environ.get("MINIS_ROOT"))
+
+# 多源证书透明度 / 被动 DNS（无 Key 优先；单个源 502/超时不影响其他源）
+CT_SOURCES = ("crt.sh", "certspotter", "otx", "hackertarget", "anubis", "rapiddns")
+
+CDN_HEADERS = (
+    "cf-ray", "cf-cache-status", "x-cache", "via", "x-amz-cf-id", "x-amz-cf-pop",
+    "x-fastly-request-id", "x-akamai-transformed", "x-cdn", "x-served-by",
+    "x-cache-hits", "x-timer", "server-timing",
+)
+
+
+def _err(msg):
+    print(msg, file=sys.stderr)
+
+
+def is_ipv4(s):
+    if netdns is not None:
+        return netdns.is_ipv4(s)
+    return bool(re.match(r"^\d{1,3}(?:\.\d{1,3}){3}$", str(s)))
+
+
+def doh_resolve(name, timeout=6.0):
+    """DoH 真 IP；无 netdns 时退回系统解析。返回 [(ip, 'doh'|'system')]"""
+    if netdns is not None:
+        ips = netdns.doh_query(name, "A", timeout)
+        if ips:
+            return [(ip, "doh") for ip in ips]
+        ips = netdns.system_resolve(name, timeout)
+        return [(ip, "system") for ip in ips if not netdns.is_fake_ip(ip)]
+    try:
+        return [(socket.gethostbyname(name), "system")]
+    except OSError:
+        return []
+
+
+def _host_in_domain(name, domain):
+    """规范化主机名并校验属于目标域；不合法返回 None。"""
+    name = str(name or "").strip().lower().lstrip("*.").rstrip(".")
+    if name and (name == domain or name.endswith("." + domain)):
+        return name
+    return None
+
+
+def _dedup_hosts(cands, domain, seen, out):
+    for raw in cands:
+        n = _host_in_domain(raw, domain)
+        if n and n not in seen:
+            seen.add(n)
+            out.append(n)
+
+
+def http_get_text(url, timeout=8.0, retry=1):
+    """GET 文本（带 UA + 重试）；失败抛异常由调用方兜底。"""
+    last = None
+    for attempt in range(retry + 1):
+        try:
+            req = urllib.request.Request(url, headers={"user-agent": UA})
+            with urllib.request.urlopen(req, timeout=timeout) as r:
+                return r.read(8 << 20).decode("utf-8", "ignore")
+        except Exception as e:
+            last = e
+            if attempt < retry:
+                time.sleep(0.8)
+    raise last
+
+
+def parse_crtsh(text, domain):
+    """解析 crt.sh JSON → 该域名下的主机名（去通配符、去重、保序）。"""
+    try:
+        rows = json.loads(text)
+    except (ValueError, TypeError):
+        return []
+    if not isinstance(rows, list):
+        return []
+    out, seen = [], set()
+    for row in rows:
+        if isinstance(row, dict):
+            _dedup_hosts(str(row.get("name_value") or "").split("\n"), domain, seen, out)
+    return out
+
+
+def parse_certspotter(text, domain):
+    """CertSpotter issuances JSON → dns_names 里的主机名。"""
+    try:
+        rows = json.loads(text)
+    except (ValueError, TypeError):
+        return []
+    out, seen = [], set()
+    for row in rows if isinstance(rows, list) else []:
+        if isinstance(row, dict):
+            _dedup_hosts(row.get("dns_names") or [], domain, seen, out)
+    return out
+
+
+def parse_otx(text, domain):
+    """OTX 被动 DNS → (主机名列表, IP 列表)。"""
+    try:
+        d = json.loads(text)
+    except (ValueError, TypeError):
+        return [], []
+    rows = d.get("passive_dns") if isinstance(d, dict) else None
+    out, seen, ips = [], set(), []
+    for row in rows if isinstance(rows, list) else []:
+        if not isinstance(row, dict):
+            continue
+        host = _host_in_domain(row.get("hostname") or "", domain)
+        if not host:
+            continue                       # 只收目标域相关记录，避免无关 IP 污染候选
+        _dedup_hosts([host], domain, seen, out)
+        ip = str(row.get("address") or "").strip()
+        if netdns is not None and netdns.is_ipv4(ip) and not netdns.is_fake_ip(ip) and ip not in ips:
+            ips.append(ip)
+    return out, ips
+
+
+def parse_hackertarget(text, domain):
+    """HackerTarget hostsearch CSV（host,ip）→ (主机名, IP)。"""
+    out, seen, ips = [], set(), []
+    for line in str(text or "").splitlines():
+        parts = [x.strip() for x in line.split(",")]
+        if len(parts) != 2:
+            continue
+        host = _host_in_domain(parts[0], domain)
+        if not host:
+            continue                       # 只收目标域相关主机/IP
+        _dedup_hosts([host], domain, seen, out)
+        if netdns is not None and netdns.is_ipv4(parts[1]) and not netdns.is_fake_ip(parts[1]) and parts[1] not in ips:
+            ips.append(parts[1])
+    return out, ips
+
+
+def parse_anubis(text, domain):
+    """Anubis(jldc.me) 子域 JSON 数组 → 主机名。"""
+    try:
+        rows = json.loads(text)
+    except (ValueError, TypeError):
+        return []
+    out, seen = [], set()
+    _dedup_hosts(rows if isinstance(rows, list) else [], domain, seen, out)
+    return out
+
+
+def parse_rapiddns(text, domain):
+    """RapidDNS HTML → 页面中的子域文本（宽松匹配）。"""
+    out, seen = [], set()
+    for m in re.finditer(r"([a-zA-Z0-9][a-zA-Z0-9._-]*\." + re.escape(domain) + r")", str(text or "")):
+        _dedup_hosts([m.group(1)], domain, seen, out)
+    return out
+
+
+def _ct_fetch(name, domain, timeout):
+    """单个 CT/被动 DNS 源 → (hosts, ips)。"""
+    q = urllib.parse.quote(domain, safe="")
+    if name == "crt.sh":
+        text = http_get_text("https://crt.sh/?q=" + urllib.parse.quote("%." + domain, safe="") + "&output=json",
+                             timeout, retry=1)
+        return parse_crtsh(text, domain), []
+    if name == "certspotter":
+        text = http_get_text("https://api.certspotter.com/v1/issuances?domain=%s&include_subdomains=true&expand=dns_names" % q,
+                             timeout)
+        return parse_certspotter(text, domain), []
+    if name == "otx":
+        text = http_get_text("https://otx.alienvault.com/api/v1/indicators/domain/%s/passive_dns" % q, timeout)
+        return parse_otx(text, domain)
+    if name == "hackertarget":
+        text = http_get_text("https://api.hackertarget.com/hostsearch/?q=%s" % q, timeout)
+        return parse_hackertarget(text, domain)
+    if name == "anubis":
+        text = http_get_text("https://jldc.me/anubis/subdomains/%s" % q, timeout)
+        return parse_anubis(text, domain), []
+    if name == "rapiddns":
+        text = http_get_text("https://rapiddns.io/subdomain/%s?full=1" % q, timeout)
+        return parse_rapiddns(text, domain), []
+    return [], []
+
+
+def fetch_passive_names(domain, timeout=8.0, sources=None, max_hosts=80, budget=None):
+    """多源并发查询 CT/被动 DNS：单源失败不影响其他；总预算到期返回部分结果。
+    返回 (hosts, passive_ips, per_source_status)。"""
+    sources = list(sources or CT_SOURCES)
+    budget = budget if budget and budget > 0 else (10.0 if ISH else 15.0)
+    workers = min(4 if ISH else 6, max(1, len(sources)))
+    hosts, seen, ips, status = [], set(), [], {}
+    with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as ex:
+        futs = {ex.submit(_ct_fetch, name, domain, timeout): name for name in sources}
+        try:
+            for fut in concurrent.futures.as_completed(futs, timeout=budget):
+                src = futs[fut]
+                try:
+                    h, ip = fut.result()
+                except Exception as e:
+                    status[src] = "失败: %s" % str(e)[:60]
+                    _err("[CT] %-12s 失败，跳过: %s" % (src, str(e)[:80]))
+                    continue
+                status[src] = len(h) + (len(ip) and len(ip) or 0) * 0  # 仅主机数用于展示
+                for n in h:
+                    if n not in seen and len(hosts) < max_hosts:
+                        seen.add(n)
+                        hosts.append(n)
+                for ipv in ip:
+                    if ipv not in ips:
+                        ips.append(ipv)
+                _err("[CT] %-12s %d 主机%s" % (src, len(h), (" / %d IP" % len(ip)) if ip else ""))
+        except concurrent.futures.TimeoutError:
+            status["_budget"] = "总预算 %.0fs 超时（返回部分）" % budget
+            _err("[CT] 总预算 %.0fs 到期，使用已返回的 %d 个主机" % (budget, len(hosts)))
+        finally:
+            ex.shutdown(wait=False, cancel_futures=True)
+    return hosts, ips, status
+
+
+def cdn_markers(headers):
+    return [h for h in CDN_HEADERS if h in headers]
+
+
+def extract_title(body):
+    m = re.search(r"<title[^>]*>(.*?)</title>", body, re.I | re.S)
+    if not m:
+        return ""
+    t = re.sub(r"\s+", " ", m.group(1)).strip()
+    return t[:120]
+
+
+def parse_response(data):
+    """原始 HTTP 响应 → dict(status/title/server/headers/cdn/location)。"""
+    head, _, body = data.partition(b"\r\n\r\n")
+    lines = head.decode("iso-8859-1", "ignore").split("\r\n")
+    status = lines[0].strip() if lines else ""
+    headers = {}
+    for line in lines[1:]:
+        if ":" in line:
+            k, v = line.split(":", 1)
+            headers[k.strip().lower()] = v.strip()
+    return {
+        "status": status,
+        "title": extract_title(body.decode("utf-8", "ignore")),
+        "server": headers.get("server", ""),
+        "location": headers.get("location", ""),
+        "headers": headers,
+        "cdn": cdn_markers(headers),
+        "body_sha1": hashlib.sha1(body).hexdigest()[:12] if body else "",
+    }
+
+
+def http_probe(ip, domain, timeout=8.0, method="GET"):
+    """直连 IP + 正确 SNI/Host 的 HTTPS 探测（不校验证书，仅采集指纹）。"""
+    ctx = ssl.create_default_context()
+    ctx.check_hostname = False
+    ctx.verify_mode = ssl.CERT_NONE
+    raw = socket.create_connection((ip, 443), timeout=timeout)
+    try:
+        sock = ctx.wrap_socket(raw, server_hostname=domain)   # SSLContext.wrap_socket 不接受 timeout 参数
+        sock.settimeout(timeout)
+        try:
+            cert = sock.getpeercert() or {}
+            sans = [v for k, v in cert.get("subjectAltName", ()) if k == "DNS"]
+            req = ("%s / HTTP/1.1\r\nHost: %s\r\nUser-Agent: %s\r\n"
+                   "Accept: */*\r\nConnection: close\r\n\r\n") % (method, domain, UA)
+            sock.sendall(req.encode("utf-8", "ignore"))
+            data = b""
+            end = time.monotonic() + max(timeout, 2.0)     # 读取总预算，防慢连接挂住
+            while len(data) < 131072 and time.monotonic() < end:
+                try:
+                    chunk = sock.recv(16384)
+                except (socket.timeout, OSError):
+                    break
+                if not chunk:
+                    break
+                data += chunk
+        finally:
+            sock.close()
+    finally:
+        try:
+            raw.close()
+        except Exception:
+            pass
+    info = parse_response(data)
+    info["cert_sans"] = sans
+    info["cert_domain_ok"] = any(
+        domain == x or domain.endswith("." + x.lstrip("*.")) for x in sans
+    )
+    return info
+
+
+def score_candidate(base, cand):
+    """候选 vs 基线指纹打分：同标题/同 Server/无 CDN 头/证书匹配加分。"""
+    score, reasons = 0, []
+    if base.get("title") and cand.get("title") and cand["title"] == base["title"]:
+        score += 3
+        reasons.append("title")
+    if base.get("status") and cand.get("status") == base.get("status"):
+        score += 1
+    if base.get("server") and cand.get("server") == base.get("server"):
+        score += 1
+        reasons.append("server")
+    if base.get("cdn") and not cand.get("cdn"):
+        score += 2
+        reasons.append("no-cdn-headers")
+    if cand.get("cert_domain_ok"):
+        score += 1
+        reasons.append("cert-san")
+    if cand.get("cdn"):
+        if base.get("cdn") and set(cand["cdn"]) & set(base["cdn"]):
+            score -= 2                       # 与基线同一 CDN = 还是 CDN 边缘，不是源站
+            reasons.append("same-cdn")
+        else:
+            score -= 1
+            reasons.append("cdn")
+    return score, reasons
+
+
+def collect_candidates(domain, max_candidates=40, timeout=8.0, use_ct=True,
+                       ct_sources=None, max_ct=80, ct_budget=0.0):
+    """收集候选 (name, ip, sources)：漏质子域 + 多源 CT/被动 DNS。"""
+    names = ["www." + domain, domain] + ["%s.%s" % (s, domain) for s in SUBS]
+    passive_ips = []
+    if use_ct:
+        ct, passive_ips, _status = fetch_passive_names(
+            domain, timeout, sources=ct_sources, max_hosts=max_ct, budget=ct_budget)
+        if ct:
+            _err("[CT] 合计 %d 个主机名" % len(ct))
+            names.extend(ct)
+        if passive_ips:
+            _err("[被动DNS] 合计 %d 个 IP 并入候选" % len(passive_ips))
+    uniq, seen = [], set()
+    for n in names:
+        n = n.strip().lower().rstrip(".")
+        if n and n not in seen and (n == domain or n.endswith("." + domain)):
+            seen.add(n)
+            uniq.append(n)
+    uniq = uniq[:max_candidates]
+    _err("[候选] 解析 %d 个主机（DoH）" % len(uniq))
+
+    def one(name):
+        return name, doh_resolve(name, timeout)
+
+    by_ip = {}
+    with concurrent.futures.ThreadPoolExecutor(max_workers=(4 if ISH else 8)) as ex:
+        for name, pairs in ex.map(one, uniq):
+            for ip, src in pairs:
+                srcs = by_ip.setdefault(ip, {"ips": set(), "sources": set()})
+                srcs["sources"].add(name)
+    for ip in passive_ips:
+        by_ip.setdefault(ip, {"ips": set(), "sources": set()})["sources"].add("passive-dns")
+    return by_ip
+
+
+def main(argv=None):
+    args = list(sys.argv[1:] if argv is None else argv)
+    domains, as_json, no_verify, no_ct = [], False, False, False
+    max_c, timeout, budget = 40, 8.0, 120.0
+    ct_sources, max_ct, ct_budget = None, 80, 0.0
+    i = 0
+    while i < len(args):
+        a = args[i]
+        if a == "--json":
+            as_json = True
+        elif a == "--no-verify":
+            no_verify = True
+        elif a == "--no-ct":
+            no_ct = True
+        elif a in ("--max-candidates", "--timeout", "--budget", "--max-ct", "--ct-budget", "--ct-sources"):
+            if i + 1 >= len(args):
+                _err("%s 缺少取值" % a)
+                return 2
+            try:
+                if a == "--max-candidates":
+                    max_c = max(1, int(args[i + 1]))
+                elif a == "--budget":
+                    budget = max(10.0, float(args[i + 1]))
+                elif a == "--max-ct":
+                    max_ct = max(1, int(args[i + 1]))
+                elif a == "--ct-budget":
+                    ct_budget = max(3.0, float(args[i + 1]))
+                elif a == "--ct-sources":
+                    val = args[i + 1].strip()
+                    if val.lower() == "all":
+                        ct_sources = None
+                    else:
+                        picked = [x.strip() for x in val.split(",") if x.strip()]
+                        if not picked or any(x not in CT_SOURCES for x in picked):
+                            _err("--ct-sources 取值非法（可选: %s 或 all）" % ",".join(CT_SOURCES))
+                            return 2
+                        ct_sources = picked
+                else:
+                    timeout = max(1.0, min(float(args[i + 1]), 30.0))
+            except (TypeError, ValueError):
+                _err("%s 取值非法: %s" % (a, args[i + 1]))
+                return 2
+            i += 1
+        elif a.startswith("-"):
+            _err("未知参数: %s" % a)
+            return 2
+        else:
+            domains.append(a)
+        i += 1
+    if not domains:
+        _err(__doc__)
+        return 2
+
+    rc = 0
+    report = {}
+    for domain in domains:
+        _err("\n[*] %s" % domain)
+        root_ips = doh_resolve(domain, timeout)
+        real_root = [ip for ip, _s in root_ips]
+        if not real_root:
+            _err("[!] %s DoH 解析失败（检查出网/DoH 端点）" % domain)
+            rc = 1
+            continue
+        _err("[根] %s → %s" % (domain, ",".join(real_root)))
+
+        base = None
+        if not no_verify:
+            try:
+                base = http_probe(real_root[0], domain, timeout)
+                _err("[基线] %s | server=%s | cdn=%s | title=%r" % (
+                    base["status"], base["server"], ",".join(base["cdn"]) or "-", base["title"][:40]))
+            except Exception as e:
+                _err("[基线] 探测失败: %s（继续）" % e)
+
+        by_ip = collect_candidates(domain, max_c, timeout, use_ct=not no_ct,
+                                   ct_sources=ct_sources, max_ct=max_ct, ct_budget=ct_budget)
+        for ip in real_root:                       # 根 IP 纳入候选
+            by_ip.setdefault(ip, {"sources": set()})["sources"].add(domain)
+
+        rows = []
+        end = time.monotonic() + budget
+        pending = [(ip, meta) for ip, meta in by_ip.items()]
+
+        def verify(item):
+            ip, meta = item
+            row = {"ip": ip, "sources": sorted(meta["sources"]),
+                   "score": 0, "reasons": [], "cdn": [], "title": "", "server": "",
+                   "status": "", "cert_sans": []}
+            if not no_verify and time.monotonic() < end:
+                try:
+                    cand = http_probe(ip, domain, timeout)
+                    row.update({k: cand.get(k) for k in
+                                ("status", "title", "server", "cdn", "cert_sans")})
+                    row["score"], row["reasons"] = score_candidate(base or {}, cand)
+                except Exception as e:
+                    row["error"] = str(e)[:80]
+            return row
+
+        if no_verify or not pending:
+            rows = [verify(it) for it in pending]
+        else:
+            with concurrent.futures.ThreadPoolExecutor(max_workers=6) as ex:
+                futs = {ex.submit(verify, it): it for it in pending}
+                try:
+                    for f in concurrent.futures.as_completed(futs, timeout=max(5.0, budget)):
+                        try:
+                            rows.append(f.result())
+                        except Exception:
+                            pass
+                except concurrent.futures.TimeoutError:
+                    _err("[预算] 验证超过 %.0fs，先返回已完成的候选" % budget)
+                finally:
+                    ex.shutdown(wait=False, cancel_futures=True)
+        rows.sort(key=lambda r: (-r["score"], r["ip"]))
+        report[domain] = rows
+
+        if as_json:
+            continue
+        print("\n%s 候选源站（分数高=更像源站）" % domain)
+        print("%-16s %5s  %-28s %-10s %s" % ("ip", "score", "sources", "server", "title"))
+        for r in rows[:max_c]:
+            print("%-16s %5d  %-28s %-10s %s%s" % (
+                r["ip"], r["score"], ",".join(r["sources"])[:27] or "-",
+                (r["server"] or "-")[:10], (r["title"] or "")[:40],
+                ("  [cdn:%s]" % ",".join(r["cdn"])) if r["cdn"] else ""))
+        strong = [r for r in rows if r["score"] >= 4 and not r["cdn"]]
+        if strong:
+            print("[结论] 高置信候选（无 CDN 头）: %s" % ", ".join(r["ip"] for r in strong[:5]))
+        elif rows:
+            print("[结论] 无高置信候选：可能源站未暴露/全站 CDN/TUN 干扰（用 --json 看细节）")
+            rc = rc or 1
+        else:
+            rc = 1
+
+    if as_json:
+        print(json.dumps(report, ensure_ascii=False, indent=2))
+    return rc
+
+
+if __name__ == "__main__":
+    sys.exit(main())