agent-bypass 2.17.0 → 2.20.0

package/scripts/pivot_queries.py

@@ -0,0 +1,321 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""pivot_queries — 资产扩散/同 IP 旁站查询生成器（FOFA / Shodan / Quake / ZoomEye / Hunter / Censys / Netlas）
+
+场景：主站打不动（CDN/WAF/无洞/加固严），用测绘引擎按「同 IP、C 段、同证书、同 favicon、
+同 body、同 title、同 ASN、同端口/指纹」反查同簇资产，找同系统/同模板的兄弟站，先打最弱一环，
+再凭共享凭据/共享 DB/共享存储/反代配置回打主站。
+
+本工具只生成查询与扫描命令模板，不联网（favicon 若给 URL 才下载一次）；只读。
+先确认关系再打：同 IP 不等于同资产（CDN 共享 IP 会误伤无关站点），必须多维交叉 + 授权范围内。
+
+用法:
+  python3 pivot_queries.py --domain example.com --ip 1.2.3.4
+  python3 pivot_queries.py --domain example.com --favicon https://example.com/favicon.ico
+  python3 pivot_queries.py --domain example.com --cert example.com --asn AS13335 --json
+  python3 pivot_queries.py --domain example.com --title "Example" --body "unique-snippet"
+"""
+import base64
+import hashlib
+import json
+import os
+import re
+import sys
+import urllib.request
+
+try:
+    import argsmini as argparse
+except Exception:                     # 直接运行/单测环境兜底
+    import argparse  # type: ignore
+
+UA = "Mozilla/5.0 (compatible; pivot-queries/1.0)"
+ENGINES = ("fofa", "shodan", "quake", "zoomeye", "hunter", "censys", "netlas")
+
+
+# ---------------------------------------------------------------- 哈希
+
+def murmur3_32(data, seed=0):
+    """MurmurHash3 x86 32-bit（与 mmh3.hash 一致，返回 signed int）。"""
+    data = bytes(data)
+    c1, c2 = 0xCC9E2D51, 0x1B873593
+    length = len(data)
+    h1 = seed
+    end = length - (length % 4)
+    for i in range(0, end, 4):
+        k1 = int.from_bytes(data[i:i + 4], "little")
+        k1 = (k1 * c1) & 0xFFFFFFFF
+        k1 = ((k1 << 15) | (k1 >> 17)) & 0xFFFFFFFF
+        k1 = (k1 * c2) & 0xFFFFFFFF
+        h1 ^= k1
+        h1 = ((h1 << 13) | (h1 >> 19)) & 0xFFFFFFFF
+        h1 = (h1 * 5 + 0xE6546B64) & 0xFFFFFFFF
+    tail = data[end:]
+    k1 = 0
+    if len(tail) >= 3:
+        k1 ^= tail[2] << 16
+    if len(tail) >= 2:
+        k1 ^= tail[1] << 8
+    if len(tail) >= 1:
+        k1 ^= tail[0]
+        k1 = (k1 * c1) & 0xFFFFFFFF
+        k1 = ((k1 << 15) | (k1 >> 17)) & 0xFFFFFFFF
+        k1 = (k1 * c2) & 0xFFFFFFFF
+        h1 ^= k1
+    h1 ^= length
+    h1 ^= h1 >> 16
+    h1 = (h1 * 0x85EBCA6B) & 0xFFFFFFFF
+    h1 ^= h1 >> 13
+    h1 = (h1 * 0xC2B2AE35) & 0xFFFFFFFF
+    h1 ^= h1 >> 16
+    return h1 - 0x100000000 if h1 >= 0x80000000 else h1
+
+
+def favicon_hashes(data):
+    """favicon → (FOFA icon_hash, 备用 raw mmh3)。
+
+    FOFA/Shodan 主流口径：mmh3(base64(favicon)) 的 signed 32-bit；
+    部分实现/新版引擎可能用原始字节，故同时给出 raw 值备用。
+    """
+    b64 = base64.b64encode(bytes(data))
+    return murmur3_32(b64), murmur3_32(bytes(data))
+
+
+def fetch_favicon(url, timeout=8.0):
+    req = urllib.request.Request(url, headers={"user-agent": UA})
+    with urllib.request.urlopen(req, timeout=timeout) as r:
+        return r.read(1 << 20)
+
+
+def favicon_from_arg(arg):
+    if re.match(r"^https?://", arg or "", re.I):
+        return fetch_favicon(arg)
+    with open(arg, "rb") as f:
+        return f.read(1 << 20)
+
+
+# ---------------------------------------------------------------- 查询生成
+
+def build_queries(info):
+    """按输入生成各引擎查询（列表），返回 {engine: [query...]}。"""
+    ip = (info.get("ip") or "").strip()
+    cidr = (info.get("cidr") or "").strip()
+    domain = (info.get("domain") or "").strip()
+    cert = (info.get("cert") or "").strip()
+    title = (info.get("title") or "").strip()
+    body = (info.get("body") or "").strip()
+    asn = (info.get("asn") or "").strip().upper().replace("AS", "")
+    port = str(info.get("port") or "").strip()
+    icon = info.get("icon_hash")
+    icon_raw = info.get("icon_hash_raw")
+    out = {}
+
+    fofa = []
+    if domain:
+        fofa.append('domain="%s"' % domain)
+    if ip:
+        fofa.append('ip="%s"' % ip)
+    if cidr:
+        fofa.append('ip="%s"' % cidr)
+    if cert:
+        fofa.append('cert="%s"' % cert)
+    if icon is not None:
+        fofa.append('icon_hash="%s"' % icon)
+    if body:
+        fofa.append('body="%s"' % body)
+    if title:
+        fofa.append('title="%s"' % title)
+    if asn:
+        fofa.append('asn="%s"' % asn)
+    if port:
+        fofa.append('port="%s"' % port)
+    out["fofa"] = fofa
+
+    shodan = []
+    if ip:
+        shodan.append("ip:%s" % ip)
+    if cidr:
+        shodan.append("net:%s" % cidr)
+    if domain:
+        shodan.append("hostname:%s" % domain)
+    if cert:
+        shodan.append('ssl.cert.subject.cn:"%s"' % cert)
+    if icon is not None:
+        shodan.append("http.favicon.hash:%s" % icon)
+    if title:
+        shodan.append('http.title:"%s"' % title)
+    if body:
+        shodan.append('http.html:"%s"' % body)
+    if asn:
+        shodan.append("asn:AS%s" % asn)
+    if port:
+        shodan.append("port:%s" % port)
+    out["shodan"] = shodan
+
+    quake = []
+    if ip:
+        quake.append('ip:"%s"' % ip)
+    if cidr:
+        quake.append('ip:"%s"' % cidr)
+    if domain:
+        quake.append('domain:"%s"' % domain)
+    if cert:
+        quake.append('cert:"%s"' % cert)
+    if title:
+        quake.append('title:"%s"' % title)
+    if body:
+        quake.append('body:"%s"' % body)
+    if icon is not None:
+        quake.append('favicon:"%s"' % icon)
+    if port:
+        quake.append('port:"%s"' % port)
+    out["quake"] = quake
+
+    zoomeye = []
+    if ip:
+        zoomeye.append("ip:%s" % ip)
+    if cidr:
+        zoomeye.append("cidr:%s" % cidr)
+    if cert:
+        zoomeye.append("ssl.cert:%s" % cert)
+    if title:
+        zoomeye.append("title:%s" % title)
+    if icon is not None:
+        zoomeye.append("iconhash:%s" % icon)
+    if port:
+        zoomeye.append("port:%s" % port)
+    out["zoomeye"] = zoomeye
+
+    hunter = []
+    if ip:
+        hunter.append('ip="%s"' % ip)
+    if cidr:
+        hunter.append('ip="%s"' % cidr)
+    if domain:
+        hunter.append('domain="%s"' % domain)
+    if cert:
+        hunter.append('cert="%s"' % cert)
+    if title:
+        hunter.append('web.title="%s"' % title)
+    if body:
+        hunter.append('web.body="%s"' % body)
+    if port:
+        hunter.append('port="%s"' % port)
+    out["hunter"] = hunter
+
+    censys = []
+    if ip:
+        censys.append("ip: %s" % ip)
+    if cidr:
+        censys.append("ip: %s" % cidr)
+    if asn:
+        censys.append("autonomous_system.asn: %s" % asn)
+    if cert:
+        censys.append('services.tls.certificates.leaf_data.subject.common_name: "%s"' % cert)
+    if title:
+        censys.append('services.http.response.html_title: "%s"' % title)
+    out["censys"] = censys
+
+    netlas = []
+    if ip:
+        netlas.append("ip:%s" % ip)
+    if domain:
+        netlas.append("domain:%s" % domain)
+    if cert:
+        netlas.append("cert:%s" % cert)
+    if icon is not None:
+        netlas.append("favicon.hash:%s" % icon)
+    out["netlas"] = netlas
+
+    return out
+
+
+def build_scan_templates(info):
+    """后续扫描/验证命令模板（只读，需操作员自行确认授权）。"""
+    ip = (info.get("ip") or "<IP>").strip()
+    domain = (info.get("domain") or "<DOMAIN>").strip()
+    port = str(info.get("port") or "").strip()
+    pflag = port if port else "-"
+    return [
+        "# 端口与服务（-n 禁解析，避免 fake-IP；-Pn 跳主机发现）",
+        "nmap -n -Pn -sV -p- --min-rate 2000 %s" % ip,
+        "",
+        "# 同 IP 多 vhost 枚举（Host 头换绑，对比状态码/长度/标题）",
+        "for h in $(cat subs.txt); do "
+        "curl -sk -o /dev/null -w \"%%{http_code} %%{size_download} $h\\n\" -H \"Host: $h\" http://%s/; done" % ip,
+        "",
+        "# 指定端口 + SNI 验证（绕过 CDN/fake-IP 直接打真实 IP）",
+        "curl -sk --resolve %s:%s:%s https://%s/" % (domain, pflag, ip, domain),
+        "",
+        "# 指纹/CVE/面板/默认口令（按指纹选模板；先 -severity 降噪）",
+        "nuclei -l urls.txt -tags cve -severity critical,high -stats",
+        "nuclei -u https://%s -tags exposed-panels,default-logins,tech" % domain,
+        "",
+        "# 同簇扩散：把命中的兄弟站加入 urls.txt，凭据/接口交叉复用后回打主站",
+    ]
+
+
+# ---------------------------------------------------------------- 入口
+
+def collect_info(a):
+    info = {
+        "domain": getattr(a, "domain", None),
+        "ip": getattr(a, "ip", None),
+        "cidr": getattr(a, "cidr", None),
+        "cert": getattr(a, "cert", None),
+        "title": getattr(a, "title", None),
+        "body": getattr(a, "body", None),
+        "asn": getattr(a, "asn", None),
+        "port": getattr(a, "port", None),
+    }
+    if getattr(a, "favicon", None):
+        data = favicon_from_arg(a.favicon)
+        h_b64, h_raw = favicon_hashes(data)
+        info["icon_hash"] = h_b64
+        info["icon_hash_raw"] = h_raw
+        info["icon_md5"] = hashlib.md5(data).hexdigest()
+    return info
+
+
+def main(argv=None):
+    ap = argparse.ArgumentParser(description="资产扩散/同 IP 旁站查询生成器（离线，只读）")
+    ap.add_argument("--domain")
+    ap.add_argument("--ip")
+    ap.add_argument("--cidr")
+    ap.add_argument("--cert")
+    ap.add_argument("--title")
+    ap.add_argument("--body")
+    ap.add_argument("--asn", help="AS 号，可带或不带 AS 前缀")
+    ap.add_argument("--port", type=int)
+    ap.add_argument("--favicon", help="favicon URL 或本地文件路径（计算 icon_hash）")
+    ap.add_argument("--json", action="store_true")
+    a = ap.parse_args(argv)
+
+    if not any([a.domain, a.ip, a.cidr, a.cert, a.title, a.body, a.asn, a.favicon]):
+        print("[错误] 至少给一个输入：--domain/--ip/--cidr/--cert/--title/--body/--asn/--favicon", file=sys.stderr)
+        return 2
+    info = collect_info(a)
+    queries = build_queries(info)
+    templates = build_scan_templates(info)
+    payload = {"input": info, "queries": queries, "scan_templates": templates}
+
+    if a.json:
+        print(json.dumps(payload, ensure_ascii=False, indent=2))
+        return 0
+
+    print("# 输入: " + ", ".join("%s=%s" % (k, v) for k, v in info.items() if v not in (None, "")))
+    for engine in ENGINES:
+        qs = [q for q in queries.get(engine, []) if q]
+        if not qs:
+            continue
+        print("\n## %s" % engine.upper())
+        for q in qs:
+            print(q)
+    print("\n## 后续扫描/验证模板")
+    for line in templates:
+        print(line)
+    print("\n# 先多维交叉确认「同簇/同源」再打；CDN 共享 IP 会误伤无关站点；只在授权范围内扩散。")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/tests/test_regressions.py

@@ -355,10 +355,165 @@ class HangProtectionTests(unittest.TestCase):
         self.assertIn("same-cdn", r_cdn)
         self.assertLess(s_cdn, s_org)
 
+    def test_passive_ct_parsers(self):
+        cs = '[{"dns_names":["www.example.com","*.example.com","x.test"]}]'
+        self.assertEqual(self.ot.parse_certspotter(cs, "example.com"), ["www.example.com", "example.com"])
+        otx = ('{"passive_dns":[{"hostname":"mail.example.com","address":"1.2.3.4"},'
+               '{"hostname":"x.test","address":"9.9.9.9"},'
+               '{"hostname":"cdn.example.com","address":"198.18.0.9"}]}')
+        hosts, ips = self.ot.parse_otx(otx, "example.com")
+        self.assertIn("mail.example.com", hosts)
+        self.assertEqual(ips, ["1.2.3.4"], "被动 DNS 的 fake-IP 应被过滤")
+        hosts, ips = self.ot.parse_hackertarget("vpn.example.com,5.6.7.8\nAPI count exceeded", "example.com")
+        self.assertEqual(hosts, ["vpn.example.com"])
+        self.assertEqual(ips, ["5.6.7.8"])
+        self.assertEqual(self.ot.parse_anubis('["a.example.com","b.test"]', "example.com"), ["a.example.com"])
+        self.assertEqual(self.ot.parse_rapiddns("<td>oa.example.com</td><td>x.test</td>", "example.com"),
+                         ["oa.example.com"])
+        for src in ("crt.sh", "certspotter", "otx", "hackertarget", "anubis", "rapiddns"):
+            self.assertIn(src, self.ot.CT_SOURCES)
+
+    def test_ct_multi_source_fallback(self):
+        def fake(name, domain, timeout):
+            if name == "crt.sh":
+                raise RuntimeError("502 Bad Gateway")
+            return (["s1.example.com", "s2.example.com"], ["1.2.3.4"])
+        orig = self.ot._ct_fetch
+        self.ot._ct_fetch = fake
+        try:
+            hosts, ips, status = self.ot.fetch_passive_names(
+                "example.com", sources=["crt.sh", "otx"], budget=5)
+        finally:
+            self.ot._ct_fetch = orig
+        self.assertEqual(hosts, ["s1.example.com", "s2.example.com"])
+        self.assertEqual(ips, ["1.2.3.4"])
+        self.assertIn("失败", status["crt.sh"], "单源 502 应被记录并跳过")
+
     def test_origin_bad_flag_rejected(self):
         r = _tmp_run([sys.executable, str(SCRIPTS / "origin_trace.py"), "--bogus"])
         self.assertEqual(r.returncode, 2)
 
 
+class PivotQueryTests(unittest.TestCase):
+    """搜索引警扩面：murmur3/favicon hash、7 引擎查询生成、CLI JSON（离线）。"""
+
+    @classmethod
+    def setUpClass(cls):
+        sys.path.insert(0, str(SCRIPTS))
+        import pivot_queries
+        cls.pq = pivot_queries
+
+    def test_murmur3_known_vectors(self):
+        self.assertEqual(self.pq.murmur3_32(b""), 0)
+        self.assertEqual(self.pq.murmur3_32(b"foo"), -156908512)
+        self.assertEqual(self.pq.murmur3_32(b"hello"), 613153351)
+
+    def test_favicon_hash_wiring(self):
+        import base64
+        data = b"\x89PNG\r\n\x1a\nfake-favicon-bytes"
+        hb, hr = self.pq.favicon_hashes(data)
+        self.assertEqual(hb, self.pq.murmur3_32(base64.b64encode(data)))
+        self.assertEqual(hr, self.pq.murmur3_32(data))
+
+    def test_build_queries_across_engines(self):
+        info = {"domain": "t.com", "ip": "1.2.3.4", "cidr": "1.2.3.0/24", "cert": "t.com",
+                "title": "Admin", "body": "uniq", "asn": "AS13335", "port": 8443, "icon_hash": -123}
+        q = self.pq.build_queries(info)
+        for eng in ("fofa", "shodan", "quake", "zoomeye", "hunter", "censys", "netlas"):
+            self.assertIn(eng, q)
+        self.assertIn('ip="1.2.3.4"', q["fofa"])
+        self.assertIn('icon_hash="-123"', q["fofa"])
+        self.assertIn("http.favicon.hash:-123", q["shodan"])
+        self.assertIn("net:1.2.3.0/24", q["shodan"])
+        self.assertIn('cert:"t.com"', q["quake"])
+        self.assertIn("cidr:1.2.3.0/24", q["zoomeye"])
+        self.assertIn('web.title="Admin"', q["hunter"])
+
+    def test_cli_json_and_favicon_file(self):
+        with tempfile.TemporaryDirectory() as d:
+            fav = Path(d) / "favicon.ico"
+            data = b"\x00\x01\x02fake"
+            fav.write_bytes(data)
+            r = _tmp_run([sys.executable, str(SCRIPTS / "pivot_queries.py"),
+                          "--domain", "t.com", "--ip", "1.2.3.4",
+                          "--favicon", str(fav), "--json"])
+            self.assertEqual(r.returncode, 0, r.stderr)
+            payload = json.loads(r.stdout)
+            self.assertEqual(payload["input"]["icon_hash"], self.pq.favicon_hashes(data)[0])
+            self.assertTrue(payload["queries"]["fofa"])
+            self.assertTrue(payload["scan_templates"])
+
+    def test_cli_rejects_empty_input(self):
+        r = _tmp_run([sys.executable, str(SCRIPTS / "pivot_queries.py")])
+        self.assertEqual(r.returncode, 2)
+
+
+class EnvChainTests(unittest.TestCase):
+    """多路径攻击：.env 解析/分类/脱敏/链生成（离线）。"""
+
+    SAMPLE = "\n".join([
+        "APP_ENV=production",
+        "DB_HOST=10.0.0.5",
+        "DB_PORT=3306",
+        "DB_USER=appuser",
+        "DB_PASSWORD=SuperSecret1",
+        "REDIS_HOST=10.0.0.6",
+        "REDIS_PASSWORD=redispass",
+        "AWS_ACCESS_KEY_ID=AKIAFAKE123",
+        "AWS_SECRET_ACCESS_KEY=fakesecret",
+        "JWT_SECRET=jwtsecret123",
+        "SMTP_HOST=smtp.example.com",
+        "SSH_USER=deploy",
+    ]) + "\n"
+
+    @classmethod
+    def setUpClass(cls):
+        sys.path.insert(0, str(SCRIPTS))
+        import env_chain
+        cls.ec = env_chain
+
+    def test_parse_env_and_classify(self):
+        env = self.ec.parse_env('export DB_HOST=1.2.3.4\nDB_USER="u"\n# comment\nBADLINE\n')
+        self.assertEqual(env["DB_HOST"], "1.2.3.4")
+        self.assertEqual(env["DB_USER"], "u")
+        self.assertEqual(self.ec.classify("DB_PASSWORD"), "mysql")
+        self.assertEqual(self.ec.classify("REDIS_URL"), "db_url")
+        self.assertEqual(self.ec.classify("AWS_SECRET_ACCESS_KEY"), "aws")
+        self.assertEqual(self.ec.classify("JWT_SECRET"), "secret")
+        self.assertEqual(self.ec.classify("SMTP_PASSWORD"), "smtp")
+        self.assertEqual(self.ec.classify("SSH_PRIVATE_KEY"), "ssh")
+
+    def test_extract_and_chains_masking(self):
+        env = self.ec.parse_env(self.SAMPLE)
+        data = self.ec.extract(env)
+        services = {s["service"] for s in data["services"]}
+        self.assertIn("mysql", services)
+        self.assertIn("redis", services)
+        chains = self.ec.build_chains(data)
+        text = json.dumps(chains, ensure_ascii=False)
+        self.assertNotIn("SuperSecret1", text, "默认必须脱敏")
+        self.assertNotIn("redispass", text)
+        revealed = json.dumps(self.ec.build_chains(data, reveal=True), ensure_ascii=False)
+        self.assertIn("SuperSecret1", revealed)
+        self.assertIn(".env → MySQL", revealed)
+
+    def test_cli_json_roundtrip(self):
+        with tempfile.TemporaryDirectory() as d:
+            f = Path(d) / ".env"
+            f.write_text(self.SAMPLE, encoding="utf-8")
+            r = _tmp_run([sys.executable, str(SCRIPTS / "env_chain.py"), str(f), "--json"])
+            self.assertEqual(r.returncode, 0, r.stderr)
+            payload = json.loads(r.stdout)
+            self.assertIn("chains", payload)
+            self.assertNotIn("SuperSecret1", r.stdout)
+            r2 = _tmp_run([sys.executable, str(SCRIPTS / "env_chain.py"), str(f), "--json", "--reveal"])
+            self.assertEqual(r2.returncode, 0, r2.stderr)
+            self.assertIn("SuperSecret1", r2.stdout)
+            empty = Path(d) / "empty.txt"
+            empty.write_text("# nothing\n", encoding="utf-8")
+            r3 = _tmp_run([sys.executable, str(SCRIPTS / "env_chain.py"), str(empty)])
+            self.assertEqual(r3.returncode, 2)
+
+
 if __name__ == "__main__":
     unittest.main()