agent-bypass 2.16.0 → 2.18.0

package/scripts/portscan.py

@@ -1,19 +1,32 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 """
-并发端口扫描（无需 nmap）
+并发端口扫描（无需 nmap）——iSH / TUN / fake-IP 友好版
 
 提示: 手机环境（iSH/Termux）优先用 fastrecon.py scan —— selectors 并发实测快 5-45 倍
 
+iSH + TUN（Clash/mihomo/sing-box）常见问题：系统 DNS 被劫持成 fake-IP（198.18.0.0/15），
+直接扫域名会得到不可路由假地址 → 全部超时/误判。本脚本默认自动检测并用 DoH 拿真实 IP。
+
 用法:
   python3 portscan.py <host> [--ports 1-65535] [--conc 500] [--timeout 2]
   python3 portscan.py 1.2.3.4 --ports top100
-  python3 portscan.py 1.2.3.4 --ports 22,80,443,3306,6379
+  python3 portscan.py example.com --resolve doh        # 强制 DoH 解析
+  python3 portscan.py example.com --resolve none       # 只扫 IP，不解析
+  python3 portscan.py example.com --allow-fake         # 允许扫 fake-IP（一般不要）
 
-输出: jsonl（每行一个开放端口）
+输出: jsonl（每行一个开放端口，含 ip / host / port / service）
+退出码: 0 正常 / 2 参数错误 / 3 目标解析失败或只剩 fake-IP
 """
 import argsmini as argparse
-import asyncio, json, re, sys
+import asyncio, json, os, re, sys, time
+
+try:                       # 共享 DNS 工具（同目录 scripts/netdns.py）
+    import netdns
+except Exception:          # 单文件拷贝场景：退化为基础 socket 解析
+    netdns = None
+
+ISH = os.path.isdir('/var/minis') or bool(os.environ.get('MINIS_ROOT'))
 
 # 100 端口 = 40 高价值服务 + 60 经典（对齐 nmap top100 覆盖面）
 TOP100 = [21,22,23,25,53,80,110,111,135,139,143,443,445,993,995,1433,1521,1723,
@@ -40,6 +53,11 @@ COMMON = {
     50000:"db2"
 }
 
+FAKE_HINT = ("[fake-IP] 目标解析到 fake-IP（TUN 劫持 DNS），该地址不可信：\n"
+             "  1) 用 DoH 真 IP：python3 portscan.py <真实IP>  或 --resolve doh\n"
+             "  2) 关掉代理的 TUN 全局/DNS 劫持，或把目标加 DIRECT/fake-ip-filter\n"
+             "  3) 溯源同理：先 DoH 拿真 IP，再扫 IP（nmap -n -Pn <ip>）")
+
 
 def parse_ports(s):
     s = (s or "").strip().lower()
@@ -74,6 +92,53 @@ def parse_ports(s):
     return sorted(out)
 
 
+def split_host(raw):
+    """'https://example.com:8443/x' / 'example.com' / '1.2.3.4' → ('example.com', 8443|None)"""
+    s = (raw or "").strip()
+    s = re.sub(r'^[a-zA-Z][a-zA-Z0-9+.-]*://', '', s)
+    s = s.split('/', 1)[0]
+    m = re.match(r'^(\[[^\]]+\]|[^:\s/]+)(?::(\d{1,5}))?$', s)
+    if not m:
+        return None, None
+    host = m.group(1).strip('[]')
+    port = int(m.group(2)) if m.group(2) else None
+    if port is not None and not (1 <= port <= 65535):
+        return None, None
+    return host, port
+
+
+def is_ipv4_literal(s):
+    if netdns is not None:
+        return netdns.is_ipv4(s)
+    return bool(re.match(r'^\d{1,3}(?:\.\d{1,3}){3}$', str(s)))
+
+
+def is_fake_ip(ip):
+    if netdns is not None:
+        return netdns.is_fake_ip(ip)
+    return str(ip).startswith(('198.18.', '198.19.', '240.', '198.51.100.', '203.0.113.'))
+
+
+def resolve_target(host, mode="auto", timeout=6.0):
+    """返回 (ip, source, fake_list)。mode: auto/system/doh/none。"""
+    if is_ipv4_literal(host):
+        return host, "ip", []
+    if mode == "none":
+        return None, "none", []
+    if netdns is not None:
+        ips, source = netdns.resolve_real(host, prefer=("system" if mode == "system" else "doh" if mode == "doh" else "auto"),
+                                          timeout=timeout)
+    else:
+        try:
+            import socket as _s
+            ips, source = [_s.gethostbyname(host)], "system"
+        except Exception:
+            ips, source = [], "fail"
+    fake = [ip for ip in ips if is_fake_ip(ip)]
+    real = [ip for ip in ips if not is_fake_ip(ip)]
+    return (real[0] if real else (ips[0] if ips else None)), source, fake
+
+
 async def probe(host, port, timeout=2.0):
     try:
         r, w = await asyncio.wait_for(
@@ -88,31 +153,45 @@ async def probe(host, port, timeout=2.0):
         return None
 
 
-async def scan(host, ports, conc=500, timeout=2.0):
+async def scan(host, ports, conc=500, timeout=2.0, deadline=None):
+    """deadline（time.monotonic 秒）用于总时限：到点停发新探测，返回已扫到的部分结果。
+    返回 (open_ports, timed_out)。"""
     sem = asyncio.Semaphore(conc)
     open_ports = []
+    timed_out = False
 
     async def one(p):
+        nonlocal timed_out
+        if deadline is not None and time.monotonic() > deadline:
+            timed_out = True
+            return
         async with sem:
             r = await probe(host, p, timeout)
             if r:
                 open_ports.append(r)
 
     await asyncio.gather(*[one(p) for p in ports])
-    return sorted(open_ports)
+    return sorted(open_ports), timed_out
 
 
 def main():
-    ap = argparse.ArgumentParser(description="并发端口扫描（无需 nmap）")
+    ap = argparse.ArgumentParser(description="并发端口扫描（无需 nmap；自动绕 fake-IP）")
     ap.add_argument("host")
     ap.add_argument("--ports", default="top20",
                     help="1-65535 / top100 / top20 / 22,80,443")
     ap.add_argument("--conc", type=int, default=500, help="并发数")
     ap.add_argument("--timeout", type=float, default=2.0)
     ap.add_argument("--json", action="store_true", help="只输出 JSONL")
+    ap.add_argument("--resolve", choices=["auto", "system", "doh", "none"], default="auto",
+                    help="DNS 解析方式: auto(系统→DoH) / system / doh / none(只接受 IP)")
+    ap.add_argument("--allow-fake", action="store_true",
+                    help="允许扫描 fake-IP 地址（默认拒绝，防误判）")
+    ap.add_argument("--deadline", type=float, default=0,
+                    help="总时限秒（0=自动：按端口数/并发估算，最长 600s；到点返回部分结果）")
     a = ap.parse_args()
 
-    if not a.host or re.search(r"[\s/]", a.host):
+    name, inline_port = split_host(a.host)
+    if not name:
         print("[错误] host 只接受域名或 IP（可带 :端口）", file=sys.stderr)
         sys.exit(2)
     try:
@@ -124,20 +203,47 @@ def main():
     if conc != a.conc:
         print(f"[提示] 并发钳制为 {conc}（允许 1-2000）", file=sys.stderr)
     timeout = max(0.1, min(a.timeout, 30.0))
-    if timeout != a.timeout:
-        print(f"[提示] 超时钳制为 {timeout}s（允许 0.1-30）", file=sys.stderr)
+    if ISH and conc > 200:
+        conc = 200
+        print("[提示] iSH 环境并发降至 200（CPU 模拟 + 弱网更稳）", file=sys.stderr)
+    if ISH and timeout <= 2.0:
+        timeout = 3.0
+        print("[提示] iSH 环境超时上调至 3.0s", file=sys.stderr)
+    if inline_port and inline_port not in ports:
+        ports = sorted(set(ports) | {inline_port})
+
+    ip, source, fake = resolve_target(name, mode=a.resolve, timeout=max(timeout, 6.0))
+    if not ip:
+        print(f"[错误] 无法解析 {name}（--resolve {a.resolve}）", file=sys.stderr)
+        print(FAKE_HINT, file=sys.stderr)
+        sys.exit(3)
+    if fake and not any(not is_fake_ip(x) for x in [ip]):
+        if not a.allow_fake:
+            print(f"[错误] {name} 只解析到 fake-IP：{','.join(fake)}", file=sys.stderr)
+            print(FAKE_HINT, file=sys.stderr)
+            sys.exit(3)
+        print(f"[警告] 正在扫描 fake-IP（--allow-fake）：{ip}（结果不可信）", file=sys.stderr)
+    elif source == "doh":
+        print(f"[提示] 系统解析为 fake/IP 不可信，已用 DoH 真 IP: {ip}", file=sys.stderr)
+
     if not a.json:
-        print(f"# 扫描 {a.host} 共 {len(ports)} 端口 (并发 {conc})",
+        print(f"# 扫描 {name} → {ip}（{source}）共 {len(ports)} 端口 (并发 {conc})",
               file=sys.stderr)
 
-    open_ports = asyncio.run(scan(a.host, ports, conc, timeout))
+    overall = a.deadline if a.deadline > 0 else min(600.0, max(20.0, len(ports) * timeout / conc * 3 + 10))
+    deadline = time.monotonic() + max(5.0, min(overall, 3600.0))
+    open_ports, timed_out = asyncio.run(scan(ip, ports, conc, timeout, deadline))
+    if timed_out:
+        print(f"# 达到总时限 {overall:.0f}s：返回部分结果（可加大 --deadline 或降低端口数）", file=sys.stderr)
 
     for p in open_ports:
-        rec = {"host": a.host, "port": p, "service": COMMON.get(p, "?")}
+        rec = {"host": name, "ip": ip, "port": p, "service": COMMON.get(p, "?")}
         print(json.dumps(rec, ensure_ascii=False))
 
     if not a.json:
         print(f"# 开放 {len(open_ports)} 个: {open_ports}", file=sys.stderr)
+        if ISH:
+            print("# iSH 提示：并发已限制、超时已上调；长扫描用 --deadline 控制，配合 nohup 后台跑", file=sys.stderr)
 
 
 if __name__ == "__main__":

package/tests/test_regressions.py

@@ -250,5 +250,149 @@ class DoctorTests(unittest.TestCase):
             self.assertEqual(list(Path(home).rglob("*")), [], "doctor 不应写文件")
 
 
+class NetDnsScanOriginTests(unittest.TestCase):
+    """fake-IP/TUN：netdns 判定 + portscan 解析守卫 + origin_trace 解析（全部离线）。"""
+
+    @classmethod
+    def setUpClass(cls):
+        sys.path.insert(0, str(SCRIPTS))
+        import netdns
+        import portscan
+        import origin_trace
+        cls.nd = netdns
+        cls.ps = portscan
+        cls.ot = origin_trace
+
+    def test_fake_ip_ranges(self):
+        for ip in ("198.18.0.5", "198.19.255.1", "240.1.2.3", "198.51.100.7", "203.0.113.9"):
+            self.assertTrue(self.nd.is_fake_ip(ip), ip)
+        for ip in ("1.1.1.1", "8.8.8.8", "192.168.1.1", "10.0.0.1"):
+            self.assertFalse(self.nd.is_fake_ip(ip), ip)
+
+    def test_parse_dns_json(self):
+        sample = ('{"Answer":[{"type":1,"data":"93.184.216.34"},'
+                  '{"type":28,"data":"2606:2800::1"},{"type":1,"data":"198.18.0.1"}]}')
+        self.assertEqual(self.nd._parse_dns_json(sample, "A"), ["93.184.216.34", "198.18.0.1"])
+        self.assertEqual(self.nd._parse_dns_json(sample, "AAAA"), ["2606:2800::1"])
+        self.assertEqual(self.nd._parse_dns_json("not-json", "A"), [])
+
+    def test_portscan_split_host_fake_guard(self):
+        self.assertEqual(self.ps.split_host("https://example.com:8443/x"), ("example.com", 8443))
+        self.assertEqual(self.ps.split_host("1.2.3.4"), ("1.2.3.4", None))
+        self.assertIsNone(self.ps.split_host("bad host")[0])
+        self.assertTrue(self.ps.is_fake_ip("198.18.1.2"))
+        ip, src, fake = self.ps.resolve_target("198.18.1.2", mode="auto")
+        self.assertEqual((ip, src), ("198.18.1.2", "ip"))
+        self.assertEqual(fake, [])
+
+    def test_origin_trace_parsers(self):
+        ct = '[{"name_value":"a.example.com\\n*.example.com\\nother.test"}]'
+        self.assertEqual(self.ot.parse_crtsh(ct, "example.com"), ["a.example.com", "example.com"])
+        self.assertEqual(self.ot.extract_title("<title>  Hi   There </title>"), "Hi There")
+        self.assertEqual(self.ot.cdn_markers({"cf-ray": "x", "server": "nginx"}), ["cf-ray"])
+        base = {"title": "X", "server": "nginx", "status": "HTTP/1.1 200 OK", "cdn": ["cf-ray"]}
+        cand = {"title": "X", "server": "nginx", "status": "HTTP/1.1 200 OK", "cdn": [], "cert_domain_ok": True}
+        score, reasons = self.ot.score_candidate(base, cand)
+        self.assertGreaterEqual(score, 5)
+        self.assertIn("title", reasons)
+        self.assertLess(self.ot.score_candidate(base, {"cdn": ["cf-ray"]})[0], score)
+
+    def test_doh_shell_syntax(self):
+        sh = shutil.which("sh")
+        if not sh:
+            self.skipTest("no sh")
+        r = _tmp_run([sh, "-n", str(SCRIPTS / "doh_resolve.sh")])
+        self.assertEqual(r.returncode, 0, r.stderr)
+
+
+class SkillCompactTests(unittest.TestCase):
+    """SKILL.md 体积/去重回归：防止再次膨胀。"""
+
+    def test_skill_structure_and_size(self):
+        skill = (ROOT / "SKILL.md").read_text(encoding="utf-8")
+        self.assertEqual(skill.count("<!-- module:"), 35)
+        self.assertNotIn("## 修正对照", skill, "重复的修正对照表已移除，勿再加回")
+        self.assertLess(len(skill.encode("utf-8")), 132000, "SKILL.md 体积超限（保持精简，长内容放 references/）")
+
+
+class HangProtectionTests(unittest.TestCase):
+    """防卡死：DNS 硬超时、扫描总时限、溯源预算/同名 CDN 扣分（全部离线）。"""
+
+    @classmethod
+    def setUpClass(cls):
+        sys.path.insert(0, str(SCRIPTS))
+        import netdns, portscan, origin_trace
+        cls.nd = netdns
+        cls.ps = portscan
+        cls.ot = origin_trace
+
+    def test_system_resolve_hard_timeout(self):
+        import time as _t
+        orig = self.nd._system_resolve_impl
+        self.nd._system_resolve_impl = lambda name: (_t.sleep(3), ["1.2.3.4"])[1]
+        t0 = _t.monotonic()
+        try:
+            ips = self.nd.system_resolve("example.com", timeout=0.2)
+        finally:
+            self.nd._system_resolve_impl = orig
+        self.assertEqual(ips, [])
+        self.assertLess(_t.monotonic() - t0, 1.5, "system_resolve 必须有硬超时，不能挂住")
+
+    def test_portscan_total_deadline(self):
+        import asyncio, time as _t
+        found, timed_out = asyncio.run(
+            self.ps.scan("127.0.0.1", [9, 10], conc=1, timeout=0.05,
+                         deadline=_t.monotonic() - 1))
+        self.assertEqual(found, [])
+        self.assertTrue(timed_out)
+
+    def test_origin_same_cdn_penalty(self):
+        base = {"title": "X", "server": "nginx", "status": "HTTP/1.1 200 OK", "cdn": ["cf-ray"]}
+        cand_cdn = {"title": "X", "server": "nginx", "status": "HTTP/1.1 200 OK", "cdn": ["cf-ray"]}
+        cand_origin = {"title": "X", "server": "nginx", "status": "HTTP/1.1 200 OK", "cdn": []}
+        s_cdn, r_cdn = self.ot.score_candidate(base, cand_cdn)
+        s_org, r_org = self.ot.score_candidate(base, cand_origin)
+        self.assertIn("same-cdn", r_cdn)
+        self.assertLess(s_cdn, s_org)
+
+    def test_passive_ct_parsers(self):
+        cs = '[{"dns_names":["www.example.com","*.example.com","x.test"]}]'
+        self.assertEqual(self.ot.parse_certspotter(cs, "example.com"), ["www.example.com", "example.com"])
+        otx = ('{"passive_dns":[{"hostname":"mail.example.com","address":"1.2.3.4"},'
+               '{"hostname":"x.test","address":"9.9.9.9"},'
+               '{"hostname":"cdn.example.com","address":"198.18.0.9"}]}')
+        hosts, ips = self.ot.parse_otx(otx, "example.com")
+        self.assertIn("mail.example.com", hosts)
+        self.assertEqual(ips, ["1.2.3.4"], "被动 DNS 的 fake-IP 应被过滤")
+        hosts, ips = self.ot.parse_hackertarget("vpn.example.com,5.6.7.8\nAPI count exceeded", "example.com")
+        self.assertEqual(hosts, ["vpn.example.com"])
+        self.assertEqual(ips, ["5.6.7.8"])
+        self.assertEqual(self.ot.parse_anubis('["a.example.com","b.test"]', "example.com"), ["a.example.com"])
+        self.assertEqual(self.ot.parse_rapiddns("<td>oa.example.com</td><td>x.test</td>", "example.com"),
+                         ["oa.example.com"])
+        for src in ("crt.sh", "certspotter", "otx", "hackertarget", "anubis", "rapiddns"):
+            self.assertIn(src, self.ot.CT_SOURCES)
+
+    def test_ct_multi_source_fallback(self):
+        def fake(name, domain, timeout):
+            if name == "crt.sh":
+                raise RuntimeError("502 Bad Gateway")
+            return (["s1.example.com", "s2.example.com"], ["1.2.3.4"])
+        orig = self.ot._ct_fetch
+        self.ot._ct_fetch = fake
+        try:
+            hosts, ips, status = self.ot.fetch_passive_names(
+                "example.com", sources=["crt.sh", "otx"], budget=5)
+        finally:
+            self.ot._ct_fetch = orig
+        self.assertEqual(hosts, ["s1.example.com", "s2.example.com"])
+        self.assertEqual(ips, ["1.2.3.4"])
+        self.assertIn("失败", status["crt.sh"], "单源 502 应被记录并跳过")
+
+    def test_origin_bad_flag_rejected(self):
+        r = _tmp_run([sys.executable, str(SCRIPTS / "origin_trace.py"), "--bogus"])
+        self.assertEqual(r.returncode, 2)
+
+
 if __name__ == "__main__":
     unittest.main()