agent-bypass 2.11.5 → 2.11.6

package/scripts/dirbrute.py

@@ -1,25 +1,92 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 """
-并发目录/文件爆破
+并发目录/文件爆破（双引擎：curl -Z 批量 / py 线程池）
 
 用法:
   python3 dirbrute.py https://target.com
   python3 dirbrute.py https://target.com --wordlist my.txt
   python3 dirbrute.py https://target.com --ext .php,.bak,.zip --mc 200,301,302,403
   python3 dirbrute.py https://target.com --conc 30 --delay 0.1
+  python3 dirbrute.py https://target.com --engine py --no-soft404 --retries 3
 
-输出: jsonl
+引擎:
+  curl  curl -Z 批量并行（默认，最快）；并发由 --batch 控制
+  py    线程池（--conc 精确控制；--delay 为全局总速率限制）
+  auto  --delay>0 或无 curl 时走 py，否则走 curl
+
+两引擎行为一致，jsonl 字段一致: {"url","status","len","location","server","engine"}
+  软404过滤（默认开启，两引擎共用同一套基线）:
+    先用若干随机路径取基线 (code, size)，命中项若 code 与基线相同且
+    |len - 基线len| <= SOFT404_TOL(64) 字节则判定为误报丢弃；--no-soft404 关闭。
+  瞬态失败（连接被拒/超时）默认重试 --retries 次：指数退避 + 逐轮降低并发，
+    仍失败则计入 error 并在 stderr 汇总（不再静默丢结果）。
 """
-import argparse, json, sys, ssl, time
-from concurrent.futures import ThreadPoolExecutor, as_completed
-import urllib.request, urllib.error
+import argsmini as argparse
+import json
+import random
+import re
+import os
+import shutil
+import string
+import sys
+import threading
+import time
+from collections import Counter
+
+
+# ---- 懒加载/popen 优化：ssl/urllib/concurrent import 各耗 1-5s（iSH），仅首次用时载入 ----
+_ssl_m = None
+def _get_ssl():
+    global _ssl_m
+    if _ssl_m is None: import ssl as _s; _ssl_m = _s
+    return _ssl_m
+_url_m = None
+def _get_urllib():
+    global _url_m
+    if _url_m is None: import urllib.request as _r; import urllib.error as _e; _url_m = (_r, _e)
+    return _url_m
+_cf_m = None
+def _get_cf():
+    global _cf_m
+    if _cf_m is None: import concurrent.futures as _c; _cf_m = _c
+    return _cf_m
+def _urljoin(base, url):
+    import urllib.parse as _p
+    return _p.urljoin(base, url)
+_SAFE = frozenset("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789@%_+=:,./-")
+def _shq(s):
+    if s and all(c in _SAFE for c in s): return s
+    return "'%s'" % s.replace("'", "'" + chr(92) + "'" + "'")
+def _run_curl(cargs):
+    p = os.popen(" ".join(_shq(a) for a in cargs) + " 2>/dev/null")
+    out = p.read(); p.close(); return out
+
+def _new_ctx():
+    """不用 create_default_context()：本脚本已关闭证书校验，加载 CA 库纯属浪费时间
+    （实测多花 ~2s 启动，iSH 上尤其明显）。"""
+    ssl = _get_ssl()
+    c = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
+    c.check_hostname = False
+    c.verify_mode = ssl.CERT_NONE
+    return c
+
 
-CTX = ssl.create_default_context()
-CTX.check_hostname = False
-CTX.verify_mode = ssl.CERT_NONE
+_CTX = None
+
+
+def _ctx():
+    global _CTX
+    if _CTX is None:
+        _CTX = _new_ctx()
+    return _CTX
 UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/120 Safari/537.36"
 
+SOFT404_TOL = 64          # 软404 尺寸容差（字节）
+SOFT404_PROBES = 3        # 软404 基线随机路径探测次数
+MAX_BODY = 4 * 1024 * 1024
+BACKOFF = (0.3, 0.8, 1.5, 3.0)
+
 # 内置字典（常用路径/文件）
 DEFAULT = """admin
 login
@@ -207,92 +274,459 @@ assets.html
 """.strip().split("\n")
 
 
-def req(url, timeout=8, no_redirect=True):
-    h = {"User-Agent": UA, "Accept": "*/*"}
-    if no_redirect:
-        class NoRedir(urllib.request.HTTPRedirectHandler):
-            def redirect_request(self, *a, **k):
+# ---------------------------------------------------------------- 通用工具
+def _hget(headers, name, default=""):
+    """大小写不敏感取响应头，headers 可能为 None。"""
+    try:
+        v = headers.get(name)
+        return v if v is not None else default
+    except Exception:
+        return default
+
+
+def _abs_location(url, loc):
+    """把相对 Location 补成绝对 URL，与 curl 的 %{redirect_url} 对齐。"""
+    if not loc:
+        return ""
+    try:
+        return _urljoin(url, loc)
+    except Exception:
+        return loc
+
+
+def _size_from(fp, headers, cap=MAX_BODY):
+    """响应体字节数。优先用 Content-Length（与 curl 的 %{size_download} 对齐），
+    缺失时流式读取计数（最多 cap 字节）。返回 (size, truncated)。"""
+    cl = _hget(headers, "Content-Length", None)
+    if cl is not None:
+        try:
+            n = int(cl)
+            if n >= 0:
+                return n, n > cap
+        except (TypeError, ValueError):
+            pass
+    total = 0
+    try:
+        while total < cap:
+            buf = fp.read(min(65536, cap - total))
+            if not buf:
+                break
+            total += len(buf)
+    except Exception:
+        pass
+    return total, total >= cap
+
+
+_NOREDIR = None
+
+
+def _noredir_cls():
+    """显式抛 HTTPError 的 no-redirect handler（首次用时构造，懒加载 urllib）。"""
+    global _NOREDIR
+    if _NOREDIR is None:
+        _r, _e = _get_urllib()
+
+        class _NoRedir(_r.HTTPRedirectHandler):
+            def redirect_request(self, req, fp, code, msg, headers, newurl):
                 return None
-        op = urllib.request.build_opener(NoRedir)
-    else:
-        op = urllib.request.build_opener()
-    # 关键：context 必须装在 opener 的 handler 上，不能传给 open()
-    https = urllib.request.HTTPSHandler(context=CTX)
-    op.add_handler(https)
+
+            def http_error_302(self, req, fp, code, msg, headers):
+                raise _e.HTTPError(req.full_url, code, msg, headers, fp)
+
+            http_error_301 = http_error_303 = http_error_307 = http_error_308 = http_error_302
+
+        _NOREDIR = _NoRedir
+    return _NOREDIR
+
+
+_OPENERS = {}
+
+
+def _get_opener(no_redirect=True):
+    """opener 全局缓存（SSL context 与 handler 只装一次）。"""
+    key = bool(no_redirect)
+    if key not in _OPENERS:
+        _r, _e = _get_urllib()
+        handlers = [_r.HTTPSHandler(context=_ctx())]
+        if key:
+            handlers.insert(0, _noredir_cls()())
+        _OPENERS[key] = _r.build_opener(*handlers)
+    return _OPENERS[key]
+
+
+class RateLimiter:
+    """全局速率限制：--delay 表示所有 worker 合计的每请求最小间隔
+    （原实现是每个 worker 各自 sleep，20 并发下 0.1s 实际会打出 ~200 req/s）。"""
+
+    def __init__(self, delay):
+        self.delay = float(delay or 0)
+        self._lock = threading.Lock()
+        self._next = 0.0
+
+    def wait(self):
+        if self.delay <= 0:
+            return
+        with self._lock:
+            now = time.monotonic()
+            if self._next < now:
+                self._next = now
+            self._next += self.delay
+            sleep_for = self._next - now
+        if sleep_for > 0:
+            time.sleep(sleep_for)
+
+
+# ---------------------------------------------------------------- py 引擎
+def _probe_py(url, timeout):
+    """单请求探测，返回 (code|None, size, location, server, err)。
+
+    code=None 表示传输层失败（连接被拒/DNS/超时等）。
+    3xx 不跟随重定向，状态码照常返回（见 _NoRedir）。
+    """
     try:
-        r = urllib.request.Request(url, headers=h)
-        with op.open(r, timeout=timeout) as x:
-            return x.status, len(x.read(200000)), dict(x.headers)
-    except urllib.error.HTTPError as e:
-        return e.code, 0, dict(e.headers)
+        _r, _e = _get_urllib()
+        req = _r.Request(url, headers={"User-Agent": UA, "Accept": "*/*"})
+    except ValueError as e:
+        return None, 0, "", "", "URL 非法: %s" % e
+    try:
+        with _get_opener(True).open(req, timeout=timeout) as x:
+            size, _ = _size_from(x, x.headers)
+            return (x.status, size,
+                    _abs_location(url, _hget(x.headers, "Location")),
+                    _hget(x.headers, "Server"), None)
+    except _e.HTTPError as e:
+        try:
+            size, _ = _size_from(e, e.headers)
+        except Exception:
+            size = 0
+        loc = _abs_location(url, _hget(e.headers, "Location"))
+        srv = _hget(e.headers, "Server")
+        try:
+            e.close()
+        except Exception:
+            pass
+        return e.code, size, loc, srv, None
     except Exception as e:
-        # 不再静默吞掉，首次报错打到 stderr 便于排查
-        if not req._warned:
-            print(f"[req 错误] {url}: {type(e).__name__}: {str(e)[:80]}",
-                  file=sys.stderr)
-            req._warned = True
-        return None, 0, {}
+        return None, 0, "", "", "%s: %s" % (type(e).__name__, str(e)[:90])
 
 
-req._warned = False
+def _probe_py_retry(url, timeout, retries):
+    """带退避重试的 py 探测。"""
+    last_err = ""
+    for attempt in range(retries + 1):
+        code, size, loc, srv, err = _probe_py(url, timeout)
+        if code is not None:
+            return code, size, loc, srv, None
+        last_err = err or "未知错误"
+        if attempt < retries:
+            time.sleep(BACKOFF[min(attempt, len(BACKOFF) - 1)])
+    return None, 0, "", "", last_err
+
+
+# ---------------------------------------------------------------- curl 引擎
+def curl_available():
+    return shutil.which("curl") is not None
+
+
+_CURL_HDR_OK = None
+
+
+def _curl_supports_header():
+    """%header{name} 需要 curl >= 7.84。"""
+    global _CURL_HDR_OK
+    if _CURL_HDR_OK is None:
+        _CURL_HDR_OK = False
+        try:
+            out = _run_curl(["curl", "--version"])
+            m = re.search(r"curl\s+(\d+)\.(\d+)", out)
+            if m:
+                _CURL_HDR_OK = (int(m.group(1)), int(m.group(2))) >= (7, 84)
+        except Exception:
+            pass
+    return _CURL_HDR_OK
+
+
+def curl_batch(urls, timeout, parallel, with_server=None):
+    """curl -Z 批量探测。
+
+    返回 {url: (code, size, location, server)}；code=0 表示该 URL 传输失败。
+    with_server=False 时不追加 %header{server}（老版本 curl），此时 server 恒为 ""。
+    """
+    if not urls:
+        return {}
+    if with_server is None:
+        with_server = _curl_supports_header()
+    par = max(1, min(int(parallel), len(urls)))
+    fmt = "%{http_code} %{size_download} %{redirect_url} %{url_effective}"
+    if with_server:
+        fmt += " %header{server}"
+    fmt += "\n"
+    cargs = ["curl", "-sZ", "--parallel-max", str(par),
+             "--connect-timeout", "%g" % min(timeout, 5.0),
+             "-m", "%g" % timeout, "-w", fmt]
+    for u in urls:
+        cargs += ["-o", "/dev/null", u]
+    try:
+        out = _run_curl(cargs)
+    except OSError:
+        return {u: (0, 0, "", "") for u in urls}
+    res = {}
+    nfields = 5 if with_server else 4
+    for line in out.splitlines():
+        parts = line.split(" ", nfields - 1)
+        if len(parts) < 4:
+            continue
+        code = int(parts[0]) if parts[0].isdigit() else 0
+        size = int(parts[1]) if parts[1].isdigit() else 0
+        server = parts[4].strip() if (with_server and len(parts) > 4) else ""
+        # parts[3] 是 url_effective；极少数情况下 curl 会把它做归一化，做一次容错匹配
+        res[parts[3]] = (code, size, parts[2], server)
+    return res
+
+
+def curl_run(urls, timeout, batch, retries, on_result, on_progress=None):
+    """分批发车 + 逐轮降并发重试。返回最终仍失败的 URL 列表。
+
+    每轮按 chunk 分期调用 curl（避免超长命令行撞 ARG_MAX），
+    chunk 内的并发由 --parallel-max 控制。
+    """
+    pending = list(urls)
+    total = len(pending)
+    decided = 0
+    par = batch
+    chunk = max(batch, 200)
+    for attempt in range(retries + 1):
+        failed = []
+        for i in range(0, len(pending), chunk):
+            part = pending[i:i + chunk]
+            res = curl_batch(part, timeout, par)
+            for u in part:
+                got = res.get(u)
+                if got is None or got[0] == 0:
+                    failed.append(u)
+                else:
+                    decided += 1
+                    on_result(u, got)
+            if on_progress:
+                on_progress(decided, total, len(failed), attempt)
+        if not failed or attempt >= retries:
+            return failed
+        time.sleep(BACKOFF[min(attempt, len(BACKOFF) - 1)])
+        par = max(2, min(8, par // 2))
+        print("# 重试第 %d 轮: %d 个失败 (并发降到 %d)"
+              % (attempt + 1, len(failed), min(par, len(failed))), file=sys.stderr)
+        pending = failed
+    return pending
+
+
+# ---------------------------------------------------------------- 主程序
+def _rand_path(n=12):
+    return "".join(random.choices(string.ascii_lowercase + string.digits, k=n))
+
+
+def _parse_mc(ap, txt):
+    mc = set()
+    for x in txt.split(","):
+        x = x.strip()
+        if not x:
+            continue
+        if not x.isdigit():
+            ap.error("--mc 含非法状态码: %r" % x)
+        mc.add(int(x))
+    if not mc:
+        ap.error("--mc 不能为空")
+    return mc
+
+
+def _load_words(ap, path):
+    if not path:
+        return list(DEFAULT)
+    try:
+        with open(path, "r", encoding="utf-8", errors="replace") as f:
+            words = [l.strip() for l in f if l.strip()]
+    except OSError as e:
+        ap.error("读取字典失败 %s: %s" % (path, e))
+    return words
+
+
+def soft404_baseline(engine, base, timeout, retries):
+    """随机路径软404基线探测，返回 (code, size)；全部失败返回 None。"""
+    probes = ["%s/%s" % (base, _rand_path()) for _ in range(SOFT404_PROBES)]
+    cand = []
+    if engine == "curl":
+        res = curl_batch(probes, timeout, len(probes))
+        for u in probes:
+            got = res.get(u)
+            if got and got[0] != 0:
+                cand.append((got[0], got[1]))
+    else:
+        for u in probes:
+            code, size, _loc, _srv, err = _probe_py_retry(u, timeout, retries)
+            if code is not None:
+                cand.append((code, size))
+    if not cand:
+        return None
+    return Counter(cand).most_common(1)[0][0]
 
 
 def main():
-    ap = argparse.ArgumentParser(description="并发目录爆破")
-    ap.add_argument("base")
+    ap = argparse.ArgumentParser(description="并发目录/文件批量爆破")
+    ap.add_argument("base", help="目标，如 https://target.com（缺协议按 http:// 处理）")
     ap.add_argument("--wordlist", help="字典文件（一行一条）")
     ap.add_argument("--ext", default="", help="附加扩展名，如 .php,.bak")
     ap.add_argument("--mc", default="200,204,301,302,307,401,403,405,500",
                     help="关注的状态码")
-    ap.add_argument("--conc", type=int, default=20)
-    ap.add_argument("--delay", type=float, default=0.0, help="每次请求间隔（秒）")
+    ap.add_argument("--conc", type=int, default=20, help="py 引擎并发数（1-512）")
+    ap.add_argument("--delay", type=float, default=0.0, help="全局每请求最小间隔（秒）")
     ap.add_argument("--timeout", type=float, default=8)
+    ap.add_argument("--engine", choices=["auto", "curl", "py"], default="auto",
+                    help="curl=批量并行(快) py=线程池(精细/delay) auto=自动")
+    ap.add_argument("--batch", type=int, default=48, help="curl 引擎每批并行数（1-512）")
+    ap.add_argument("--retries", type=int, default=2,
+                    help="瞬态失败(连接被拒/超时)重试次数，默认 2")
+    ap.add_argument("--no-soft404", action="store_true",
+                    help="关闭软404基线过滤（默认开启，两引擎共用）")
+    ap.add_argument("--force", action="store_true", help="跳过预检（连接失败仍全量跑）")
     a = ap.parse_args()
 
-    words = DEFAULT
-    if a.wordlist:
-        words = [l.strip() for l in open(a.wordlist) if l.strip()]
+    # ---- 参数校验 ----
+    if a.conc < 1:
+        ap.error("--conc 必须 >= 1")
+    if a.batch < 1:
+        ap.error("--batch 必须 >= 1")
+    if a.retries < 0:
+        ap.error("--retries 必须 >= 0")
+    if a.timeout <= 0:
+        ap.error("--timeout 必须 > 0")
+    if a.delay < 0:
+        ap.error("--delay 必须 >= 0")
+    if a.conc > 512:
+        a.conc = 512
+    if a.batch > 512:
+        a.batch = 512
 
-    exts = [e.strip() for e in a.ext.split(",") if e.strip()] if a.ext else [""]
-    paths = []
+    base = a.base.strip()
+    if not base:
+        ap.error("目标不能为空")
+    if not re.match(r"^[A-Za-z][A-Za-z0-9+.\-]*://", base):
+        base = "http://" + base
+        print("# 目标未带协议，按 %s 处理" % base, file=sys.stderr)
+    base = base.rstrip("/")
+
+    mc = _parse_mc(ap, a.mc)
+    words = _load_words(ap, a.wordlist)
+    exts = ([""] + [e.strip() for e in a.ext.split(",") if e.strip()]) if a.ext else [""]
+    paths, seen = [], set()
     for w in words:
         for e in exts:
-            paths.append(w + e)
+            p = (w + e).lstrip("/")
+            if p and p not in seen:
+                seen.add(p)
+                paths.append(p)
+    if not paths:
+        ap.error("字典为空，无探测路径")
+    urls = ["%s/%s" % (base, p) for p in paths]
 
-    base = a.base.rstrip("/")
-    mc = {int(x) for x in a.mc.split(",") if x.strip()}
+    engine = a.engine
+    if engine == "auto":
+        engine = "py" if (a.delay and a.delay > 0) or not curl_available() else "curl"
+    if engine == "curl" and a.delay and a.delay > 0:
+        print("# --delay>0 时 curl 批量引擎无法逐请求限速，自动改用 py 引擎", file=sys.stderr)
+        engine = "py"
+    if engine == "curl" and not curl_available():
+        print("# 未找到 curl，自动改用 py 引擎", file=sys.stderr)
+        engine = "py"
+    if a.conc > 200:
+        print("# 警告: --conc %d 过高，本地/目标可能大量连接被拒（会触发重试）" % a.conc,
+              file=sys.stderr)
 
-    print(f"# {base} 共 {len(paths)} 条 (并发 {a.conc})", file=sys.stderr)
+    soft = not a.no_soft404
+    _purl = "%s/%s" % (base, _rand_path())
+    if engine == "curl":
+        _pg = curl_batch([_purl], min(a.timeout, 6.0), 1).get(_purl)
+        _pcode = _pg[0] if _pg else 0
+    else:
+        _pc = _probe_py(_purl, min(a.timeout, 6.0))[0]
+        _pcode = _pc if _pc is not None else 0
+    if _pcode == 0 and not a.force:
+        print("# [预检失败] %s 不可达，已跳过全量。可达请加 --force。" % base, file=sys.stderr)
+        sys.exit(3)
+    baseline = None
+    if soft:
+        baseline = soft404_baseline(engine, base, a.timeout, a.retries)
+        if baseline is None:
+            soft = False
+            print("# 软404基线探测全部失败，已自动关闭软404过滤", file=sys.stderr)
+        else:
+            print("# 软404基线: %d / %dB (随机路径 x%d)" % (baseline[0], baseline[1], SOFT404_PROBES),
+                  file=sys.stderr)
 
-    hits = []
+    def is_soft(code, size):
+        return bool(soft and baseline and code == baseline[0]
+                    and abs(size - baseline[1]) <= SOFT404_TOL)
 
-    def one(p):
-        if a.delay:
-            time.sleep(a.delay)
-        url = f"{base}/{p}"
-        st, ln, hd = req(url, a.timeout)
-        if st in mc:
-            return {"url": url, "status": st, "len": ln,
-                    "location": hd.get("Location", ""),
-                    "server": hd.get("Server", "")}
-        return None
+    def emit(url, code, size, loc, srv):
+        r = {"url": url, "status": code, "len": size,
+             "location": loc or "", "server": srv or "", "engine": engine}
+        print(json.dumps(r, ensure_ascii=False))
+        return r
+
+    hits, errors = [], []
+    t0 = time.time()
+
+    if engine == "curl":
+        print("# %s 共 %d 条 (curl 引擎, 批 %d, 软404%s)"
+              % (base, len(paths), min(a.batch, len(paths)),
+                 "开启" if soft else "关闭"), file=sys.stderr)
+
+        def on_result(url, got):
+            code, size, loc, srv = got
+            if code in mc and not is_soft(code, size):
+                hits.append(emit(url, code, size, loc, srv))
+
+        def on_progress(done_n, total_n, waiting, attempt):
+            print("  ... 已探测 %d/%d  命中 %d  待重试 %d%s"
+                  % (done_n, total_n, len(hits), waiting,
+                     "" if attempt == 0 else " [第 %d 轮]" % (attempt + 1)),
+                  file=sys.stderr)
+
+        failed = curl_run(urls, a.timeout, a.batch, a.retries, on_result, on_progress)
+        errors.extend([(u, "传输失败(重试后仍失败)") for u in failed])
+        for u, e in errors[:5]:
+            print("# 失败: %s (%s)" % (u, e), file=sys.stderr)
+    else:
+        print("# %s 共 %d 条 (py 引擎, 并发 %d, delay %gs, 软404%s)"
+              % (base, len(paths), a.conc, a.delay, "开启" if soft else "关闭"),
+              file=sys.stderr)
+        limiter = RateLimiter(a.delay)
+        done = [0]
+
+        def one(url):
+            limiter.wait()
+            return url, _probe_py_retry(url, a.timeout, a.retries)
+
+        with _get_cf().ThreadPoolExecutor(max_workers=a.conc) as ex:
+            futs = {ex.submit(one, u): u for u in urls}
+            for fu in _get_cf().as_completed(futs):
+                done[0] += 1
+                try:
+                    url, (code, size, loc, srv, err) = fu.result()
+                except Exception as e:
+                    url, code, size, loc, srv = futs[fu], None, 0, "", ""
+                    err = "%s: %s" % (type(e).__name__, e)
+                if code is None:
+                    errors.append((url, err or "未知错误"))
+                elif code in mc and not is_soft(code, size):
+                    hits.append(emit(url, code, size, loc, srv))
+                if done[0] % 200 == 0 or done[0] == len(urls):
+                    print("  ... %d/%d  命中 %d  错误 %d"
+                          % (done[0], len(urls), len(hits), len(errors)), file=sys.stderr)
+        for u, e in errors[:5]:
+            print("# 失败: %s (%s)" % (u, e), file=sys.stderr)
 
-    done = 0
-    with ThreadPoolExecutor(max_workers=a.conc) as ex:
-        futs = {ex.submit(one, p): p for p in paths}
-        for fu in as_completed(futs):
-            done += 1
-            try:
-                r = fu.result()
-            except Exception:
-                r = None
-            if r:
-                hits.append(r)
-                print(json.dumps(r, ensure_ascii=False))
-            if done % 200 == 0:
-                print(f"  ... {done}/{len(paths)}  命中 {len(hits)}", file=sys.stderr)
-
-    print(f"# 完成: 命中 {len(hits)}", file=sys.stderr)
+    dt = time.time() - t0
+    print("# 完成: 命中 %d, 错误 %d, 共 %d 条, 耗时 %.2fs (%s 引擎)"
+          % (len(hits), len(errors), len(urls), dt, engine), file=sys.stderr)
 
 
 if __name__ == "__main__":