argo-search 1.0.1 → 2.9.8

package/scripts/article.py

@@ -0,0 +1,136 @@
+#!/usr/bin/env python3
+"""article.py — 微信公众号文章全文抓取。
+
+用法: argo article <url> [--json]
+      argo article <url> --json > article.json
+
+绕过 robots.txt 限制：公众号 robots 禁爬，但文章页对真实浏览器开放，
+用手机 UA 直连（curl/urllib 不遵守 robots），解析 js_content 提取正文。
+
+输出: 标题 / 作者 / 发布时间 / 正文纯文本 / 图片 URL 列表
+"""
+import argparse, json, re, sys, html as htmllib, urllib.request
+from cli_io import dumps
+
+UA = ("Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) "
+      "AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1")
+
+HEADERS = {
+    "User-Agent": UA,
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
+    "Referer": "https://mp.weixin.qq.com/",
+}
+
+
+def fetch(url: str, timeout: int = 30) -> str:
+    req = urllib.request.Request(url, headers=HEADERS)
+    # 起点域名白名单挡不住服务端 302：重定向的每一跳都要过 SSRF 校验
+    # （http/https scheme + 非本机/内网目标），被拒即抛错而非跟随
+    opener = urllib.request.build_opener(_SafeRedirectHandler)
+    with opener.open(req, timeout=timeout) as r:
+        return r.read().decode("utf-8", errors="ignore")
+
+
+class _SafeRedirectHandler(urllib.request.HTTPRedirectHandler):
+    """每跳重定向目标过 url_safety.check_url，拦截内网/非 http(s) 目标。"""
+
+    def redirect_request(self, req, fp, code, msg, headers, newurl):
+        from url_safety import check_url
+        ok, reason = check_url(newurl)
+        if not ok:
+            raise ValueError(f"重定向目标被 SSRF 防护拦截（{reason}）: {newurl}")
+        return super().redirect_request(req, fp, code, msg, headers, newurl)
+
+
+def strip_tags(s: str) -> str:
+    s = re.sub(r"<script[\s\S]*?</script>|<style[\s\S]*?</style>", "", s)
+    s = re.sub(r"<br\s*/?>", "\n", s)
+    s = re.sub(r"</(p|div|section|h\d|li|tr|blockquote)>", "\n", s)
+    s = re.sub(r"<[^>]+>", "", s)
+    s = htmllib.unescape(s)
+    lines = [l.strip() for l in s.split("\n")]
+    return "\n".join(l for l in lines if l)
+
+
+def fetch_article(url: str, timeout: int = 30) -> dict:
+    """抓取并解析公众号文章，返回结构化 dict（CLI 与 MCP 共用主链）。
+
+    URL 非 mp.weixin.qq.com 时抛 ValueError（CLI 走 stderr 退出，
+    MCP 转 isError）；抓取失败/反爬拦截返回 {"ok": False, "error": ...}。
+    """
+    if not re.match(r"https?://mp\.weixin\.qq\.com/", url):
+        raise ValueError("仅支持 mp.weixin.qq.com 链接")
+
+    try:
+        raw = fetch(url, timeout)
+    except Exception as e:
+        return {"ok": False, "error": f"抓取失败: {e}"}
+
+    if "js_content" not in raw and "环境异常" in raw:
+        return {"ok": False, "error": "微信反爬拦截（环境异常），请稍后重试"}
+
+    def grab(pattern, default=""):
+        m = re.search(pattern, raw)
+        return m.group(1).strip() if m else default
+
+    title = grab(r'<meta property="og:title" content="([^"]+)"') or \
+        grab(r'<h1[^>]*class="rich_media_title"[^>]*>([\s\S]*?)</h1>')
+    author = grab(r'<meta property="og:article:author" content="([^"]+)"') or \
+        grab(r'id="js_name"[^>]*>([^<]+)<')
+    pubtime = grab(r'id="publish_time"[^>]*>([^<]+)<')
+
+    m = re.search(r'id="js_content"[^>]*>([\s\S]*?)</div>\s*<script', raw)
+    if not m:
+        m = re.search(r'id="js_content"[^>]*>([\s\S]*?)</div>', raw)
+    body_html = m.group(1) if m else ""
+
+    text = strip_tags(body_html)
+    imgs = re.findall(r'data-src="(https://mmbiz\.qpic\.cn/[^"]+)"', body_html)
+    if not imgs:
+        imgs = re.findall(r'src="(https://mmbiz\.qpic\.cn/[^"]+)"', body_html)
+
+    return {
+        "ok": True,
+        "url": url,
+        "title": title,
+        "author": author,
+        "publish_time": pubtime,
+        "char_count": len(text),
+        "image_count": len(imgs),
+        "images": imgs,
+        "content": text,
+    }
+
+
+def main():
+    ap = argparse.ArgumentParser(description="微信公众号文章全文抓取")
+    ap.add_argument("url", help="mp.weixin.qq.com 文章链接")
+    ap.add_argument("--json", action="store_true", help="输出 JSON")
+    args = ap.parse_args()
+
+    try:
+        out = fetch_article(args.url)
+    except ValueError as e:
+        print(str(e), file=sys.stderr)
+        sys.exit(1)
+    if not out["ok"]:
+        print(dumps(out))
+        sys.exit(1)
+
+    if args.json:
+        print(dumps(out))
+    else:
+        print(f"标题: {out['title']}")
+        print(f"作者: {out['author']}  时间: {out['publish_time']}  "
+              f"正文 {out['char_count']} 字 / {out['image_count']} 图")
+        print("---")
+        print(out["content"])
+        if out["images"]:
+            print("---")
+            for i, u in enumerate(out["images"], 1):
+                print(f"[图{i}] {u}")
+
+
+if __name__ == "__main__":
+    main()

package/scripts/article_meta.py

@@ -0,0 +1,131 @@
+#!/usr/bin/env python3
+"""article_meta.py — 从已抓取的 HTML 里提取文章级元数据与正文链接。
+
+为什么要单独一层（2026-10-22，对标 open-websearch 的 includeLinks/byline）：
+`readability_extract` 只交付正文文本，链接、作者、站点名、摘要这些**结构化
+线索**在提取时被丢掉。Agent 要顺着文章里的引用继续取证，只能再抓一次页面；
+研究/报告场景里作者与站点名也是引用格式的必需字段。它们都在同一份 HTML 里，
+一次解析就能拿到，不该让调用方付第二次网络往返。
+
+边界：
+  - 纯标准库 HTMLParser，毫秒级；不做正文判定（那是 readability 的职责）。
+  - 链接只取 <article>/<main> 内。没有语义容器时宁可不给，也不把
+    nav/header/footer 的导航链接当正文引用；相对链接按页面 URL 解析。
+  - 标题/摘要/站点名只读 meta，不从正文猜；猜错比缺失更糟。
+  - 全部字段都可能缺失，调用方按 key 是否存在判断，不返回空字符串占位。
+"""
+
+from __future__ import annotations
+
+from html.parser import HTMLParser
+from urllib.parse import urljoin, urlparse
+
+MAX_LINKS = 30
+MAX_TEXT = 300
+_SKIP_TAGS = {"nav", "header", "footer", "aside", "script", "style", "noscript"}
+_META_KEYS = {
+    "author": "byline",
+    "article:author": "byline",
+    "og:article:author": "byline",
+    "og:site_name": "site_name",
+    "application-name": "site_name",
+    "og:description": "excerpt",
+    "description": "excerpt",
+    "twitter:description": "excerpt",
+    "article:published_time": "published_at",
+}
+
+
+class _MetaParser(HTMLParser):
+    def __init__(self) -> None:
+        super().__init__(convert_charrefs=True)
+        self.meta: dict[str, str] = {}
+        self.article_links: list[dict[str, str]] = []
+        self._in_article = 0
+        self._skip = 0
+        self._href: str | None = None
+        self._text: list[str] = []
+
+    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        tag = tag.lower()
+        if tag == "meta":
+            data = {k.lower(): (v or "") for k, v in attrs if k}
+            key = (data.get("property") or data.get("name") or "").strip().lower()
+            value = data.get("content") or ""
+            field = _META_KEYS.get(key)
+            if field and value and field not in self.meta:
+                self.meta[field] = value.strip()[:MAX_TEXT]
+            return
+        if tag in _SKIP_TAGS:
+            self._skip += 1
+            return
+        if tag in ("article", "main"):
+            self._in_article += 1
+        elif tag == "a" and self._skip == 0:
+            href = ""
+            for k, v in attrs:
+                if k.lower() == "href" and v:
+                    href = v.strip()
+                    break
+            if href:
+                self._href = href
+                self._text = []
+
+    def handle_endtag(self, tag: str) -> None:
+        tag = tag.lower()
+        if tag in _SKIP_TAGS:
+            self._skip = max(0, self._skip - 1)
+            return
+        if tag in ("article", "main"):
+            self._in_article = max(0, self._in_article - 1)
+        elif tag == "a" and self._href is not None:
+            text = " ".join("".join(self._text).split())[:120]
+            if self._in_article:
+                self.article_links.append({"url": self._href, "text": text})
+            self._href = None
+            self._text = []
+
+    def handle_data(self, data: str) -> None:
+        if self._href is not None and self._skip == 0:
+            self._text.append(data)
+
+
+def extract_article_meta(html: str, base_url: str,
+                         max_links: int = MAX_LINKS) -> dict:
+    """返回 {byline, excerpt, site_name, published_at, links}，缺失的键不出现。
+
+    links: [{"url": 绝对 URL, "text": 锚文本}]，按出现顺序去重，最多 max_links。
+    同页锚点、javascript/mailto、非 http(s)、空文本一律丢弃。
+    """
+    if not html:
+        return {}
+    p = _MetaParser()
+    try:
+        p.feed(html)
+        p.close()
+    except Exception:
+        return {}
+    out: dict = dict(p.meta)
+    raw_links = p.article_links
+    if raw_links:
+        links: list[dict[str, str]] = []
+        seen: set[str] = set()
+        for item in raw_links:
+            href = item["url"].strip()
+            if not href or href.startswith(("#", "javascript:", "mailto:", "tel:")):
+                continue
+            url = urljoin(base_url, href)
+            if urlparse(url).scheme not in ("http", "https"):
+                continue
+            if url.split("#", 1)[0] == base_url.split("#", 1)[0]:
+                continue
+            text = item["text"]
+            if len(text) < 2 or url in seen:
+                continue
+            seen.add(url)
+            links.append({"url": url, "text": text})
+            if len(links) >= max(1, int(max_links)):
+                break
+        if links:
+            out["links"] = links
+    return out

package/scripts/arxiv_source.py

@@ -0,0 +1,201 @@
+"""arXiv 源码包（e-print）获取与安全解包 — 论文 LaTeX 精读的取数层。
+
+P1 论文深读三能力之一（2026-09-30 P0 实测后立项）：外部论文工具的
+get_paper_latex 实测 3.4s 拿到 main tex + 源文件清单，argo 此前无此能力。
+
+安全边界（全部内存态，不落盘解包）：
+  1. 只认 arxiv.org 的 https URL（scheme + 域名白名单，拒环回/私有地址）
+  2. 下载总量封顶（默认 64MB），成员数封顶（500），单文件解压封顶（2MB）
+  3. tar 成员名拒绝绝对路径 / 路径穿越 / 盘符——本模块只读内存，永不写文件
+  4. 只收 .tex/.bbl 文本（其余成员只列名不收内容）
+
+缓存：走 fetch 缓存（同 id 1h 内二读零成本），与 pdf 提取缓存同一纪律。
+"""
+from __future__ import annotations
+
+import gzip
+import io
+import tarfile
+from typing import Any
+
+from net_proxy import open_url
+from pdf_extract import CONTENT_WARNING, chunk_text
+
+# 域名白名单（e-print 只在 arxiv.org 主站）
+_ALLOWED_HOSTS = {"arxiv.org", "www.arxiv.org", "export.arxiv.org"}
+_MAX_DOWNLOAD = 64 * 1024 * 1024        # 压缩包下载上限
+_MAX_MEMBERS = 500                       # 成员数上限
+_MAX_TOTAL_UNCOMPRESSED = 64 * 1024 * 1024  # 解压总量上限
+_MAX_FILE_UNCOMPRESSED = 2 * 1024 * 1024    # 单文件解压上限（.tex/.bbl）
+_MAX_CACHE_TEXTS = 2 * 1024 * 1024       # 进缓存的文本总量上限（超出只存清单）
+_TEX_EXTS = (".tex", ".bbl")
+_DOCUMENTCLASS_RE = None  # 惰性编译（见 _is_main_candidate）
+
+
+class EprintError(Exception):
+    """e-print 取数/解包失败（消息可直接进用户输出）。"""
+
+
+def _is_main_candidate(name: str, text: str) -> bool:
+    global _DOCUMENTCLASS_RE
+    if _DOCUMENTCLASS_RE is None:
+        import re
+        _DOCUMENTCLASS_RE = re.compile(r"\\documentclass")
+    return _DOCUMENTCLASS_RE.search(text) is not None
+
+
+def _validate_url(url: str) -> None:
+    """仅允许白名单域名的 http(s) URL（拒环回/私有/保留地址）。"""
+    from urllib.parse import urlparse
+    p = urlparse(url)
+    if p.scheme not in ("http", "https"):
+        raise EprintError(f"仅允许 http(s) URL：{url[:80]}")
+    host = (p.hostname or "").lower()
+    if host not in _ALLOWED_HOSTS:
+        raise EprintError(f"域名不在白名单（{'/'.join(sorted(_ALLOWED_HOSTS))}）：{host}")
+
+
+def _safe_name(name: str) -> bool:
+    """tar 成员名安全判定：拒绝对路径/盘符/穿越（只读内存态，防御性双保险）。"""
+    if not name or name.startswith(("/", "\\")):
+        return False
+    if ":" in name.split("/")[0]:  # C:\ 盘符形态
+        return False
+    parts = name.replace("\\", "/").split("/")
+    return ".." not in parts
+
+
+def _pick_main(texts: dict[str, str]) -> str | None:
+    """主 tex：含 \\documentclass 的最大文件；没有则最大的 .tex。"""
+    candidates = [n for n in texts if n.endswith(".tex")]
+    if not candidates:
+        return None
+    doc_class = [n for n in candidates if _is_main_candidate(n, texts[n])]
+    pool = doc_class or candidates
+    return max(pool, key=lambda n: len(texts[n]))
+
+
+def parse_eprint(body: bytes, *, max_members: int = _MAX_MEMBERS,
+                 max_total: int = _MAX_TOTAL_UNCOMPRESSED,
+                 max_file: int = _MAX_FILE_UNCOMPRESSED) -> dict[str, Any]:
+    """解析 e-print 字节流（tar.gz / 单文件 gzip / 纯 tex），全内存态。
+
+    返回 {format, files:[{name,size}], texts:{name:content}, rejected:[...]}，
+    非 LaTeX 源包（如误传 PDF）抛 EprintError。
+    """
+    if body[:5] == b"%PDF-":
+        raise EprintError("这是 PDF 不是 LaTeX 源包（e-print 端点才返回源码）")
+    tf = None
+    try:
+        try:
+            tf = tarfile.open(fileobj=io.BytesIO(body), mode="r:*")
+            members = [(m.name, m.size, tf.extractfile(m)) for m in tf.getmembers()]
+            fmt = "tar"
+        except tarfile.ReadError:
+            if body[:2] == b"\x1f\x8b":  # 单文件 gzip（老论文常见：单个 .tex.gz）
+                text = gzip.decompress(body).decode("utf-8", errors="replace")
+                name = "main.tex"
+                return {"format": "gz", "main_file": name,
+                        "files": [{"name": name, "size": len(text)}],
+                        "texts": {name: text}, "rejected": []}
+            text = body.decode("utf-8", errors="replace")
+            return {"format": "plain", "main_file": "main.tex",
+                    "files": [{"name": "main.tex", "size": len(text)}],
+                    "texts": {"main.tex": text}, "rejected": []}
+    except EprintError:
+        raise
+    except Exception as e:
+        raise EprintError(f"源码包解析失败: {str(e)[:120]}")
+
+    files: list[dict[str, Any]] = []
+    texts: dict[str, str] = {}
+    rejected: list[str] = []
+    total = 0
+    if len(members) > max_members:
+        raise EprintError(f"成员数 {len(members)} 超上限 {max_members}，疑似恶意包")
+    for raw_name, size, fh in members:
+        if not _safe_name(raw_name):
+            rejected.append(raw_name)
+            continue
+        if size > max_file:
+            rejected.append(f"{raw_name}（单文件 {size} 超上限）")
+            continue
+        total += size
+        if total > max_total:
+            raise EprintError(f"解压总量超上限 {max_total // (1024 * 1024)}MB")
+        if not raw_name.lower().endswith(_TEX_EXTS) or fh is None:
+            files.append({"name": raw_name, "size": size})
+            continue
+        data = fh.read(max_file + 1)
+        if len(data) > max_file:
+            rejected.append(f"{raw_name}（解压超限）")
+            continue
+        text = data.decode("utf-8", errors="replace")
+        files.append({"name": raw_name, "size": size})
+        texts[raw_name] = text
+    main = _pick_main(texts)
+    return {"format": fmt, "files": files, "texts": texts,
+            "main_file": main, "rejected": rejected}
+
+
+def fetch_eprint(arxiv_id: str, use_cache: bool = True) -> dict[str, Any]:
+    """拉取并解析 arXiv 源码包；带 fetch 缓存（1h 内二读零成本）。
+
+    返回 parse_eprint 结果 + {"content_warning": ...}；texts 总量超缓存上限时
+    只缓存清单并置 texts_omitted（本次调用仍返回全文）。
+    """
+    aid = arxiv_id.strip()
+    if not aid or any(c.isspace() for c in aid):
+        raise EprintError(f"arXiv id 形态不对：{aid[:40]}")
+    url = f"https://arxiv.org/e-print/{aid}"
+    _validate_url(url)
+
+    cache = None
+    cache_key = f"arxiv-eprint:{aid}"
+    if use_cache:
+        try:
+            from cache import SearchCache
+            cache = SearchCache()
+            hit = cache.get_fetch(cache_key)
+            if isinstance(hit, dict) and hit.get("files"):
+                hit["_cached"] = True
+                return hit
+        except Exception:
+            cache = None
+
+    with open_url(url, timeout=60) as resp:
+        body = resp.read(_MAX_DOWNLOAD + 1)
+    if len(body) > _MAX_DOWNLOAD:
+        raise EprintError(f"源码包超过下载上限 {_MAX_DOWNLOAD // (1024 * 1024)}MB")
+
+    result = parse_eprint(body)
+    result["arxiv_id"] = aid
+    result["content_warning"] = CONTENT_WARNING
+    if cache is not None:
+        try:
+            payload = {k: v for k, v in result.items() if k != "_cached"}
+            if sum(len(t) for t in payload.get("texts", {}).values()) > _MAX_CACHE_TEXTS:
+                payload["texts"] = {}
+                payload["texts_omitted"] = True
+            cache.set_fetch(cache_key, payload)
+        except Exception:
+            pass  # 缓存写失败不影响本次结果
+    return result
+
+
+def latex_read(source: dict[str, Any], file: str | None = None,
+               start: int = 0, max_chars: int | None = None) -> dict[str, Any]:
+    """从已解析的源码包按文件分页读取（默认主 tex）；chunk 契约与 pdf 相同。"""
+    texts = source.get("texts") or {}
+    name = file or source.get("main_file")
+    if not name:
+        raise EprintError("源码包内没有可读的 .tex 文件")
+    if name not in texts:
+        if source.get("texts_omitted"):
+            raise EprintError("该源码包文本未入缓存（体积超限），请重新拉取")
+        available = ", ".join(sorted(texts))[:200] or "（无 .tex 成员）"
+        raise EprintError(f"文件 {name} 不在源码包内。可用：{available}")
+    seg = chunk_text(texts[name], start, max_chars or 12000)
+    return {"content_warning": CONTENT_WARNING, "file": name,
+            "main_file": source.get("main_file"), "source_format": source.get("format"),
+            **seg}