argo-search 1.0.1 → 2.9.8

package/scripts/social_engines/v2ex_engine.py

@@ -0,0 +1,101 @@
+#!/usr/bin/env python3
+"""V2EX 社区搜索引擎（公开网页，零密钥）
+
+复用 argo route 库 v2ex builder 的 HTML 解析逻辑，保持一致 social_engines schema。
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import urllib.parse
+import urllib.request
+from typing import Any
+
+# 出口调度唯一入口（issue #13 同类修复）：urlopen 不认 config.yaml 的
+# network.proxy，裸用会在「必须经代理才能出网」的环境里整源连不上。
+import os
+import sys
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+from net_proxy import open_url  # noqa: E402
+
+
+def _http_get(url: str, timeout: int = 10) -> str:
+    req = urllib.request.Request(
+        url,
+        headers={"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"},
+    )
+    with open_url(req, timeout=timeout) as resp:
+        return resp.read().decode("utf-8", "replace")
+
+
+def search(query: str, n: int = 5) -> list[dict[str, Any]]:
+    """V2EX 站内搜索（q 参数 + 结果页解析）。"""
+    if not query or not query.strip():
+        return []
+    url = f"https://www.v2ex.com/search?q={urllib.parse.quote(query.strip())}"
+    try:
+        html = _http_get(url)
+    except Exception as e:
+        # 吞异常返空会让 MCP 社交搜索把网络故障静默报成「平台成功、0 结果」
+        # （mcp_handlers 的 err=None 设计依赖引擎抛异常或返 error 占位，
+        # 见 tests/test_regression_p0p1.py「零结果应以 errors 提示」契约）。
+        # 对齐 zhihu_engine 范式：失败必须带着 error 字段浮上来。
+        return [{"error": f"v2ex {type(e).__name__}: {e}", "source": "v2ex"}]
+
+    results: list[dict[str, Any]] = []
+    # 主题列表：<a href="/t/xxxx"> 标题 </a>
+    topic_re = re.compile(r'<a[^>]*href="(/t/\d+)[^"]*"[^>]*>(.*?)</a>', re.DOTALL)
+    seen = set()
+    for m in topic_re.finditer(html):
+        path, raw_title = m.group(1), m.group(2)
+        if path in seen:
+            continue
+        seen.add(path)
+        title = re.sub(r"<[^>]+>", "", raw_title).strip()
+        if not title or len(title) < 2:
+            continue
+        results.append({
+            "title": title[:100] + ("..." if len(title) > 100 else ""),
+            "url": f"https://www.v2ex.com{path}",
+            "snippet": title[:300],
+            "source": "v2ex",
+            "score": max(1.0 - len(results) * 0.1, 0.1),
+            "social_meta": {
+                "platform": "v2ex",
+                "content_type": "topic",
+                # 剥的是**前缀**，不是字符集：lstrip("/t/") 会连 topic id 里
+                # 出现的 't' 一起吃掉（当前 href 正则限定 \d+ 才没暴露）。
+                "id": path.removeprefix("/t/"),
+                "provider": "v2ex_html",
+            },
+        })
+        if len(results) >= n:
+            break
+    return results
+
+
+def main():
+    import argparse
+    parser = argparse.ArgumentParser(description="V2EX search engine")
+    parser.add_argument("action", nargs="?", default="search")
+    parser.add_argument("query", nargs="?")
+    parser.add_argument("-n", type=int, default=5)
+    parser.add_argument("--json", action="store_true")
+    args = parser.parse_args()
+
+    if not args.query:
+        print("[]")
+        return
+    results = search(args.query, args.n)
+    if args.json:
+        print(json.dumps(results, ensure_ascii=False))
+    else:
+        for i, r in enumerate(results, 1):
+            print(f"### {i}. {r['title']}")
+            print(f"- **URL**: {r['url']}")
+            print()
+
+
+if __name__ == "__main__":
+    main()

package/scripts/social_engines/weibo_engine.py

@@ -0,0 +1,128 @@
+#!/usr/bin/env python3
+"""微博搜索引擎
+
+使用微博公开搜索 API（无需登录可搜索热门内容）。
+"""
+
+import json
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+
+# 出口调度唯一入口（issue #13 同类修复）：urlopen 原生只认标准 HTTP(S)_PROXY
+# 环境变量，**不认** config.yaml 的 network.proxy —— 裸用会在「必须经代理才能
+# 出网」的环境里整源连不上。本文件在 scripts/ 子目录，父目录不在 sys.path 上。
+import os
+import sys
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+from net_proxy import open_url  # noqa: E402
+
+
+def _http_get_with_retry(url: str, headers: dict, timeout: int = 10, max_retries: int = 2):
+    """带重试的 HTTP GET，尊重 429 + Retry-After。
+
+    仅使用 stdlib，不引入第三方依赖。
+    返回 (body_bytes, status_code)。失败时抛出最后一次异常。
+    """
+    for attempt in range(max_retries + 1):
+        try:
+            req = urllib.request.Request(url, headers=headers)
+            with open_url(req, timeout=timeout) as resp:
+                return resp.read(), resp.status
+        except urllib.error.HTTPError as e:
+            if e.code == 429 and attempt < max_retries:
+                retry_after = int(e.headers.get("Retry-After", "5"))
+                time.sleep(min(retry_after, 30))
+                continue
+            raise
+        except (urllib.error.URLError, OSError):
+            if attempt < max_retries:
+                time.sleep(2 ** attempt + 0.5)
+                continue
+            raise
+    return b"", 0
+
+
+def search(query: str, n: int = 5) -> list[dict]:
+    """通过微博搜索 API 搜索"""
+    encoded = urllib.parse.quote(query)
+    url = f"https://m.weibo.cn/api/container/getIndex?containerid=100103type%3D1%26q%3D{encoded}&page_type=searchall"
+    headers = {
+        "User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 16_0 like Mac OS X) AppleWebKit/605.1.15",
+        "Referer": "https://m.weibo.cn"
+    }
+    try:
+        body, _ = _http_get_with_retry(url, headers, timeout=10, max_retries=2)
+        data = json.loads(body.decode("utf-8"))
+        return _parse_weibo_response(data, n)
+    except Exception as e:
+        # 吞异常返空会让 MCP 社交搜索把网络故障静默报成「平台成功、0 结果」
+        # （mcp_handlers 的 err=None 设计依赖引擎抛异常或返 error 占位，
+        # 见 tests/test_regression_p0p1.py「零结果应以 errors 提示」契约）。
+        # 对齐 zhihu_engine 范式：失败必须带着 error 字段浮上来。
+        return [{"error": f"weibo {type(e).__name__}: {e}", "source": "weibo"}]
+
+
+def _parse_weibo_response(data: dict, n: int) -> list[dict]:
+    """解析微博 API 响应"""
+    results = []
+    for card in data.get("data", {}).get("cards", []):
+        for card_group in card.get("card_group", []):
+            mblog = card_group.get("mblog", {})
+            if not mblog:
+                continue
+            text = mblog.get("text", "")
+            import re
+            text = re.sub(r'<[^>]+>', '', text).strip()
+            if len(text) < 10:
+                continue
+            results.append({
+                "title": text[:100] + ("..." if len(text) > 100 else ""),
+                "url": f"https://weibo.com/{mblog.get('user', {}).get('id', '')}/{mblog.get('bid', '')}",
+                "snippet": text[:300],
+                "source": "weibo",
+                "score": max(1.0 - len(results) * 0.1, 0.1),
+                "social_meta": {
+                    "platform": "weibo",
+                    "content_type": "post",
+                    "author": mblog.get("user", {}).get("screen_name", ""),
+                    "likes": mblog.get("attitudes_count", 0),
+                    "reposts": mblog.get("reposts_count", 0),
+                    "comments": mblog.get("comments_count", 0),
+                    "verified": mblog.get("user", {}).get("verified", False),
+                }
+            })
+            if len(results) >= n:
+                break
+        if len(results) >= n:
+            break
+    return results
+
+
+def main():
+    import argparse
+    parser = argparse.ArgumentParser(description="Weibo search engine")
+    parser.add_argument("action", nargs="?", default="search")
+    parser.add_argument("query", nargs="?")
+    parser.add_argument("-n", type=int, default=5)
+    parser.add_argument("--json", action="store_true")
+    args = parser.parse_args()
+
+    if not args.query:
+        print("[]")
+        return
+
+    results = search(args.query, args.n)
+    if args.json:
+        print(json.dumps(results, ensure_ascii=False))
+    else:
+        for i, r in enumerate(results, 1):
+            print(f"### {i}. {r['title']}")
+            print(f"- **URL**: {r['url']}")
+            print(f"- {r['snippet'][:200]}")
+            print()
+
+
+if __name__ == "__main__":
+    main()

package/scripts/social_engines/xiaohongshu_engine.py

@@ -0,0 +1,137 @@
+#!/usr/bin/env python3
+"""小红书搜索引擎
+
+使用 xhs-cli (xiaohongshu-cli) 工具。
+需要先通过 `xhs login` 登录获取 cookies。
+"""
+
+import json
+import os
+import subprocess
+
+
+# 内部子进程超时（秒）。此前硬编码 15s，比调度层的单引擎墙钟预算（10s）还长，
+# 属「内层白跑」：外层 subprocess 会在预算处 kill，但内层已消耗的时间追不回。
+# 收到 8s —— 与 execution.default_timeout 一致，且短于预算，
+# 使内层能在被 kill 前主动超时返回（走 except 分支），行为更干净。
+# 可用 ARGO_XHS_TIMEOUT 覆盖，便于慢网调试。
+_DEFAULT_TIMEOUT = 8
+
+
+def _subprocess_timeout() -> int:
+    try:
+        v = int(os.environ.get("ARGO_XHS_TIMEOUT", "").strip())
+        return v if v > 0 else _DEFAULT_TIMEOUT
+    except (TypeError, ValueError):
+        return _DEFAULT_TIMEOUT
+
+
+def search(query: str, n: int = 5) -> list[dict]:
+    """主搜索入口"""
+    try:
+        result = subprocess.run(
+            ["xhs", "search", query],
+            capture_output=True, text=True, timeout=_subprocess_timeout(),
+            encoding="utf-8", errors="replace",  # Windows GBK locale 下 UTF-8 输出会解码崩
+        )
+        if result.returncode == 0 and result.stdout.strip():
+            return _parse_xhs_output(result.stdout, n)
+    except (FileNotFoundError, subprocess.TimeoutExpired) as e:
+        # CLI 未安装/超时不是「平台 0 结果」：带 error 浮上来，与社交引擎
+        # 统一范式一致（zhihu_engine 范式，mcp_handlers 的 err=None 设计
+        # 依赖引擎抛异常或返 error 占位）。
+        return [{"error": f"xhs CLI {type(e).__name__}", "source": "xiaohongshu"}]
+    return []
+
+
+def _parse_xhs_output(raw: str, n: int) -> list[dict]:
+    """解析 xhs-cli 输出"""
+    results = []
+    try:
+        data = json.loads(raw)
+        items = data.get("items", data.get("data", []))
+        if isinstance(items, dict):
+            items = items.get("items", [])
+        for item in items[:n]:
+            if not isinstance(item, dict):
+                continue
+            note_card = item.get("note_card", item)
+            title = note_card.get("display_title", note_card.get("title", ""))
+            desc = note_card.get("desc", "")
+            interact = note_card.get("interact_info", {})
+
+            # URL 可核验性（关键）：
+            # 小红书强制 xsec_token —— 裸 note_id 拼出的 /explore/<id> 会被
+            # 302 拦截，无法打开（实测确认）。因此**优先使用上游返回的原始
+            # URL/带 token 的链接**，只有在拿不到时才回退到裸 ID 拼接，
+            # 并显式标注 url_verifiable=False，让调用方知道这条链接不可复核
+            # （与 V2EX 引擎同一契约：宁可不给链接，也不给打不开的链接）。
+            raw_url = (item.get("url") or item.get("note_url")
+                       or note_card.get("url") or "")
+            xsec = (item.get("xsec_token") or note_card.get("xsec_token") or "")
+            note_id = item.get("id") or note_card.get("note_id") or ""
+            if raw_url:
+                url = raw_url
+                verifiable = True
+            elif note_id and xsec:
+                url = (f"https://www.xiaohongshu.com/explore/{note_id}"
+                       f"?xsec_token={xsec}")
+                verifiable = True
+            elif note_id:
+                url = f"https://www.xiaohongshu.com/explore/{note_id}"
+                verifiable = False   # 裸 ID 必被 302 拦截
+            else:
+                continue             # 无 ID 无 URL：不可核验，不产出
+
+            results.append({
+                "title": title[:100] + ("..." if len(title) > 100 else ""),
+                "url": url,
+                "snippet": (desc or title)[:300],
+                "source": "xiaohongshu",
+                "score": max(1.0 - len(results) * 0.1, 0.1),
+                "social_meta": {
+                    "platform": "xiaohongshu",
+                    "content_type": "note",
+                    "author": note_card.get("user", {}).get("nickname", ""),
+                    "likes": interact.get("liked_count", 0),
+                    "comments": interact.get("comment_count", 0),
+                    "collects": interact.get("collected_count", 0),
+                    "type": note_card.get("type", "normal"),  # normal / video
+                    "note_id": note_id,
+                    "has_xsec_token": bool(xsec),
+                    # False = 该链接为裸 ID 拼接，打开会被平台拦截（不可复核）
+                    "url_verifiable": verifiable,
+                }
+            })
+    except (json.JSONDecodeError, ValueError):
+        # 尝试文本解析
+        pass
+    return results
+
+
+def main():
+    import argparse
+    parser = argparse.ArgumentParser(description="Xiaohongshu search engine")
+    parser.add_argument("action", nargs="?", default="search")
+    parser.add_argument("query", nargs="?")
+    parser.add_argument("-n", type=int, default=5)
+    parser.add_argument("--json", action="store_true")
+    args = parser.parse_args()
+
+    if not args.query:
+        print("[]")
+        return
+
+    results = search(args.query, args.n)
+    if args.json:
+        print(json.dumps(results, ensure_ascii=False))
+    else:
+        for i, r in enumerate(results, 1):
+            print(f"### {i}. {r['title']}")
+            print(f"- **URL**: {r['url']}")
+            print(f"- {r['snippet'][:200]}")
+            print()
+
+
+if __name__ == "__main__":
+    main()

package/scripts/social_engines/zhihu_engine.py

@@ -0,0 +1,139 @@
+#!/usr/bin/env python3
+"""知乎搜索引擎（知乎开放平台 zhihu_search API）
+
+密钥：环境变量 ZHIHU_ACCESS_SECRET（或 ARGO_ZHIHU_ACCESS_SECRET），
+未配置时返回空列表（不抛错、不静默假成功——error 由调用侧显示）。
+保持一致 social_engines 统一 schema，sentiment 聚合按 social_meta 直接可用。
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import time
+import urllib.parse
+import urllib.request
+from typing import Any
+
+# 出口调度唯一入口（issue #13 同类修复）：urlopen 不认 config.yaml 的
+# network.proxy，裸用会在「必须经代理才能出网」的环境里整源连不上。
+import sys as _sys
+_sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+from net_proxy import open_url  # noqa: E402
+
+ZHIHU_API = "https://developer.zhihu.com/api/v1/content/zhihu_search"
+
+
+def _secret() -> str:
+    # 走 engine_env 规范访问器：os.environ 优先，~/.config/argo/env 热读保底
+    # （密钥轮换改文件即生效，无需重启）
+    try:
+        from engine_env import get_env
+        return get_env(["ARGO_ZHIHU_ACCESS_SECRET", "ZHIHU_ACCESS_SECRET"])
+    except ImportError:
+        return os.environ.get("ZHIHU_ACCESS_SECRET") or os.environ.get("ARGO_ZHIHU_ACCESS_SECRET", "")
+
+
+def search(query: str, n: int = 5) -> list[dict[str, Any]]:
+    """通过知乎开放平台 zhihu_search 搜索站内内容。"""
+    if not query or not query.strip():
+        return []
+    secret = _secret()
+    if not secret:
+        return []
+
+    params = urllib.parse.urlencode({
+        "Query": query.strip(),
+        "Count": str(max(1, min(int(n), 20))),
+        "SearchDB": "zhihu",
+    })
+    url = f"{ZHIHU_API}?{params}"
+    headers = {
+        "Authorization": f"Bearer {secret}",
+        "Content-Type": "application/json",
+        "X-Request-Timestamp": str(int(time.time())),
+        "User-Agent": "argo-search/2.6 (unified-search@local)",
+    }
+    try:
+        req = urllib.request.Request(url, headers=headers)
+        with open_url(req, timeout=10) as resp:
+            data = json.loads(resp.read().decode("utf-8", "replace"))
+    except urllib.error.HTTPError as e:
+        # 密钥无效/过期（401/403）等必须暴露为 auth-failed，而不是静默
+        # no-results——调用侧把「没配置」「没结果」区分开才可行动
+        return [{"error": f"zhihu API HTTP {e.code}", "source": "zhihu"}]
+    except Exception as e:
+        return [{"error": f"zhihu API {type(e).__name__}: {e}", "source": "zhihu"}]
+
+    if data.get("Code") not in (0, None):
+        return [{"error": f"zhihu API Code={data.get('Code')} "
+                          f"{str(data.get('Message') or '')[:100]}",
+                 "source": "zhihu"}]
+
+    results: list[dict[str, Any]] = []
+    items = (data.get("Data") or {}).get("Items") or []
+    for item in items[:n]:
+        if not isinstance(item, dict):
+            continue
+        title = (item.get("Title") or "").strip()
+        # API 标题统一带「 - 知乎」尾巴：截断前剥掉（先剥再切，尾巴不占正文）
+        if title.endswith(" - 知乎"):
+            title = title[: -len(" - 知乎")].rstrip()
+        url_ = item.get("Url") or ""
+        snippet = (item.get("ContentText") or title or "")[:300]
+        if not title and not url_:
+            continue
+        content_id = str(item.get("ContentID") or "")
+        author = item.get("AuthorName") or ""
+        likes = item.get("VoteUpCount") or 0
+        comments = item.get("CommentCount") or 0
+        edit_ts = item.get("EditTime")
+        published = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(edit_ts)) if edit_ts else None
+
+        results.append({
+            "title": title[:100] + ("..." if len(title) > 100 else ""),
+            "url": url_,
+            "snippet": snippet,
+            "source": "zhihu",
+            "score": max(1.0 - len(results) * 0.1, 0.1),
+            "published_at": published,
+            "social_meta": {
+                "platform": "zhihu",
+                "content_type": (item.get("ContentType") or "answer").lower(),
+                "id": content_id,
+                "author": author,
+                "likes": likes,
+                "comments": comments,
+                "provider": "zhihu_openapi",
+            },
+        })
+    return results
+
+
+def main():
+    import argparse
+    parser = argparse.ArgumentParser(description="Zhihu search engine (zhihu_openapi)")
+    parser.add_argument("action", nargs="?", default="search")
+    parser.add_argument("query", nargs="?")
+    parser.add_argument("-n", type=int, default=5)
+    parser.add_argument("--json", action="store_true")
+    args = parser.parse_args()
+
+    if not args.query:
+        print("[]")
+        return
+    results = search(args.query, args.n)
+    if args.json:
+        print(json.dumps(results, ensure_ascii=False))
+    else:
+        for i, r in enumerate(results, 1):
+            meta = r.get("social_meta") or {}
+            print(f"### {i}. {r['title']}")
+            print(f"- **URL**: {r['url']}")
+            print(f"- {r['snippet'][:200]}")
+            print(f"- likes={meta.get('likes')} comments={meta.get('comments')} @{meta.get('author', '')}")
+            print()
+
+
+if __name__ == "__main__":
+    main()