argo-search 1.0.1 → 2.9.7

package/scripts/social_engines/twitter_engine.py

@@ -0,0 +1,397 @@
+#!/usr/bin/env python3
+"""Twitter/X 搜索引擎
+
+后端优先级：
+  1. FxTwitter API v2（api.fxtwitter.com，零 Key）— 主路径
+  2. twitter CLI（tw）
+  3. nitter 公开实例 HTML
+
+FxTwitter 能力：
+  - GET /2/search?q=...&count=N  推文搜索
+  - GET /2/status/{id}           单条推文
+  - 旧版 GET /:user?/status/:id  兼容
+
+文档：https://docs.fxembed.com/api/twitter/
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import subprocess
+import sys
+import time
+import urllib.parse
+from typing import Any
+
+# 出口调度唯一入口（issue #13 同类修复）：urlopen 不认 config.yaml 的
+# network.proxy，裸用会在「必须经代理才能出网」的环境里整源连不上。
+import os
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+from fxtwitter_api import extract_status_id as _fx_extract_status_id  # noqa: E402
+from fxtwitter_api import http_get as _fx_http_get  # noqa: E402
+
+FXTWITTER_BASE = "https://api.fxtwitter.com"
+USER_AGENT = "argo-search/1.0 (+https://github.com/taxueseek/argo; fxtwitter)"
+
+# 推文 URL / snowflake id 的解析与 HTTP 出口统一收在 fxtwitter_api（打包器
+# tweet.py 用的是同一份）；本模块只保留「引擎侧阈值」——纯数字要求 ≥10 位，
+# 见 _extract_status_id。此前这里自带一份正则，与打包器那份迟早漂移。
+
+
+def _http_get_with_retry(
+    url: str,
+    headers: dict | None = None,
+    timeout: int = 10,
+    max_retries: int = 2,
+) -> tuple[bytes, int]:
+    """带重试的 HTTP GET（实现唯一在 fxtwitter_api.http_get）。
+
+    保留本名与签名：既有测试 patch 的正是 `tw._http_get_with_retry`，薄壳让
+    patch 语义不变；同时引擎与打包器共用同一份「重试 + 代理 + SSRF 守卫」纪律，
+    不再各写一遍。
+    """
+    return _fx_http_get(url, headers=headers, timeout=timeout,
+                        max_retries=max_retries)
+    return b"", 0
+
+
+def _extract_status_id(query: str) -> str | None:
+    """从查询中提取推文 ID（URL 或纯数字）。
+
+    解析器唯一实现在 fxtwitter_api.extract_status_id；这里只固定搜索侧的阈值
+    min_digits=10——`argo search "20"` 里的 20 可能是页码或数量，不该当推文 ID；
+    而 `argo tweet "20"` 是用户显式给的 ID（那边传 1）。
+    """
+    return _fx_extract_status_id(query or "", min_digits=10)
+
+
+def _status_to_result(item: dict[str, Any], rank: int = 0) -> dict[str, Any] | None:
+    """将 FxTwitter status 对象转为统一 schema。"""
+    if not isinstance(item, dict):
+        return None
+    # v2 status 可能包在 status 键下，或直接是 status 对象
+    if "text" not in item and isinstance(item.get("status"), dict):
+        item = item["status"]
+    text = (item.get("text") or "").strip()
+    url = item.get("url") or ""
+    sid = str(item.get("id") or "")
+    if not url and sid:
+        author = item.get("author") if isinstance(item.get("author"), dict) else {}
+        handle = author.get("screen_name") or "i"
+        url = f"https://x.com/{handle}/status/{sid}"
+    if not text and not url:
+        return None
+
+    author = item.get("author") if isinstance(item.get("author"), dict) else {}
+    likes = item.get("likes") or 0
+    reposts = item.get("reposts") if item.get("reposts") is not None else item.get("retweets") or 0
+    replies = item.get("replies") or 0
+    views = item.get("views")
+    quotes = item.get("quotes") or 0
+    bookmarks = item.get("bookmarks") or 0
+
+    title = text[:100] + ("..." if len(text) > 100 else "")
+    if not title:
+        title = f"Tweet {sid}" if sid else url
+
+    # 互动分：简单归一化，便于排序参考
+    engagement = int(likes or 0) + int(reposts or 0) * 2 + int(replies or 0)
+    score = max(1.0 - rank * 0.05, 0.1)
+    if engagement > 0:
+        score = min(1.0, score + min(engagement / 10000.0, 0.3))
+
+    return {
+        "title": title,
+        "url": url,
+        "snippet": text[:300],
+        "source": "twitter",
+        "score": round(score, 3),
+        "published_at": item.get("created_at") or None,
+        "social_meta": {
+            "platform": "twitter",
+            "content_type": "tweet",
+            "id": sid,
+            "author": author.get("screen_name") or "",
+            "author_name": author.get("name") or "",
+            "likes": likes,
+            "retweets": reposts,
+            "reposts": reposts,
+            "replies": replies,
+            "views": views,
+            "quotes": quotes,
+            "bookmarks": bookmarks,
+            "lang": item.get("lang"),
+            "provider": "fxtwitter",
+        },
+    }
+
+
+def fetch_status(status_id: str, timeout: int = 10) -> list[dict[str, Any]]:
+    """按推文 ID 拉取单条（v2 优先，旧端点保底）。"""
+    endpoints = [
+        f"{FXTWITTER_BASE}/2/status/{status_id}",
+        f"{FXTWITTER_BASE}/status/{status_id}",
+    ]
+    last_err: Exception | None = None
+    for url in endpoints:
+        try:
+            body, _ = _http_get_with_retry(url, timeout=timeout, max_retries=2)
+            data = json.loads(body.decode("utf-8"))
+            if not isinstance(data, dict):
+                continue
+            code = data.get("code", 200)
+            if code not in (200, None, "200"):
+                continue
+            # v2: {status: {...}} ; v1: {tweet: {...}}
+            item = data.get("status") or data.get("tweet") or data
+            result = _status_to_result(item if isinstance(item, dict) else {}, rank=0)
+            if result:
+                return [result]
+        except Exception as e:
+            last_err = e
+            continue
+    if last_err is not None:
+        # 全部端点异常 ≠ 「没有这条推文」：带 error 浮上来（同 search_fxtwitter）
+        return [{"error": f"twitter {type(last_err).__name__}: {last_err}",
+                 "source": "twitter"}]
+    return []
+
+
+def search_fxtwitter(query: str, n: int = 5, timeout: int = 10) -> list[dict[str, Any]]:
+    """通过 FxTwitter /2/search 搜索推文。"""
+    if not query or not query.strip():
+        return []
+
+    # URL / ID 查询 → 单条拉取
+    status_id = _extract_status_id(query)
+    if status_id:
+        return fetch_status(status_id, timeout=timeout)[:n]
+
+    params = urllib.parse.urlencode({"q": query.strip(), "count": max(1, min(int(n), 20))})
+    url = f"{FXTWITTER_BASE}/2/search?{params}"
+    try:
+        body, _ = _http_get_with_retry(url, timeout=timeout, max_retries=2)
+        data = json.loads(body.decode("utf-8"))
+    except Exception as e:
+        # 吞异常返空会让 MCP 社交搜索把网络故障静默报成「平台成功、0 结果」
+        # （mcp_handlers 的 err=None 设计依赖引擎抛异常或返 error 占位，
+        # 见 tests/test_regression_p0p1.py「零结果应以 errors 提示」契约）。
+        # 对齐 zhihu_engine 范式：失败必须带着 error 字段浮上来。
+        return [{"error": f"twitter {type(e).__name__}: {e}", "source": "twitter"}]
+
+    if not isinstance(data, dict):
+        return []
+    code = data.get("code", 200)
+    if code not in (200, None, "200"):
+        return []
+
+    results: list[dict[str, Any]] = []
+    for i, item in enumerate(data.get("results") or []):
+        if not isinstance(item, dict):
+            continue
+        # 仅处理 status 类型；忽略用户卡等
+        if item.get("type") and item.get("type") != "status":
+            continue
+        parsed = _status_to_result(item, rank=i)
+        if parsed:
+            results.append(parsed)
+        if len(results) >= n:
+            break
+    return results
+
+
+# Nitter 实例健康状态（进程内记忆）：记录每个实例的连续失败次数和
+# 最后健康时间，优先尝试最近健康的实例。实例不可达时不永久封禁，
+# 而是逐步降权——Nitter 实例可用性波动大，单次失败不等于永久不可用。
+_nitter_health: dict[str, dict] = {}
+_NITTER_MAX_FAILURES = 3
+
+
+def _nitter_healthy_instances() -> list[str]:
+    """按健康状态排序的 Nitter 实例列表，最近健康的优先。"""
+    instances = [
+        "https://nitter.net",
+        "https://nitter.privacydev.net",
+        "https://nitter.poast.org",
+        "https://nitter.privacyredirect.com",
+        "https://nitter.tiekoetter.com",
+    ]
+    def _sort_key(base: str) -> tuple:
+        h = _nitter_health.get(base)
+        if h is None:
+            return (0, 0.0)  # 未记录过，优先尝试
+        fails = h.get("fails", 0)
+        last_ok = h.get("last_ok", 0.0)
+        return (fails, -last_ok)  # 失败少的优先，最近健康的优先
+    return sorted(instances, key=_sort_key)
+
+
+def search_nitter(query: str, n: int = 5) -> list[dict]:
+    """通过 nitter 公开实例搜索推文（零认证，保底）。
+
+    多实例故障转移：按健康状态排序，优先尝试最近健康的实例。
+    实例失败时降权（不永久封禁），成功时恢复权重。
+    """
+    encoded_query = urllib.parse.quote(query)
+
+    for base in _nitter_healthy_instances():
+        try:
+            url = f"{base}/search?f=tweets&q={encoded_query}&since=2025-01-01"
+            headers = {
+                "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"
+            }
+            body, _ = _http_get_with_retry(url, headers=headers, timeout=8, max_retries=1)
+            html = body.decode("utf-8", errors="replace")
+            # 更新健康状态
+            _nitter_health[base] = {"fails": 0, "last_ok": time.time()}
+            results = _parse_nitter_html(html, n)
+            # 空结果也算实例健康（只是没有匹配内容），不降权
+            return results
+        except Exception:
+            # 失败时增加失败计数
+            h = _nitter_health.get(base) or {}
+            h["fails"] = h.get("fails", 0) + 1
+            _nitter_health[base] = h
+            continue
+    return []
+
+
+def _parse_nitter_html(html: str, n: int) -> list[dict]:
+    """解析 nitter 搜索结果 HTML。"""
+    results = []
+    tweet_pattern = re.compile(
+        r'<div class="tweet-content[^"]*".*?'
+        r'<a class="tweet-link" href="([^"]+)".*?'
+        r'<div class="tweet-content[^"]*"[^>]*>(.*?)</div>',
+        re.DOTALL,
+    )
+    for match in tweet_pattern.finditer(html):
+        url = match.group(1)
+        content = match.group(2)
+        content = re.sub(r"<[^>]+>", " ", content).strip()
+        content = re.sub(r"\s+", " ", content)
+        if len(content) > 10:
+            results.append({
+                "title": content[:100] + ("..." if len(content) > 100 else ""),
+                "url": url if url.startswith("http") else f"https://twitter.com{url}",
+                "snippet": content[:300],
+                "source": "twitter",
+                "score": max(1.0 - len(results) * 0.1, 0.1),
+                "social_meta": {
+                    "platform": "twitter",
+                    "content_type": "tweet",
+                    "provider": "nitter",
+                },
+            })
+        if len(results) >= n:
+            break
+    return results
+
+
+def search(query: str, n: int = 5) -> list[dict]:
+    """主搜索入口：FxTwitter → tw CLI → nitter。"""
+    # 1) FxTwitter（主路径）
+    try:
+        results = search_fxtwitter(query, n=n, timeout=10)
+        if results:
+            return results
+    except Exception:
+        pass
+
+    # 2) twitter CLI
+    try:
+        result = subprocess.run(
+            ["tw", "search", query, "--limit", str(n), "--json"],
+            capture_output=True,
+            text=True,
+            encoding="utf-8",  # Windows GBK locale 下 UTF-8 输出会解码崩
+            errors="replace",
+            timeout=10,
+        )
+        if result.returncode == 0 and result.stdout.strip():
+            parsed = _parse_tw_json(result.stdout, n)
+            if parsed:
+                return parsed
+    except (FileNotFoundError, subprocess.TimeoutExpired):
+        pass
+
+    # 3) nitter 保底
+    return search_nitter(query, n)
+
+
+def _parse_tw_json(raw: str, n: int) -> list[dict]:
+    """解析 twitter CLI JSON 输出。"""
+    results = []
+    for line in raw.strip().split("\n"):
+        try:
+            tweet = json.loads(line)
+            if not isinstance(tweet, dict):
+                continue
+            results.append({
+                "title": (tweet.get("text", "") or tweet.get("full_text", ""))[:100],
+                "url": tweet.get("url", f"https://twitter.com/i/status/{tweet.get('id', '')}"),
+                "snippet": tweet.get("text", "") or tweet.get("full_text", ""),
+                "source": "twitter",
+                "score": max(1.0 - len(results) * 0.1, 0.1),
+                "social_meta": {
+                    "platform": "twitter",
+                    "content_type": "tweet",
+                    "author": tweet.get("user", {}).get("screen_name", ""),
+                    "likes": tweet.get("favorite_count", 0),
+                    "retweets": tweet.get("retweet_count", 0),
+                    "replies": tweet.get("reply_count", 0),
+                    "provider": "tw_cli",
+                },
+            })
+        except (json.JSONDecodeError, ValueError):
+            continue
+        if len(results) >= n:
+            break
+    return results
+
+
+def main():
+    import argparse
+
+    parser = argparse.ArgumentParser(description="Twitter search engine (FxTwitter)")
+    parser.add_argument("action", nargs="?", default="search")
+    parser.add_argument("query", nargs="?")
+    parser.add_argument("-n", type=int, default=5)
+    parser.add_argument("--json", action="store_true")
+    args = parser.parse_args()
+
+    if not args.query:
+        print("[]")
+        return
+
+    action = (args.action or "search").lower()
+    if action == "status" and args.query:
+        results = fetch_status(args.query)
+    elif action in ("search", "s") and args.query:
+        results = search(args.query, args.n)
+    elif args.query:
+        # 未知 action 时把 query 当搜索词
+        results = search(args.query, args.n)
+    else:
+        # 兼容：`twitter_engine.py "查询词"`
+        results = search(args.action, args.n)
+
+    if args.json:
+        print(json.dumps(results, ensure_ascii=False))
+    else:
+        for i, r in enumerate(results, 1):
+            print(f"### {i}. {r['title']}")
+            print(f"- **URL**: {r['url']}")
+            print(f"- {r['snippet'][:200]}")
+            meta = r.get("social_meta") or {}
+            if meta.get("likes") is not None:
+                print(
+                    f"- likes={meta.get('likes')} reposts={meta.get('retweets') or meta.get('reposts')} "
+                    f"replies={meta.get('replies')} @{meta.get('author', '')}"
+                )
+            print()
+
+
+if __name__ == "__main__":
+    main()

package/scripts/social_engines/v2ex_engine.py

@@ -0,0 +1,101 @@
+#!/usr/bin/env python3
+"""V2EX 社区搜索引擎（公开网页，零密钥）
+
+复用 argo route 库 v2ex builder 的 HTML 解析逻辑，保持一致 social_engines schema。
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import urllib.parse
+import urllib.request
+from typing import Any
+
+# 出口调度唯一入口（issue #13 同类修复）：urlopen 不认 config.yaml 的
+# network.proxy，裸用会在「必须经代理才能出网」的环境里整源连不上。
+import os
+import sys
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+from net_proxy import open_url  # noqa: E402
+
+
+def _http_get(url: str, timeout: int = 10) -> str:
+    req = urllib.request.Request(
+        url,
+        headers={"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"},
+    )
+    with open_url(req, timeout=timeout) as resp:
+        return resp.read().decode("utf-8", "replace")
+
+
+def search(query: str, n: int = 5) -> list[dict[str, Any]]:
+    """V2EX 站内搜索（q 参数 + 结果页解析）。"""
+    if not query or not query.strip():
+        return []
+    url = f"https://www.v2ex.com/search?q={urllib.parse.quote(query.strip())}"
+    try:
+        html = _http_get(url)
+    except Exception as e:
+        # 吞异常返空会让 MCP 社交搜索把网络故障静默报成「平台成功、0 结果」
+        # （mcp_handlers 的 err=None 设计依赖引擎抛异常或返 error 占位，
+        # 见 tests/test_regression_p0p1.py「零结果应以 errors 提示」契约）。
+        # 对齐 zhihu_engine 范式：失败必须带着 error 字段浮上来。
+        return [{"error": f"v2ex {type(e).__name__}: {e}", "source": "v2ex"}]
+
+    results: list[dict[str, Any]] = []
+    # 主题列表：<a href="/t/xxxx"> 标题 </a>
+    topic_re = re.compile(r'<a[^>]*href="(/t/\d+)[^"]*"[^>]*>(.*?)</a>', re.DOTALL)
+    seen = set()
+    for m in topic_re.finditer(html):
+        path, raw_title = m.group(1), m.group(2)
+        if path in seen:
+            continue
+        seen.add(path)
+        title = re.sub(r"<[^>]+>", "", raw_title).strip()
+        if not title or len(title) < 2:
+            continue
+        results.append({
+            "title": title[:100] + ("..." if len(title) > 100 else ""),
+            "url": f"https://www.v2ex.com{path}",
+            "snippet": title[:300],
+            "source": "v2ex",
+            "score": max(1.0 - len(results) * 0.1, 0.1),
+            "social_meta": {
+                "platform": "v2ex",
+                "content_type": "topic",
+                # 剥的是**前缀**，不是字符集：lstrip("/t/") 会连 topic id 里
+                # 出现的 't' 一起吃掉（当前 href 正则限定 \d+ 才没暴露）。
+                "id": path.removeprefix("/t/"),
+                "provider": "v2ex_html",
+            },
+        })
+        if len(results) >= n:
+            break
+    return results
+
+
+def main():
+    import argparse
+    parser = argparse.ArgumentParser(description="V2EX search engine")
+    parser.add_argument("action", nargs="?", default="search")
+    parser.add_argument("query", nargs="?")
+    parser.add_argument("-n", type=int, default=5)
+    parser.add_argument("--json", action="store_true")
+    args = parser.parse_args()
+
+    if not args.query:
+        print("[]")
+        return
+    results = search(args.query, args.n)
+    if args.json:
+        print(json.dumps(results, ensure_ascii=False))
+    else:
+        for i, r in enumerate(results, 1):
+            print(f"### {i}. {r['title']}")
+            print(f"- **URL**: {r['url']}")
+            print()
+
+
+if __name__ == "__main__":
+    main()

package/scripts/social_engines/weibo_engine.py

@@ -0,0 +1,128 @@
+#!/usr/bin/env python3
+"""微博搜索引擎
+
+使用微博公开搜索 API（无需登录可搜索热门内容）。
+"""
+
+import json
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+
+# 出口调度唯一入口（issue #13 同类修复）：urlopen 原生只认标准 HTTP(S)_PROXY
+# 环境变量，**不认** config.yaml 的 network.proxy —— 裸用会在「必须经代理才能
+# 出网」的环境里整源连不上。本文件在 scripts/ 子目录，父目录不在 sys.path 上。
+import os
+import sys
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+from net_proxy import open_url  # noqa: E402
+
+
+def _http_get_with_retry(url: str, headers: dict, timeout: int = 10, max_retries: int = 2):
+    """带重试的 HTTP GET，尊重 429 + Retry-After。
+
+    仅使用 stdlib，不引入第三方依赖。
+    返回 (body_bytes, status_code)。失败时抛出最后一次异常。
+    """
+    for attempt in range(max_retries + 1):
+        try:
+            req = urllib.request.Request(url, headers=headers)
+            with open_url(req, timeout=timeout) as resp:
+                return resp.read(), resp.status
+        except urllib.error.HTTPError as e:
+            if e.code == 429 and attempt < max_retries:
+                retry_after = int(e.headers.get("Retry-After", "5"))
+                time.sleep(min(retry_after, 30))
+                continue
+            raise
+        except (urllib.error.URLError, OSError):
+            if attempt < max_retries:
+                time.sleep(2 ** attempt + 0.5)
+                continue
+            raise
+    return b"", 0
+
+
+def search(query: str, n: int = 5) -> list[dict]:
+    """通过微博搜索 API 搜索"""
+    encoded = urllib.parse.quote(query)
+    url = f"https://m.weibo.cn/api/container/getIndex?containerid=100103type%3D1%26q%3D{encoded}&page_type=searchall"
+    headers = {
+        "User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 16_0 like Mac OS X) AppleWebKit/605.1.15",
+        "Referer": "https://m.weibo.cn"
+    }
+    try:
+        body, _ = _http_get_with_retry(url, headers, timeout=10, max_retries=2)
+        data = json.loads(body.decode("utf-8"))
+        return _parse_weibo_response(data, n)
+    except Exception as e:
+        # 吞异常返空会让 MCP 社交搜索把网络故障静默报成「平台成功、0 结果」
+        # （mcp_handlers 的 err=None 设计依赖引擎抛异常或返 error 占位，
+        # 见 tests/test_regression_p0p1.py「零结果应以 errors 提示」契约）。
+        # 对齐 zhihu_engine 范式：失败必须带着 error 字段浮上来。
+        return [{"error": f"weibo {type(e).__name__}: {e}", "source": "weibo"}]
+
+
+def _parse_weibo_response(data: dict, n: int) -> list[dict]:
+    """解析微博 API 响应"""
+    results = []
+    for card in data.get("data", {}).get("cards", []):
+        for card_group in card.get("card_group", []):
+            mblog = card_group.get("mblog", {})
+            if not mblog:
+                continue
+            text = mblog.get("text", "")
+            import re
+            text = re.sub(r'<[^>]+>', '', text).strip()
+            if len(text) < 10:
+                continue
+            results.append({
+                "title": text[:100] + ("..." if len(text) > 100 else ""),
+                "url": f"https://weibo.com/{mblog.get('user', {}).get('id', '')}/{mblog.get('bid', '')}",
+                "snippet": text[:300],
+                "source": "weibo",
+                "score": max(1.0 - len(results) * 0.1, 0.1),
+                "social_meta": {
+                    "platform": "weibo",
+                    "content_type": "post",
+                    "author": mblog.get("user", {}).get("screen_name", ""),
+                    "likes": mblog.get("attitudes_count", 0),
+                    "reposts": mblog.get("reposts_count", 0),
+                    "comments": mblog.get("comments_count", 0),
+                    "verified": mblog.get("user", {}).get("verified", False),
+                }
+            })
+            if len(results) >= n:
+                break
+        if len(results) >= n:
+            break
+    return results
+
+
+def main():
+    import argparse
+    parser = argparse.ArgumentParser(description="Weibo search engine")
+    parser.add_argument("action", nargs="?", default="search")
+    parser.add_argument("query", nargs="?")
+    parser.add_argument("-n", type=int, default=5)
+    parser.add_argument("--json", action="store_true")
+    args = parser.parse_args()
+
+    if not args.query:
+        print("[]")
+        return
+
+    results = search(args.query, args.n)
+    if args.json:
+        print(json.dumps(results, ensure_ascii=False))
+    else:
+        for i, r in enumerate(results, 1):
+            print(f"### {i}. {r['title']}")
+            print(f"- **URL**: {r['url']}")
+            print(f"- {r['snippet'][:200]}")
+            print()
+
+
+if __name__ == "__main__":
+    main()