argo-search 1.0.1 → 2.9.7

package/scripts/social_engines/xiaohongshu_engine.py

@@ -0,0 +1,137 @@
+#!/usr/bin/env python3
+"""小红书搜索引擎
+
+使用 xhs-cli (xiaohongshu-cli) 工具。
+需要先通过 `xhs login` 登录获取 cookies。
+"""
+
+import json
+import os
+import subprocess
+
+
+# 内部子进程超时（秒）。此前硬编码 15s，比调度层的单引擎墙钟预算（10s）还长，
+# 属「内层白跑」：外层 subprocess 会在预算处 kill，但内层已消耗的时间追不回。
+# 收到 8s —— 与 execution.default_timeout 一致，且短于预算，
+# 使内层能在被 kill 前主动超时返回（走 except 分支），行为更干净。
+# 可用 ARGO_XHS_TIMEOUT 覆盖，便于慢网调试。
+_DEFAULT_TIMEOUT = 8
+
+
+def _subprocess_timeout() -> int:
+    try:
+        v = int(os.environ.get("ARGO_XHS_TIMEOUT", "").strip())
+        return v if v > 0 else _DEFAULT_TIMEOUT
+    except (TypeError, ValueError):
+        return _DEFAULT_TIMEOUT
+
+
+def search(query: str, n: int = 5) -> list[dict]:
+    """主搜索入口"""
+    try:
+        result = subprocess.run(
+            ["xhs", "search", query],
+            capture_output=True, text=True, timeout=_subprocess_timeout(),
+            encoding="utf-8", errors="replace",  # Windows GBK locale 下 UTF-8 输出会解码崩
+        )
+        if result.returncode == 0 and result.stdout.strip():
+            return _parse_xhs_output(result.stdout, n)
+    except (FileNotFoundError, subprocess.TimeoutExpired) as e:
+        # CLI 未安装/超时不是「平台 0 结果」：带 error 浮上来，与社交引擎
+        # 统一范式一致（zhihu_engine 范式，mcp_handlers 的 err=None 设计
+        # 依赖引擎抛异常或返 error 占位）。
+        return [{"error": f"xhs CLI {type(e).__name__}", "source": "xiaohongshu"}]
+    return []
+
+
+def _parse_xhs_output(raw: str, n: int) -> list[dict]:
+    """解析 xhs-cli 输出"""
+    results = []
+    try:
+        data = json.loads(raw)
+        items = data.get("items", data.get("data", []))
+        if isinstance(items, dict):
+            items = items.get("items", [])
+        for item in items[:n]:
+            if not isinstance(item, dict):
+                continue
+            note_card = item.get("note_card", item)
+            title = note_card.get("display_title", note_card.get("title", ""))
+            desc = note_card.get("desc", "")
+            interact = note_card.get("interact_info", {})
+
+            # URL 可核验性（关键）：
+            # 小红书强制 xsec_token —— 裸 note_id 拼出的 /explore/<id> 会被
+            # 302 拦截，无法打开（实测确认）。因此**优先使用上游返回的原始
+            # URL/带 token 的链接**，只有在拿不到时才回退到裸 ID 拼接，
+            # 并显式标注 url_verifiable=False，让调用方知道这条链接不可复核
+            # （与 V2EX 引擎同一契约：宁可不给链接，也不给打不开的链接）。
+            raw_url = (item.get("url") or item.get("note_url")
+                       or note_card.get("url") or "")
+            xsec = (item.get("xsec_token") or note_card.get("xsec_token") or "")
+            note_id = item.get("id") or note_card.get("note_id") or ""
+            if raw_url:
+                url = raw_url
+                verifiable = True
+            elif note_id and xsec:
+                url = (f"https://www.xiaohongshu.com/explore/{note_id}"
+                       f"?xsec_token={xsec}")
+                verifiable = True
+            elif note_id:
+                url = f"https://www.xiaohongshu.com/explore/{note_id}"
+                verifiable = False   # 裸 ID 必被 302 拦截
+            else:
+                continue             # 无 ID 无 URL：不可核验，不产出
+
+            results.append({
+                "title": title[:100] + ("..." if len(title) > 100 else ""),
+                "url": url,
+                "snippet": (desc or title)[:300],
+                "source": "xiaohongshu",
+                "score": max(1.0 - len(results) * 0.1, 0.1),
+                "social_meta": {
+                    "platform": "xiaohongshu",
+                    "content_type": "note",
+                    "author": note_card.get("user", {}).get("nickname", ""),
+                    "likes": interact.get("liked_count", 0),
+                    "comments": interact.get("comment_count", 0),
+                    "collects": interact.get("collected_count", 0),
+                    "type": note_card.get("type", "normal"),  # normal / video
+                    "note_id": note_id,
+                    "has_xsec_token": bool(xsec),
+                    # False = 该链接为裸 ID 拼接，打开会被平台拦截（不可复核）
+                    "url_verifiable": verifiable,
+                }
+            })
+    except (json.JSONDecodeError, ValueError):
+        # 尝试文本解析
+        pass
+    return results
+
+
+def main():
+    import argparse
+    parser = argparse.ArgumentParser(description="Xiaohongshu search engine")
+    parser.add_argument("action", nargs="?", default="search")
+    parser.add_argument("query", nargs="?")
+    parser.add_argument("-n", type=int, default=5)
+    parser.add_argument("--json", action="store_true")
+    args = parser.parse_args()
+
+    if not args.query:
+        print("[]")
+        return
+
+    results = search(args.query, args.n)
+    if args.json:
+        print(json.dumps(results, ensure_ascii=False))
+    else:
+        for i, r in enumerate(results, 1):
+            print(f"### {i}. {r['title']}")
+            print(f"- **URL**: {r['url']}")
+            print(f"- {r['snippet'][:200]}")
+            print()
+
+
+if __name__ == "__main__":
+    main()

package/scripts/social_engines/zhihu_engine.py

@@ -0,0 +1,139 @@
+#!/usr/bin/env python3
+"""知乎搜索引擎（知乎开放平台 zhihu_search API）
+
+密钥：环境变量 ZHIHU_ACCESS_SECRET（或 ARGO_ZHIHU_ACCESS_SECRET），
+未配置时返回空列表（不抛错、不静默假成功——error 由调用侧显示）。
+保持一致 social_engines 统一 schema，sentiment 聚合按 social_meta 直接可用。
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import time
+import urllib.parse
+import urllib.request
+from typing import Any
+
+# 出口调度唯一入口（issue #13 同类修复）：urlopen 不认 config.yaml 的
+# network.proxy，裸用会在「必须经代理才能出网」的环境里整源连不上。
+import sys as _sys
+_sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+from net_proxy import open_url  # noqa: E402
+
+ZHIHU_API = "https://developer.zhihu.com/api/v1/content/zhihu_search"
+
+
+def _secret() -> str:
+    # 走 engine_env 规范访问器：os.environ 优先，~/.config/argo/env 热读保底
+    # （密钥轮换改文件即生效，无需重启）
+    try:
+        from engine_env import get_env
+        return get_env(["ARGO_ZHIHU_ACCESS_SECRET", "ZHIHU_ACCESS_SECRET"])
+    except ImportError:
+        return os.environ.get("ZHIHU_ACCESS_SECRET") or os.environ.get("ARGO_ZHIHU_ACCESS_SECRET", "")
+
+
+def search(query: str, n: int = 5) -> list[dict[str, Any]]:
+    """通过知乎开放平台 zhihu_search 搜索站内内容。"""
+    if not query or not query.strip():
+        return []
+    secret = _secret()
+    if not secret:
+        return []
+
+    params = urllib.parse.urlencode({
+        "Query": query.strip(),
+        "Count": str(max(1, min(int(n), 20))),
+        "SearchDB": "zhihu",
+    })
+    url = f"{ZHIHU_API}?{params}"
+    headers = {
+        "Authorization": f"Bearer {secret}",
+        "Content-Type": "application/json",
+        "X-Request-Timestamp": str(int(time.time())),
+        "User-Agent": "argo-search/2.6 (unified-search@local)",
+    }
+    try:
+        req = urllib.request.Request(url, headers=headers)
+        with open_url(req, timeout=10) as resp:
+            data = json.loads(resp.read().decode("utf-8", "replace"))
+    except urllib.error.HTTPError as e:
+        # 密钥无效/过期（401/403）等必须暴露为 auth-failed，而不是静默
+        # no-results——调用侧把「没配置」「没结果」区分开才可行动
+        return [{"error": f"zhihu API HTTP {e.code}", "source": "zhihu"}]
+    except Exception as e:
+        return [{"error": f"zhihu API {type(e).__name__}: {e}", "source": "zhihu"}]
+
+    if data.get("Code") not in (0, None):
+        return [{"error": f"zhihu API Code={data.get('Code')} "
+                          f"{str(data.get('Message') or '')[:100]}",
+                 "source": "zhihu"}]
+
+    results: list[dict[str, Any]] = []
+    items = (data.get("Data") or {}).get("Items") or []
+    for item in items[:n]:
+        if not isinstance(item, dict):
+            continue
+        title = (item.get("Title") or "").strip()
+        # API 标题统一带「 - 知乎」尾巴：截断前剥掉（先剥再切，尾巴不占正文）
+        if title.endswith(" - 知乎"):
+            title = title[: -len(" - 知乎")].rstrip()
+        url_ = item.get("Url") or ""
+        snippet = (item.get("ContentText") or title or "")[:300]
+        if not title and not url_:
+            continue
+        content_id = str(item.get("ContentID") or "")
+        author = item.get("AuthorName") or ""
+        likes = item.get("VoteUpCount") or 0
+        comments = item.get("CommentCount") or 0
+        edit_ts = item.get("EditTime")
+        published = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(edit_ts)) if edit_ts else None
+
+        results.append({
+            "title": title[:100] + ("..." if len(title) > 100 else ""),
+            "url": url_,
+            "snippet": snippet,
+            "source": "zhihu",
+            "score": max(1.0 - len(results) * 0.1, 0.1),
+            "published_at": published,
+            "social_meta": {
+                "platform": "zhihu",
+                "content_type": (item.get("ContentType") or "answer").lower(),
+                "id": content_id,
+                "author": author,
+                "likes": likes,
+                "comments": comments,
+                "provider": "zhihu_openapi",
+            },
+        })
+    return results
+
+
+def main():
+    import argparse
+    parser = argparse.ArgumentParser(description="Zhihu search engine (zhihu_openapi)")
+    parser.add_argument("action", nargs="?", default="search")
+    parser.add_argument("query", nargs="?")
+    parser.add_argument("-n", type=int, default=5)
+    parser.add_argument("--json", action="store_true")
+    args = parser.parse_args()
+
+    if not args.query:
+        print("[]")
+        return
+    results = search(args.query, args.n)
+    if args.json:
+        print(json.dumps(results, ensure_ascii=False))
+    else:
+        for i, r in enumerate(results, 1):
+            meta = r.get("social_meta") or {}
+            print(f"### {i}. {r['title']}")
+            print(f"- **URL**: {r['url']}")
+            print(f"- {r['snippet'][:200]}")
+            print(f"- likes={meta.get('likes')} comments={meta.get('comments')} @{meta.get('author', '')}")
+            print()
+
+
+if __name__ == "__main__":
+    main()

package/scripts/social_research.py

@@ -0,0 +1,201 @@
+#!/usr/bin/env python3
+"""
+social_research.py — 社交舆情研究模块（从 research.py 拆分）
+
+承载跨平台 UGC 情绪与讨论分析：
+  - social_sentiment_research：并行抓取多平台 → 互动聚合 → 话题提取
+  - aggregate_social_sentiment：MCP argo_social_search mode=sentiment 用
+  - _extract_topics：中英文兼容话题提取
+  - _print_social_report：终端报告输出
+
+从 research.py 拆分以控制文件规模（code-review：单文件 <1k 行）。
+"""
+
+from __future__ import annotations
+
+import time
+from typing import Any
+
+from bounded_run import run_bounded, TaskError
+
+
+def social_sentiment_research(query: str, platforms: list[str] | None = None,
+                              max_results: int = 5) -> dict[str, Any]:
+    """社交舆情研究：跨平台 UGC 情绪与讨论分析"""
+    if platforms is None:
+        platforms = ["twitter", "reddit", "xiaohongshu"]
+
+    from search import super_search
+
+    platform_results: dict[str, list] = {}
+    all_results: list[dict] = []
+    engines_used: set[str] = set()
+    t0 = time.time()
+
+    def _one(platform: str) -> tuple[str, list[dict], set[str]]:
+        try:
+            result = super_search(query, engine=platform, n=max_results, mode="fast")
+            return platform, result.get("results", []), set(result.get("engines_used", []))
+        except Exception:
+            return platform, [], set()
+
+    # 并行抓取各平台（串行时 3 平台 × 秒级延迟累积；并行取最慢平台耗时）。
+    # 总时限 90s：到点就拿已完成部分返回，没完成的平台记空结果，不再被卡住的
+    # 平台拖住、也不会因超时异常中断整次调用。
+    social_errors: list[str] = []
+    finished, unfinished = run_bounded(
+        platforms, _one, 90.0, max_workers=min(max(len(platforms), 1), 4))
+    for platform, value in finished:
+        if isinstance(value, TaskError):
+            platform_results[platform] = []
+            social_errors.append(f"{platform}: {type(value.exc).__name__}")
+            continue
+        _platform, results, used = value
+        platform_results[_platform] = results
+        all_results.extend(results)
+        engines_used.update(used)
+    for platform in unfinished:
+        platform_results[platform] = []
+        social_errors.append(f"{platform}: timeout (>90s)")
+
+    # 互动数据聚合
+    engagement_totals = {"likes": 0, "comments": 0, "shares": 0, "views": 0}
+    titles: list[str] = []
+
+    for r in all_results:
+        meta = r.get("social_meta", {})
+        if isinstance(meta, dict):
+            engagement_totals["likes"] += meta.get("likes", meta.get("upvotes", meta.get("attitudes_count", 0)))
+            engagement_totals["comments"] += meta.get("comments", meta.get("num_comments", 0))
+            engagement_totals["shares"] += meta.get("shares", meta.get("retweets", meta.get("reposts_count", 0)))
+            engagement_totals["views"] += meta.get("views", meta.get("play_count", 0))
+        # 收集标题用于话题提取
+        title = r.get("title", "")
+        if title:
+            titles.append(title)
+
+    # 中英文兼容的话题提取
+    top_topics_list = _extract_topics(titles, top_k=10)
+    elapsed = int((time.time() - t0) * 1000)
+
+    return {
+        "query": query,
+        "mode": "social-sentiment",
+        "platforms": platforms,
+        "total_posts": len(all_results),
+        "platform_breakdown": {p: len(r) for p, r in platform_results.items()},
+        "engagement_totals": engagement_totals,
+        "top_topics": top_topics_list,
+        "cross_platform_posts": [
+            {
+                "platform": r.get("source", ""),
+                "title": r.get("title", "")[:100],
+                "url": r.get("url", ""),
+                "snippet": r.get("snippet", "")[:200],
+                "social_meta": r.get("social_meta", {}),
+            }
+            for r in all_results[:15]
+        ],
+        "engines_used": sorted(engines_used),
+        "errors": social_errors,
+        "elapsed_ms": elapsed,
+    }
+
+
+def aggregate_social_sentiment(query: str, platforms: list[str],
+                               platform_results: dict[str, list]) -> dict[str, Any]:
+    """社交舆情聚合（MCP argo_social_search mode=sentiment 用）。
+
+    输入为按平台分组的已抓取帖子（social_engines 输出，含 social_meta），
+    聚合互动数据汇总与平台分布，限流返回代表性帖子。原为 MCP 分发层的
+    内联逻辑，下沉到本模块避免逻辑放错层。
+    """
+    all_posts: list = []
+    for p in platforms:
+        all_posts.extend(platform_results.get(p) or [])
+
+    engagement_totals = {"likes": 0, "comments": 0, "reposts": 0, "shares": 0}
+    for post in all_posts:
+        meta = post.get("social_meta", {}) if isinstance(post, dict) else {}
+        engagement_totals["likes"] += meta.get("likes", 0) or meta.get("like_count", 0) or 0
+        engagement_totals["comments"] += meta.get("comments", 0) or 0
+        engagement_totals["reposts"] += meta.get("reposts", 0) or 0
+        engagement_totals["shares"] += meta.get("shares", 0) or 0
+
+    return {
+        "query": query,
+        "platforms": platforms,
+        "platform_breakdown": {p: len(platform_results.get(p) or []) for p in platforms},
+        "total_posts": len(all_posts),
+        "engagement_totals": engagement_totals,
+        "posts": all_posts[:30],  # 限流
+    }
+
+
+def _extract_topics(titles: list[str], top_k: int = 10) -> list[dict]:
+    """从标题列表提取话题（中英文兼容）。
+
+    英文用 split() 分词，中文用字符级 bigram。
+    过滤单字停用词，返回 top_k 个话题。
+    """
+    import re
+    from collections import Counter
+
+    chinese_stopwords = set(
+        '的了是在和我你她他它这就也不很但而或如果因为所以而且或者虽然但是可以应该需要已经正在'
+        '之前之后时候地方问题工作生活东西事情时间今天昨天明天今年去年明年个些吗呢啊吧哦嗯哈'
+        '呀嘛哪谁什么怎么多少几样种类下上里中后前时来回过开给让把被对从向比跟和与及当比'
+        '如例包括相关关于根据通过进行使用作为成为具有属于位于来自获得达到实现完成产生形成'
+        '存在发生发展提供包含涉及适用于'
+    )
+
+    topic_counter = Counter()
+
+    for title in titles:
+        # 英文分词（仅保留长度 >= 3 的词）
+        en_words = re.findall(r'[a-zA-Z]{2,}', title.lower())
+        for w in en_words:
+            if len(w) >= 3:
+                topic_counter[w] += 1
+
+        # 中文 bigram（连续 2 个中文字符）
+        chinese_chars = re.findall(r'[一-鿿]', title)
+        for i in range(len(chinese_chars) - 1):
+            bigram = chinese_chars[i] + chinese_chars[i + 1]
+            if bigram[0] not in chinese_stopwords and bigram[1] not in chinese_stopwords:
+                topic_counter[bigram] += 1
+
+    return [{"topic": t, "mentions": c} for t, c in topic_counter.most_common(top_k)]
+
+
+def _print_social_report(report: dict):
+    """打印社交舆情报告"""
+    print(f"\n{'='*60}")
+    print(f"社交舆情分析：{report['query']}")
+    print(f"{'='*60}")
+    print(f"平台：{', '.join(report['platforms'])} | 引擎：{', '.join(report['engines_used'])}")
+    print(f"抓取帖子：{report['total_posts']} | 耗时：{report['elapsed_ms']}ms")
+    print()
+    print("── 平台分布 ──")
+    for platform, count in report["platform_breakdown"].items():
+        print(f"  {platform}: {count} 条")
+    print()
+    print("── 互动数据汇总 ──")
+    eng = report["engagement_totals"]
+    print(f"  点赞/投票：{eng['likes']:,} | 评论：{eng['comments']:,} | 转发：{eng['shares']:,} | 观看：{eng['views']:,}")
+    print()
+    print("── 高频讨论话题 ──")
+    for topic in report["top_topics"]:
+        print(f"  「{topic['topic']}」 ({topic['mentions']} 次)")
+    print()
+    print("── 代表性内容 ──")
+    for post in report["cross_platform_posts"][:5]:
+        meta = post.get("social_meta", {})
+        engagement = ""
+        if isinstance(meta, dict):
+            likes = meta.get("likes", meta.get("upvotes", meta.get("attitudes_count", 0)))
+            comments = meta.get("comments", meta.get("num_comments", 0))
+            engagement = f" | 👍{likes} 💬{comments}"
+        print(f"  [{post['platform']}] {post['title'][:60]}{engagement}")
+        print(f"    {post['url']}")
+        print()