argo-search 1.0.1 → 2.9.8

package/scripts/fact_crosscheck.py

@@ -0,0 +1,230 @@
+#!/usr/bin/env python3
+"""fact_crosscheck.py — 数字交叉验证（方案 A）。
+
+问题：多源搜索结果中，同一数字可能不一致（如「营收 100 亿」vs「营收 120 亿」），
+Agent 无法判断哪个可信。本模块用纯 stdlib 做本地对比，零外部服务。
+
+设计约束：
+  - 零外部依赖，只用 stdlib（re）
+  - 纯函数，无全局状态
+  - 输出：{has_conflict, conflict(别名), indicator, sources, values, suggestion}
+    ——sources[i] 与 values[i] 同序配对（第 i 个值来自第 i 个源）
+
+防复发说明（2026-10-16 重建）：
+  旧文件因多次 Edit 追加 _CORE_INDICATORS 导致 2420 行、集合未闭合、
+  尾部函数体丢失。本文件把指标词表收敛到最小核心（~30 词，去重），
+  全文件 <200 行，受 test_module_size_gate.py（1000 行上限）约束。
+"""
+from __future__ import annotations
+
+import re
+from typing import Any
+
+# 数字提取正则：匹配「数字 + 单位」模式
+_NUMBER_RE = re.compile(
+    r"(\d+(?:\.\d+)?)\s*"
+    r"(亿|万|千|百|元|美元|股|%|％|倍|年|月|日|个|家|人|吨|克|千克|米|千米|平方|立方)?"
+)
+
+# 核心指标词表：唯一来源 backends/text_lexicon.json（经 text_lexicon 延迟加载）。
+# 原则：宁缺勿滥——未知词走 ""（无指标），不误报；新增词改 JSON + 补测试用例，不改代码。
+def _core_indicators() -> frozenset[str]:
+    from text_lexicon import core_indicators
+    return core_indicators()
+
+# 时间/口径范围词：不同口径的数字不得直接对比（防「生涯 350 vs 单季 15」误报）。
+_CAREER_MARKERS = ("职业生涯", "生涯", "历史总", "累计", "总数")
+_YEAR_RE = re.compile(r"(19|20)\d{2}")
+_SEASON_RE = re.compile(r"\d{4}\s*赛季|赛季|单季|本季")
+
+
+def _normalize_value(value: float, unit: str) -> float:
+    """归一化到同一量级。未知单位原样返回。"""
+    if unit == "亿":
+        return value * 1e8
+    if unit == "万":
+        return value * 1e4
+    if unit == "千":
+        return value * 1e3
+    if unit == "百":
+        return value * 1e2
+    return float(value)
+
+
+def _extract_indicator(text: str) -> str:
+    """从文本中提取第一个命中的核心指标，无命中返回 ""。长词优先。"""
+    if not text:
+        return ""
+    for ind in sorted(_core_indicators(), key=len, reverse=True):
+        if ind and ind in text:
+            return ind
+    return ""
+
+
+def _extract_scope(text: str) -> str:
+    """提取口径范围，用于判断两个数字是否可比。
+
+    返回：career / 年份如 "2024" / season / ""（无口径信息）。
+    """
+    if not text:
+        return ""
+    if any(m in text for m in _CAREER_MARKERS):
+        return "career"
+    m = _YEAR_RE.search(text)
+    if m:
+        return m.group(0)
+    if _SEASON_RE.search(text):
+        return "season"
+    return ""
+
+
+def _indicator_near(text: str, start: int, end: int, window: int = 12) -> str:
+    """取数字邻近窗口内的指标（防标题词污染片段词）。
+
+    例：标题「进球总数」+ 片段「助攻 350 次」→ 数字 350 邻近的是「助攻」，
+    不得取标题里的「进球」。无命中返回 ""（宁缺勿滥，不误报）。
+    """
+    lo = max(0, start - window)
+    hi = min(len(text), end + window // 2)
+    win = text[lo:hi]
+    num_pos = start - lo  # 数字在窗口内的偏移
+    best = ""
+    best_key = None  # (距离, -长度)
+    for ind in _core_indicators():
+        if not ind:
+            continue
+        pos = win.find(ind)
+        while pos != -1:
+            dist = abs(pos - num_pos)
+            key = (dist, -len(ind))
+            if best_key is None or key < best_key:
+                best_key = key
+                best = ind
+            pos = win.find(ind, pos + 1)
+    return best
+
+
+def _extract_numbers(text: str) -> list[dict[str, Any]]:
+    """从文本提取数字列表，每项 {value, unit, raw, indicator, scope}。"""
+    if not text:
+        return []
+    scope = _extract_scope(text)
+    out: list[dict[str, Any]] = []
+    for m in _NUMBER_RE.finditer(text):
+        try:
+            value = float(m.group(1))
+        except ValueError:
+            continue
+        unit = m.group(2) or ""
+        # 跳过年份/日期数字（防「2024 营收」里的 2024 参与数值对比）
+        if unit in ("年", "月", "日") and 1900 <= value <= 2100:
+            continue
+        if not unit and 1900 <= value <= 2100 and len(m.group(1)) == 4:
+            continue
+        indicator = _indicator_near(text, m.start(), m.end())
+        out.append({
+            "value": value,
+            "unit": unit,
+            "raw": m.group(0),
+            "indicator": indicator,
+            "scope": scope,
+        })
+    return out
+
+
+def _values_conflict(vals: list[float], rel_tol: float = 0.05) -> bool:
+    """归一化后最大值与最小值相对差异超过 rel_tol 即判冲突。"""
+    if len(vals) < 2:
+        return False
+    lo, hi = min(vals), max(vals)
+    base = max(abs(hi), 1e-9)
+    return (hi - lo) / base > rel_tol
+
+
+def _comparable_buckets(
+    items: list[tuple[float, str, str]],
+) -> list[list[tuple[float, str]]]:
+    """把 (值, scope, 源) 列表切成可比桶。
+
+    口径兼容规则（docstring 口径，唯一实现）：
+      - 有 scope 的只与同 scope 比；
+      - scope 为空（无口径信息）的与每个有口径组分别比；
+      - 全无 scope 时合成一桶互相比。
+
+    返回的每桶是 [(值, 源)]，桶内同序配对。
+    """
+    scoped: dict[str, list[tuple[float, str]]] = {}
+    for norm, scope, src in items:
+        scoped.setdefault(scope or "__none__", []).append((norm, src))
+    nones = scoped.get("__none__", [])
+    buckets: list[list[tuple[float, str]]] = []
+    for scope, lst in scoped.items():
+        if scope != "__none__":
+            buckets.append(lst + nones)
+    if nones and not buckets:
+        buckets.append(nones)
+    return buckets
+
+
+def crosscheck_numbers(
+    query: str, candidates: list[dict[str, Any]]
+) -> dict[str, Any]:
+    """交叉验证候选中的数字是否矛盾。
+
+    可比条件（同时满足才比较）：
+      1. 候选数字的 indicator 相同且非空；
+      2. 若 query 能提取到指标，只比较与 query 同指标的数字
+        （防「查进球、片段里助攻 350 vs 15」误报）；
+      3. scope 兼容：至少一方无口径信息，或双方口径相同。
+         career 与具体年份/赛季视为不兼容。
+
+    Returns:
+        {has_conflict, conflict(别名), sources, values, suggestion, indicator}
+    """
+    empty: dict[str, Any] = {
+        "has_conflict": False,
+        "conflict": False,
+        "sources": [],
+        "values": [],
+        "suggestion": "",
+        "indicator": "",
+    }
+    if not candidates:
+        return empty
+
+    query_ind = _extract_indicator(query or "")
+
+    # 按 (indicator) 分组收集
+    groups: dict[str, list[tuple[float, str, str]]] = {}
+    for c in candidates:
+        text = f"{c.get('title', '')} {c.get('snippet', '')}"
+        for n in _extract_numbers(text):
+            ind = n["indicator"]
+            if not ind:
+                continue
+            if query_ind and ind != query_ind:
+                continue
+            norm = _normalize_value(n["value"], n["unit"])
+            groups.setdefault(ind, []).append(
+                (norm, n["scope"], str(c.get("source", "")))
+            )
+
+    for ind, items in groups.items():
+        for bucket in _comparable_buckets(items):
+            vals = [v for v, _ in bucket]
+            if _values_conflict(vals):
+                srcs = [s for _, s in bucket]
+                sug = (
+                    f"同一指标「{ind}」下多源数值分歧 "
+                    f"（{min(vals):g} ~ {max(vals):g}），建议 fetch 原文核对口径后再引用。"
+                )
+                return {
+                    "has_conflict": True,
+                    "conflict": True,
+                    "sources": srcs,
+                    "values": vals,
+                    "suggestion": sug,
+                    "indicator": ind,
+                }
+
+    return empty

package/scripts/fetch.py

@@ -7,11 +7,10 @@ fetch.py — 轻量页面抓取（纯标准库）
 
 from __future__ import annotations
 
-import re
 import urllib.request
 from html.parser import HTMLParser
 from typing import Any
-from urllib.parse import urlparse
+from net_proxy import open_url  # 出口调度唯一入口（issue #13 同类修复）
 
 
 class ContentExtractor(HTMLParser):
@@ -55,12 +54,20 @@ def fetch_page(url: str, max_chars: int = 3000, timeout: int = 8,
         timeout: 超时秒数
         raw: True 时返回原始 HTML（用于 extract/crawl_sitemap 等需要 HTML 的场景）
     """
+    try:
+        from url_safety import check_url
+        ok, reason = check_url(url)
+        if not ok:
+            return {"url": url, "content": "", "html": "", "length": 0,
+                    "success": False, "error": f"URL 被 SSRF 防护拦截: {reason}"}
+    except ImportError:
+        pass
     try:
         req = urllib.request.Request(url, headers={
             "User-Agent": "unified-search/2.5 (+local-research)",
             "Accept": "text/html,application/xhtml+xml,application/xml",
         })
-        with urllib.request.urlopen(req, timeout=timeout) as resp:
+        with open_url(req, timeout=timeout) as resp:
             content_type = resp.headers.get("Content-Type", "")
             if "pdf" in content_type:
                 return {"url": url, "content": "", "html": "", "length": 0, "success": False, "error": "PDF not supported"}
@@ -92,17 +99,23 @@ def fetch_page(url: str, max_chars: int = 3000, timeout: int = 8,
 
 def fetch_pages_parallel(urls: list[str], max_chars: int = 3000,
                          timeout: int = 8, max_workers: int = 3) -> list[dict[str, Any]]:
-    """并行抓取多个页面。"""
-    from concurrent.futures import ThreadPoolExecutor, as_completed
+    """并行抓取多个页面。总时限 timeout+3 秒，到点未完成的页面以失败结果占位返回。"""
+    from bounded_run import run_bounded, TaskError
+
+    if not urls:
+        return []
+
+    def _fail(url: str) -> dict:
+        return {"url": url, "content": "", "length": 0, "success": False}
 
+    finished, unfinished = run_bounded(
+        urls, lambda u: fetch_page(u, max_chars, timeout),
+        timeout + 3, max_workers=min(len(urls), max_workers))
     results = []
-    with ThreadPoolExecutor(max_workers=min(len(urls), max_workers)) as ex:
-        futures = {ex.submit(fetch_page, url, max_chars, timeout): url for url in urls}
-        for fut in as_completed(futures, timeout=timeout + 3):
-            try:
-                results.append(fut.result())
-            except Exception:
-                results.append({"url": futures[fut], "content": "", "length": 0, "success": False})
+    for url, page in finished:
+        results.append(_fail(url) if isinstance(page, TaskError) else page)
+    for url in unfinished:
+        results.append(_fail(url))
     return results
 
 

package/scripts/fetch_browser.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+"""fetch_browser.py — fetch 降档链第二级B：Chrome CDP 浏览器档
+
+从 fetch_v3 拆出（fetch_v3 在模块体量门禁的祖父清单里，只能减不能增；
+浏览器档是自洽的一级，与 chrome_cdp / browser_auth 同域）。
+
+职责：
+  - 系统 Chrome（headless）渲染取页 + Hound 风格 actions 交互
+  - 登录态车道（A2）：auth_profile 传入 browser_auth 持久 profile 时带会话
+    抓取，结果标 login_state_used / cache_eligible=False——cache.assert_cacheable
+    据此拒绝该载荷进公共缓存；cookie 由 Chrome 自持加密，argo 不读取
+"""
+
+from __future__ import annotations
+
+
+def _fail(url: str, error: str) -> dict:
+    """失败结果（与 fetch_v3._make_result 的空内容形状逐字段一致）。"""
+    return {
+        "url": url, "content": "", "html": "", "title": "",
+        "length": 0, "success": False, "error": error,
+        "fetch_method": "browser",
+    }
+
+
+def browser_fetch(url: str, max_chars: int = 8000, timeout: float = 15.0,
+                  actions: list[dict] | None = None,
+                  auth_profile: str | None = None) -> dict:
+    """使用 Chrome CDP 驱动抓取（支持页面交互）。
+
+    auth_profile：browser_auth 的持久 profile 路径。传入时 Chrome 带登录态
+    启动（user_data_dir 指向持久目录；chrome_cdp 对持久 profile 不做清理）。
+    """
+    try:
+        from chrome_cdp import ChromeCDP
+    except ImportError:
+        return _fail(url, "chrome_cdp not available")
+
+    try:
+        cdp = ChromeCDP(auto_start=True, user_data_dir=auth_profile)
+    except Exception as e:
+        err = f"Chrome failed to start: {str(e)[:100]}"
+        if auth_profile:
+            err += "（登录态 profile 启动失败：关掉占用它的 Chrome，或 argo auth logout 后重登）"
+        return _fail(url, err)
+
+    try:
+        # 导航（真 networkidle：CDP Network 事件计数；SPA 永不 idle 时超时放行）
+        cdp.navigate(url, wait_until="networkidle")
+
+        # 执行页面交互序列（Hound actions 等价能力）
+        if actions:
+            cdp.execute_actions(actions)
+
+        # 提取内容
+        html = cdp.get_html()
+        text = cdp.get_text()
+        title = cdp.get_title()
+
+        result = {
+            "url": url,
+            "content": text[:max_chars] if text else "",
+            "html": html[:max_chars * 2] if html else "",
+            "title": title or "",
+            "length": len(text) if text else 0,
+            "success": bool(text),
+            "error": None if text else "empty content",
+            "fetch_method": "chrome_cdp",
+        }
+        if auth_profile:
+            # 登录态 provenance：与 ego-browser 通道的 envelope 语义同源
+            # （candidate_envelope：login_state_used → visibility=authenticated）
+            result["login_state_used"] = True
+            result["cache_eligible"] = False
+            try:
+                from browser_auth import touch
+                touch(url)
+            except Exception:
+                pass
+        return result
+    except Exception as e:
+        return _fail(url, f"CDP error: {str(e)[:100]}")
+    finally:
+        try:
+            cdp.stop()
+        except Exception:
+            pass

package/scripts/fetch_jsrun.py

@@ -0,0 +1,122 @@
+#!/usr/bin/env python3
+"""fetch_jsrun.py — fetch 降级链第一级E：V8 JS 执行（js-run 轻量车道）。
+
+从 fetch_v3.py 拆出（2026-10-05）：jsrun 段独立成模块，fetch_v3.py
+行数回到祖父文件上限内。职责：用 V8 沙箱执行「环境探测 + 纯计算」型
+挑战页 JS，尝试获取 clearance cookie 后重试请求。失败时返回空结果，
+由调用方降级到 tinyfish/CDP。
+"""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+from engine_env import env_flag
+
+
+def _jsrun_enabled() -> bool:
+    """js-run 车道开关：ARGO_FETCH_JSRUN=0 关闭，默认开启。"""
+    return env_flag("ARGO_FETCH_JSRUN")
+
+
+def _jsrun_challenge_fetch(url: str, html: str, max_chars: int = 8000,
+                           timeout: float = 8.0,
+                           backoff_budget_s: float | None = None) -> dict:
+    """用 js-run 执行挑战页 JS，尝试获取 clearance cookie 后重试请求。
+
+    只处理「环境探测 + 纯计算」型挑战脚本（v0 面）。失败时返回空结果，
+    由调用方降级到 tinyfish/CDP。
+
+    重试请求用 curl_cffi Chrome 指纹 impersonate（站点按 TLS 指纹风控，
+    裸 Python 指纹被静默拦截）。退避重试 3 次（间隔 5s/10s）。
+    `backoff_budget_s`（秒）给整段重试设墙钟上界：传入时退避睡眠与后续
+    尝试都不得越过它——此前 sleep(5)+sleep(10) 不受链级预算约束，cookie
+    拿到而三连败时可在 fetch 链内部多耗 ~30s（2026-10-06 审查定位）。
+    """
+    import re as _re
+    import time as _time
+
+    # 提取 <script> 内容（挑战页的通行证计算脚本）
+    scripts = _re.findall(r'<script[^>]*>(.*?)</script>', html, _re.DOTALL | _re.IGNORECASE)
+    if not scripts:
+        return {}
+
+    # 只取含环境探测/计算特征的脚本（过滤掉统计/广告等无关脚本）
+    challenge_scripts = []
+    for s in scripts:
+        if _re.search(r'navigator|document\.cookie|btoa|atob|setTimeout|__INITIAL_STATE__|challenge', s, _re.IGNORECASE):
+            challenge_scripts.append(s)
+    if not challenge_scripts:
+        return {}
+
+    try:
+        _skill_dir = Path(__file__).parent.parent / "sub-skills" / "js-run" / "scripts"
+        if str(_skill_dir) not in sys.path:
+            sys.path.insert(0, str(_skill_dir))
+        from jsrun import JsRun
+    except (ImportError, OSError):
+        return {}
+
+    try:
+        with JsRun() as jr:
+            for script in challenge_scripts:
+                try:
+                    jr.run(script, timeout_ms=3000)
+                except Exception:
+                    continue
+            # 推进逻辑时间（挑战脚本常用 setTimeout 延迟发通行证）
+            jr.advance(5000)
+            cookie = jr.get_cookie()
+
+        if not cookie or len(cookie) < 10:
+            return {}
+
+        # 用 clearance cookie 重试原请求（curl_cffi Chrome 指纹 + 退避重试）
+        from curl_cffi import requests as _cr
+
+        # 退避墙钟上界：到点不再发起后续尝试、睡眠也不越过（见 docstring）
+        _bo_deadline = (_time.monotonic() + backoff_budget_s
+                        if backoff_budget_s is not None and backoff_budget_s > 0
+                        else None)
+        session = _cr.Session(impersonate="chrome")
+        try:
+            last_err = ""
+            for attempt in range(1, 4):
+                if _bo_deadline is not None and _time.monotonic() >= _bo_deadline:
+                    break
+                try:
+                    resp = session.get(
+                        url,
+                        headers={"Cookie": cookie},
+                        timeout=timeout,
+                    )
+                    if resp.status_code == 200 and len(resp.text.strip()) >= 100:
+                        return {
+                            "url": url,
+                            "content": resp.text[:max_chars],
+                            "html": "",
+                            "title": "",
+                            "length": len(resp.text),
+                            "success": True,
+                            "error": None,
+                            "fetch_method": "jsrun_challenge",
+                            "jsrun_cookie": cookie[:100],
+                        }
+                    last_err = f"HTTP {resp.status_code}, {len(resp.text)} 字节"
+                except Exception as e:
+                    last_err = str(e)[:200]
+                if attempt < 3:
+                    _sleep = 5 * attempt  # 5s, 10s
+                    if _bo_deadline is not None:
+                        _sleep = min(_sleep, _bo_deadline - _time.monotonic())
+                    if _sleep <= 0:
+                        break
+                    _time.sleep(_sleep)
+            return {}
+        finally:
+            # curl 句柄随 Session 持有：MCP 常驻进程里不关会逐次累积
+            session.close()
+    except Exception:
+        pass
+    return {}