argo-search 1.0.1 → 2.9.7

package/scripts/plan.py

@@ -0,0 +1,422 @@
+#!/usr/bin/env python3
+"""
+plan.py — 离线搜索计划（吸纳 yichen-unified-search 契约）
+
+目标：
+  - 不联网，只产出 status / authorization / steps / limitations
+  - 分流 input_kind：keyword | url-seed | known-url
+  - 与 route_query 保持一致，供 deep/research 路径挂载元数据
+
+执行分层（产品纪律，禁止日常「先确认再搜」）：
+  - daily：fast/auto/budget + 非 deep depth → 直搜，不挂 plan，不等用户确认
+  - professional：mode=deep 或 depth=deep → 直搜同时附加 plan 元数据 + 核验提示
+  - deep_research：research.py → 先离线 plan 一次（非循环），再拆子查询采集
+
+反死循环保证：
+  - build_plan 永不 import/调用 super_search 或 deep_research
+  - 仅依赖 route_query（纯决策，无网络）
+  - 调用方最多 build_plan 一次，不得 plan→search→plan 递归
+
+用法：
+  python3 scripts/plan.py "检索词" --mode auto --json
+  python3 scripts/plan.py "https://example.com/a" --input-kind known-url --json
+  python3 scripts/plan.py "搜索引用 https://example.com/a 的报道" --input-kind url-seed --json
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import os
+import re
+import sys
+from typing import Any, Literal
+from urllib.parse import urlparse, parse_qsl, urlencode, urlunparse
+from cli_io import dumps
+
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+sys.path.insert(0, SCRIPT_DIR)
+
+_URL_RE = re.compile(r"https?://[^\s<>\"']+", re.I)
+
+ExecutionTier = Literal["daily", "professional", "deep_research"]
+
+
+def execution_tier(
+    mode: str = "auto",
+    depth: str = "fast",
+    context: str = "search",
+) -> ExecutionTier:
+    """判定执行分层。
+
+    context:
+      - search：普通 argo_search / search.py
+      - research：deep_research / argo_research
+    """
+    ctx = (context or "search").lower()
+    if ctx in ("research", "deep_research", "argo_research"):
+        return "deep_research"
+    m = (mode or "auto").lower()
+    d = (depth or "fast").lower()
+    if m == "deep" or d == "deep":
+        return "professional"
+    return "daily"
+
+
+def should_attach_plan(
+    mode: str = "auto",
+    depth: str = "fast",
+    context: str = "search",
+    plan_only: bool = False,
+) -> bool:
+    """是否在 super_search 响应中附加完整 plan 对象。
+
+    - plan_only：显式只要计划 → True
+    - daily：False（避免热路径噪音与「先确认」暗示）
+    - professional（mode/depth=deep）：True（元数据，不阻断执行）
+    - deep_research：False（由 research.py 顶层挂一次 plan，子查询不重复）
+    """
+    if plan_only:
+        return True
+    tier = execution_tier(mode, depth, context)
+    return tier == "professional"
+
+
+def requires_user_confirmation(
+    mode: str = "auto",
+    depth: str = "fast",
+    context: str = "search",
+    input_kind: str = "keyword",
+    plan_status: str | None = None,
+) -> bool:
+    """日常搜索永不要求用户确认。
+
+    仅下列情况可能为 True（且仍非交互阻塞，只是 status 提示）：
+      - needs_authorization（显式站内登录敏感查询）
+    known-url handoff 是工具分流，不是「请确认后搜索」。
+    """
+    if plan_status == "needs_authorization":
+        return True
+    # daily / professional / research 的 keyword 与 url-seed 均不弹确认
+    _ = (mode, depth, context, input_kind)
+    return False
+
+
+def extract_urls(text: str) -> list[str]:
+    if not text:
+        return []
+    return list(dict.fromkeys(_URL_RE.findall(text)))
+
+
+def is_pure_url(text: str) -> bool:
+    t = (text or "").strip()
+    if not t:
+        return False
+    return bool(re.fullmatch(r"https?://[^\s]+", t, flags=re.I))
+
+
+def _is_single_tweet(url: str) -> bool:
+    """该 URL 是否单条推文（有免登录抽取通道）。
+
+    推文 ID 提取复用引擎侧实现（engines_builders_intl.extract_tweet_id），
+    不在这里重写一套 URL 正则——两处解析必然漂移。懒加载，只在 known-url
+    交接分支里发生（罕见路径，约 39ms）。
+    """
+    try:
+        from engines_builders_intl import extract_tweet_id
+        return extract_tweet_id(url) is not None
+    except Exception:
+        return False  # engines_builders_intl 不可用或解析失败：安全退回「非 tweet URL」
+
+
+def canonicalize_url(url: str) -> str:
+    """URL 归一化（薄转发到 url_canon 唯一来源）。
+
+    本函数曾自带一份追踪参数表与归一规则，与 search/candidate_envelope/
+    research_dossier 的实现互不一致；现统一到 url_canon，保证同一条链接
+    在计划层与融合层归一成同一个键。
+    """
+    from url_canon import canonical_url as _impl
+    return _impl(url)
+
+
+def classify_input_kind(query: str, explicit: str = "auto") -> str:
+    """keyword | url-seed | known-url。
+
+    auto 规则（安全默认）：
+      - 纯 URL → known-url（应 fetch/归档，不进多引擎热搜）
+      - 含 URL 且含「搜索/引用/讨论/关联」等发现意图 → url-seed
+      - 含 URL 其余 → known-url（安全 handoff）
+      - 无 URL → keyword
+    """
+    if explicit and explicit != "auto":
+        return explicit
+    urls = extract_urls(query)
+    if not urls:
+        return "keyword"
+    if is_pure_url(query):
+        return "known-url"
+    discover = re.search(
+        r"(搜索|检索|引用|讨论|关联|提到|关于该|关于此|related|discuss|mention|site:)",
+        query or "",
+        re.I,
+    )
+    return "url-seed" if discover else "known-url"
+
+
+def build_plan(
+    query: str,
+    mode: str = "auto",
+    depth: str = "fast",
+    max_results: int = 5,
+    engine: str = "auto",
+    input_kind: str = "auto",
+    login_approved: bool = False,
+    context: str = "search",
+) -> dict[str, Any]:
+    """离线计划。永不调用搜索后端；永不回调 super_search/research。"""
+    kind = classify_input_kind(query, input_kind)
+    urls = extract_urls(query)
+    limitations: list[str] = []
+    steps: list[dict[str, Any]] = []
+    tier = execution_tier(mode, depth, context)
+
+    # ── known-url：交接 fetch，不联网搜索 ──
+    if kind == "known-url":
+        target = urls[0] if urls else query.strip()
+        # 单条推文有免登录抽取通道（twitter_syndication 走 syndication 接口）：
+        # 只提示 argo_fetch 会让用户以为得靠浏览器/登录才拿得到，而这条通道
+        # 恰好是为「按 URL 取单条」设计的。能力可发现性属于交接内容的一部分。
+        suggested = ["argo_fetch", "argo_pdf"]
+        extra_limits: list[str] = []
+        if _is_single_tweet(target):
+            # 两条免登录通道分工：`argo tweet` 吸收整份内容（正文全文 + 串/引用/
+            # 转发 + 可选媒体下载），twitter_syndication 只要一条轻量结果。
+            # 只提 argo_fetch 会让人以为得靠浏览器/登录才拿得到。
+            suggested.insert(0, "argo_tweet")
+            suggested.insert(1, "argo_search(engine=twitter_syndication)")
+            extra_limits.append(
+                "Single tweet: `argo tweet <url|id>` packages it fully without login "
+                "(full text, thread/quotes/reposts, optional --media download); "
+                "argo_search with engine=twitter_syndication returns a lightweight "
+                "single-post result."
+            )
+        return {
+            "schema_version": "1.0",
+            "status": "handoff_required",
+            "authorization": "not_applicable",
+            "input_kind": kind,
+            "query": query,
+            "execution_tier": tier,
+            "requires_confirmation": False,
+            "handoff": {
+                "action": "fetch_or_archive",
+                "url": target,
+                "canonical_url": canonicalize_url(target),
+                "suggested_tools": suggested,
+            },
+            "route": None,
+            "steps": [
+                {
+                    "action": "handoff_fetch",
+                    "url": target,
+                    "reason": "known-url is not a multi-engine search query",
+                }
+            ],
+            "limitations": [
+                "Known URL reading/download/archive is outside multi-engine search.",
+                "Use argo_fetch / argo_pdf for content; use --input-kind url-seed only to discover related public pages.",
+                *extra_limits,
+            ],
+            "mode": mode,
+            "depth": depth,
+        }
+
+    # ── url-seed：URL 仅作发现线索 ──
+    search_query = query
+    if kind == "url-seed":
+        limitations.append(
+            "URL is a discovery seed only; this plan does not read/download/archive the seed URL itself."
+        )
+        # 若用户未写 site: 且是纯发现句，可提示但默认保留原 query 交给引擎
+        if urls and "site:" not in query.lower():
+            host = urlparse(urls[0]).netloc
+            if host:
+                limitations.append(
+                    f"Consider site-index discovery around host={host}; not equal to native full-site index."
+                )
+
+    # ── 社交/登录敏感词：仅声明授权需求，不阻断公共引擎 ──
+    ql = (query or "").lower()
+    needs_login = any(
+        s in ql for s in ("小红书站内", "抖音站内", "chrome 登录", "登录态")
+    )
+    if needs_login and not login_approved:
+        return {
+            "schema_version": "1.0",
+            "status": "needs_authorization",
+            "authorization": "explicit_current_turn_login_required",
+            "input_kind": kind,
+            "query": query,
+            "execution_tier": tier,
+            "requires_confirmation": True,
+            "route": {
+                "platform": "social_native",
+                "backend": "social_engines",
+                "mode": mode,
+                "reason": "native_login_gated_search",
+                "login_state_used": True,
+            },
+            "steps": [],
+            "limitations": [
+                "State platform, exact query, and limit before requesting login authorization.",
+                "One approval does not extend to other platforms or later turns.",
+            ],
+            "mode": mode,
+            "depth": depth,
+        }
+
+    # ── 正常 keyword / url-seed：保持一致 route_query（离线，无网络）──
+    try:
+        from route import route_query
+        decision = route_query(
+            search_query, engine_override=engine, mode=mode,
+            depth=depth, context=context,
+        )
+    except Exception as e:
+        return {
+            "schema_version": "1.0",
+            "status": "invalid_request",
+            "authorization": "not_applicable",
+            "input_kind": kind,
+            "query": query,
+            "execution_tier": tier,
+            "requires_confirmation": False,
+            "route": None,
+            "steps": [],
+            "limitations": [f"route_query failed offline: {type(e).__name__}"],
+            "mode": mode,
+            "depth": depth,
+        }
+
+    engines = decision.get("engines_combo") or decision.get("engines") or []
+    primary = decision.get("engine") or (engines[0] if engines else "auto")
+    domain = decision.get("domain")
+
+    steps.append({
+        "action": "route",
+        "engine": primary,
+        "engines_combo": engines,
+        "domain": domain,
+        "reason": decision.get("reason"),
+    })
+    steps.append({
+        "action": "search",
+        "query": search_query,
+        "max_results": max_results,
+        "mode": mode,
+        "depth": depth,
+        "engines": engines,
+    })
+    # 仅 professional / deep_research 追加核验步骤（日常不暗示「先确认」）
+    if tier in ("professional", "deep_research"):
+        steps.append({
+            "action": "optional_verify_top_k",
+            "k": min(3, max_results),
+            "note": "snippet is clue only; fetch before hard claims",
+        })
+        if tier == "deep_research":
+            steps.insert(0, {
+                "action": "decompose_then_collect",
+                "note": "research path: offline plan once → sub-queries → synthesize; no plan loop",
+            })
+            limitations.append(
+                "Deep research: plan is offline metadata only; execution continues without user confirmation."
+            )
+    else:
+        limitations.append(
+            "Daily tier: direct search; plan attach skipped; no user confirmation gate."
+        )
+
+    # 常见局限
+    if any(e.startswith("local_") for e in engines):
+        limitations.append("Local engines parse public pages; may be blocked by CAPTCHA/anti-bot.")
+    if "octen" in engines:
+        limitations.append("octen requires OCTEN_API_KEY; missing key yields empty results for that engine.")
+    if decision.get("parallel"):
+        limitations.append("Parallel multi-engine; early-stop may skip later engines when primary is sufficient.")
+    if max_results > 10:
+        limitations.append("Some backends clamp max_results ≤10; request may be truncated per engine.")
+
+    return {
+        "schema_version": "1.0",
+        "status": "ready",
+        "authorization": "not_required",
+        "input_kind": kind,
+        "query": query,
+        "search_query": search_query,
+        "execution_tier": tier,
+        "requires_confirmation": False,
+        "route": {
+            "platform": "web",
+            "backend": primary,
+            "engines_combo": engines,
+            "domain": domain,
+            "mode": mode,
+            "depth": depth,
+            "reason": decision.get("reason"),
+            "login_state_used": False,
+            "confidence": decision.get("confidence"),
+            "parallel": decision.get("parallel"),
+            "tfidf_scores": decision.get("tfidf_scores") or [],
+        },
+        "steps": steps,
+        "limitations": limitations,
+        "decision": {
+            "engine": primary,
+            "engines_combo": engines,
+            "domain": domain,
+            "elapsed_ms": decision.get("elapsed_ms"),
+        },
+        "mode": mode,
+        "depth": depth,
+    }
+
+
+def main() -> int:
+    p = argparse.ArgumentParser(description="Argo 离线搜索计划（不联网）")
+    p.add_argument("query", help="查询词或 URL")
+    p.add_argument("--mode", default="auto", choices=("fast", "auto", "deep", "budget"))
+    p.add_argument("--depth", default="fast", choices=("fast", "balanced", "deep"))
+    p.add_argument("--max-results", "-n", type=int, default=5)
+    p.add_argument("--engine", "-e", default="auto")
+    p.add_argument(
+        "--input-kind",
+        default="auto",
+        choices=("auto", "keyword", "url-seed", "known-url"),
+    )
+    p.add_argument("--login-approved", action="store_true")
+    p.add_argument(
+        "--context",
+        default="search",
+        choices=("search", "research"),
+        help="search=日常/专业分层；research=深度研究分层",
+    )
+    p.add_argument("--json", action="store_true", dest="json_output", default=True)
+    args = p.parse_args()
+    plan = build_plan(
+        args.query,
+        mode=args.mode,
+        depth=args.depth,
+        max_results=args.max_results,
+        engine=args.engine,
+        input_kind=args.input_kind,
+        login_approved=args.login_approved,
+        context=args.context,
+    )
+    print(dumps(plan))
+    return 0 if plan.get("status") in ("ready", "handoff_required", "needs_authorization") else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/scripts/postprocess_signals.py

@@ -0,0 +1,150 @@
+#!/usr/bin/env python3
+"""postprocess_signals.py — 加工段两个可选信号：TF-IDF 语义抛光 + 数字交叉验证。
+
+2026-10-16 从 search_pipeline.postprocess 提取（方案二梳理的「真源收敛」一步）。
+留在主管线里的代价是实测过的：+37 行把 search_pipeline 顶破祖父上限 1033，
+且每个特性自带 try/except + 环境开关，主管线的异常面被无谓放大。提取后：
+
+  - 主管线每个特性只留 3~4 行调用点（与 search_attribution / phrase_filter
+    同一形态，本文件也只留调用点的那条纪律在这里是反过来的应用）；
+  - 开关与它守护的功能同文件：ARGO_TFIDF_RERANK / ARGO_NUMBER_CROSSCHECK
+    不再散在主管线里，读代码的人在同一处看到「功能+门控+降级」；
+  - 失败各自降级：本模块的函数不向主管线抛异常，异常只留一行日志。
+
+两个入口都是「纯加工」签名：吃主管线的中间态，吐改写后的结果。
+"""
+from __future__ import annotations
+
+from typing import Any, Callable
+
+
+def apply_tfidf_polish(
+    query: str,
+    merged: list[dict[str, Any]],
+    max_results: int,
+    mode: str,
+    reranker_status: str,
+    rank_method: str,
+    log: Callable[[str], None] | None = None,
+) -> tuple[list[dict[str, Any]], str, bool]:
+    """精排降级时的零依赖 TF-IDF 语义抛光（防 SERP 壳链靠 RRF 高分回顶）。
+
+    返回 (merged, rank_method, ran)：
+      - ran=True：确实重排过，rank_method 已带 "+tfidf"/"tfidf" 标记；
+      - ran=False：未触发（开关关 / fast 档 / 结果不足 / 非降级态）或执行
+        失败。两者对调用方含义相同——本轮排序没有 TF-IDF 贡献——由调用方
+        决定 none 兜底；异常只留一行日志，不外抛。
+
+    mode 透传给 rerank_tfidf：fast/budget 档其内部直接跳过（零延迟）。
+    """
+    from search_rank import _RERANK_DEGRADED_STATUSES
+
+    # fast/budget 档先拦：rerank_tfidf 内部虽也会跳过，但那样 ran 会被误置
+    # True、rank_method 被误标 "+tfidf"——本轮实际没有任何重排贡献
+    # （2026-10-16 本模块测试锁住该语义）。
+    if mode in ("fast", "budget"):
+        return merged, rank_method, False
+    if not merged or len(merged) <= 1:
+        return merged, rank_method, False
+    if reranker_status not in _RERANK_DEGRADED_STATUSES:
+        return merged, rank_method, False
+    try:
+        from engine_env import env_flag
+        if not env_flag("ARGO_TFIDF_RERANK", True):
+            return merged, rank_method, False
+        from rerank_tfidf import rerank as _tfidf_rerank
+        merged = _tfidf_rerank(query, merged, top_k=max_results, mode=mode)
+        rank_method = (rank_method + "+tfidf") if rank_method != "none" else "tfidf"
+        return merged, rank_method, True
+    except Exception as e:
+        if log:
+            log(f"TF-IDF rerank 跳过: {type(e).__name__}")
+        return merged, rank_method, False
+
+
+def detect_number_conflict(
+    query: str,
+    merged: list[dict[str, Any]],
+    mode: str,
+    depth: str,
+    log: Callable[[str], None] | None = None,
+) -> dict[str, Any] | None:
+    """多源同指标数值分歧信号（与 fact_alignment 并列的轻量核验提示）。
+
+    仅 deep/auto 且结果 ≥3 时启用；只对比前 10 条（O(n) 正则扫描，<1ms）。
+    有分歧返回 {has_conflict, indicator, values, sources, suggestion}，
+    无分歧 / 未启用 / 执行失败返回 None（异常只留一行日志不外抛）。
+    values[i] 与 sources[i] 同序配对。
+    """
+    if not merged or len(merged) < 3 or mode in ("fast", "budget") or depth == "fast":
+        return None
+    try:
+        from engine_env import env_flag
+        if not env_flag("ARGO_NUMBER_CROSSCHECK", True):
+            return None
+        from fact_crosscheck import crosscheck_numbers
+        nc = crosscheck_numbers(query, merged[:10])
+        if nc.get("has_conflict"):
+            return {
+                "has_conflict": True,
+                "indicator": nc.get("indicator", ""),
+                "values": nc.get("values", []),
+                "sources": nc.get("sources", []),
+                "suggestion": nc.get("suggestion", ""),
+            }
+    except Exception as e:
+        if log:
+            log(f"数字交叉验证跳过: {type(e).__name__}")
+    return None
+
+
+# ── 查询词回声（2026-10-17 实锤）──────────────────────────────────────────────
+# 上游把整句当单词去查时，返回的是查询中某一个词的词典释义/同主题页。
+# 判据与理由在 query_signals.is_query_echo；这里只做「后处理侧」的接线：
+# 主管线（search_pipeline）只留调用点，逻辑不落在那里。
+
+
+def echo_recovery_needed(results: list[dict[str, Any]], query: str) -> bool:
+    """恢复链是否该因「查询词回声」触发——有结果、但整批是查询某一个词的
+    词典释义（判据见 query_signals.is_query_echo）。
+
+    零结果走 `if not merged` 的老路；这里补的是「有结果却是垃圾」：anysearch
+    超时后仅剩的 local_bing 回声会被原样交付（2026-10-17 实锤）。
+    """
+    try:
+        from query_signals import is_query_echo
+    except ImportError:
+        return False  # 判据不可用不拦（fail-open）
+    return bool(results) and is_query_echo(results, query)
+
+
+def drop_echo_items(results: list[dict[str, Any]],
+                   query: str) -> list[dict[str, Any]]:
+    """过滤查询词回声条目，让恢复链继续往「换引擎」走。
+
+    恢复链的 L1 放宽查询后，原引擎仍会返回同一批词典释义；若把它们算作
+    「恢复成功」，L3 换引擎永不执行（2026-10-17 实测 level_used=L1 而交付的
+    仍是回声）。
+
+    两段判据，缺一不可：
+      1. 整批先过 is_query_echo——它要求「≥3 条」正是为了不误伤一两条短标题
+         的正确答案（实测 "Paris capital" 对 "what is the capital of France"
+         只命中 capital 一个词，但那就是答案）；
+      2. 整批判定为回声后，再逐条剔除「只命中 1 个查询词」的条目——一批里
+         混着的好结果不该被连坐。
+    """
+    if not results:
+        return results
+    try:
+        from query_signals import is_query_echo
+        from tfidf_router import tokenize
+    except ImportError:
+        return results  # 判据不可用不过滤（fail-open）
+    if not is_query_echo(results, query):
+        return results
+    q_set = set(tokenize(query))
+    return [
+        r for r in results
+        if not (isinstance(r, dict)
+                and 0 < len(q_set & set(tokenize(r.get("title") or ""))) <= 1)
+    ]

package/scripts/pre-commit

@@ -0,0 +1,48 @@
+#!/bin/sh
+# argo 提交前快门禁（pre-commit hook）。
+#
+# 为什么需要（2026-10-16 流程洞实证）：并发代理往主管线接线时没跑测试——
+# 6 道门禁红 + 阻塞全量收集，故障在树里躺了幾小时才被人工发现。「改了没
+# 验证就提交」没有机制拦截，只靠自觉。本 hook 把拦截点放在提交层。
+#
+# 只跑快的六道（实测合计 ~12s）：static_lint（语法/重复键/未声明 env）、
+# module_size_gate（1000 行门禁与祖父登记）、usage_doc（ARGO_* 开关总表）、
+# consistency（版本/徽章/工具数五语对账）、routing_snapshot（路由决策漂移）、
+# context_budget + engine_catalog（常驻文档预算、源文档/死源/声明一致）。
+# 后两道是 2026-10-18 补的：f702ee2 把 SKILL.md 压到 8049B、4a71693 让
+# deepseek_search 成死源，两笔都过了 hook 却在全量回归里红——正是「改完没跑
+# 到那条门」的同一个洞。全量回归 ~5 分钟留给 CI 或手动（python3 -m pytest
+# tests/ -q），不进 hook——hook 慢于 20s 就会被用户 --no-verify 绕过，那是更
+# 坏的结局。
+#
+# 安装：bash scripts/install.sh 自动装（链式：不覆盖 privacy-guard 等既有
+# hook，装成 hooks/pre-commit-argo 并追一行调用）。手动：
+#   cp scripts/pre-commit "$(git rev-parse --git-common-dir)/hooks/pre-commit-argo"
+# 跳过（不推荐）：git commit --no-verify
+set -e
+
+# 仓库根：git 运行时 cwd 即工作树根；装在 hooks/ 下或被拷贝到别处时，
+# 脚本相对推导会指向 .git 目录（2026-10-17 实测踩中），故以 show-toplevel
+# 为准、脚本相对为兜底（原产地 scripts/ 形态手动跑也成立）。
+root=$(git rev-parse --show-toplevel 2>/dev/null || true)
+if [ -z "$root" ] || [ ! -d "$root/tests" ]; then
+    here=$(cd "$(dirname "$0")" && pwd)
+    root=$(cd "$here/.." && pwd)
+fi
+cd "$root" || exit 1
+
+PY=$(command -v python3 || command -v python || true)
+if [ -z "$PY" ]; then
+    echo "[pre-commit] 找不到 python3/python，跳过了门禁（应视为失败）" >&2
+    exit 1
+fi
+
+echo "[pre-commit] argo 快门禁（~12s；全量回归请手动跑 tests/）..."
+"$PY" -m pytest -q \
+    tests/test_static_lint_gate.py \
+    tests/test_module_size_gate.py \
+    tests/test_usage_doc_gates.py \
+    tests/test_consistency_gates.py \
+    tests/test_routing_snapshot.py \
+    tests/test_context_budget.py \
+    tests/test_engine_catalog.py