argo-search 1.0.1 → 2.9.8

package/scripts/caliber_isolation.py

@@ -0,0 +1,195 @@
+#!/usr/bin/env python3
+"""caliber_isolation.py — 口径隔离：不同口径的引擎结果不直接 merge。
+
+同口径引擎的结果才能 merge 到同一条目，不同口径的结果按优先级并列输出。
+口径分类表 + 分组逻辑是纯函数，与 search_rank.py 的调度/融合职责不同。
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+_CALIBER_GROUPS = {
+    "financial_data": frozenset({
+        "fred", "worldbank", "eurostat", "frankfurter", "fx_rate",
+        "eastmoney", "sina_quote", "tencent_quote", "yahoo_finance",
+        "sec_companyfacts", "finviz", "un_comtrade",
+        "em_flow", "em_miaoxiang", "tencent_kline", "jin10",
+    }),
+    "encyclopedia": frozenset({
+        "wikipedia", "baidu_baike", "wikidata",
+        "wikisource", "wikimedia_commons", "dnb",
+    }),
+    "social": frozenset({
+        "twitter", "weibo", "zhihu", "zhihu_global", "zhihu_user",
+        "xiaohongshu", "wechat_sogou", "bilibili_hot", "douyin_hot",
+        "weibo_hot", "toutiao_hot", "hackernews", "reddit",
+    }),
+    "code": frozenset({
+        "github", "stackoverflow", "so", "local_gitlab", "deps_dev",
+        "endoflife", "pypi", "npm", "crates",
+    }),
+    "news": frozenset({
+        "google_news", "bing_rss", "cls_telegraph", "em_global_news",
+        "people_daily",
+    }),
+    "academic": frozenset({
+        "arxiv", "local_pubmed", "google_scholar", "semantic_scholar",
+        "crossref", "openalex", "doaj", "core", "gdacs",
+        "clinicaltrials", "openfda", "uniprot", "rcsb_pdb",
+    }),
+    "media": frozenset({
+        "imdb", "tvmaze", "bangumi", "deezer",
+        "itunes", "listenbrainz", "figshare", "met_museum",
+    }),
+    "reference": frozenset({
+        "open_library", "weread", "douban_book", "gutenberg", "rfc_editor", "courtlistener", "hal", "eu_opendata",
+    }),
+}
+
+# 口径优先级：数值越小优先级越高，排序时高口径的结果排在前面
+_CALIBER_PRIORITY = {
+    "financial_data": 0,
+    "academic": 1,
+    "encyclopedia": 2,
+    "news": 3,
+    "social": 4,
+    "code": 5,
+    "media": 6,
+    "reference": 7,
+    "general": 8,
+}
+
+
+def caliber_of(engine: str) -> str:
+    """返回引擎的口径分类，未分类的引擎归为 general。"""
+    for caliber, engines in _CALIBER_GROUPS.items():
+        if engine in engines:
+            return caliber
+    return "general"
+
+
+def group_by_caliber(ranked_lists: list[list[dict[str, Any]]]) -> dict[str, list[list[dict[str, Any]]]]:
+    """按口径分组：同口径的引擎结果才能 merge。
+
+    分组单位是**整条引擎列表**，不是单条结果。此前实现把每条结果包成
+    单元素列表，`merge_within_caliber` 里的 `enumerate(group)` 于是恒为
+    `i=0`，同一引擎的所有结果拿到相同 RRF 分——名次衰减被抹平，输出退化成
+    「按引擎权重整段拼接」（2026-10-17 实测：anysearch 5 条同分 0.017213）。
+    一个列表通常来自一个引擎；若上游混入多引擎，按口径拆开但保持原序。
+    """
+    caliber_groups: dict[str, list[list[dict[str, Any]]]] = {}
+    for results in ranked_lists:
+        if not results:
+            continue
+        by_caliber: dict[str, list[dict[str, Any]]] = {}
+        for r in results:
+            if not isinstance(r, dict):
+                continue
+            eng = r.get("_engine") or r.get("source", "") or ""
+            by_caliber.setdefault(caliber_of(eng), []).append(r)
+        for cal, group in by_caliber.items():
+            if group:
+                caliber_groups.setdefault(cal, []).append(group)
+    return caliber_groups
+
+
+def merge_within_caliber(
+    groups: list[list[dict[str, Any]]],
+    k: int = 60,
+    weighted: bool = True,
+    lang: str | None = None,
+    _canonical_url=None,
+    _engine_weight=None,
+) -> list[dict[str, Any]]:
+    """同口径内做 RRF 融合。"""
+    if _canonical_url is None:
+        from url_canon import canonical_url as _canonical_url
+    if _engine_weight is None:
+        from search_rank import _engine_weight as _engine_weight
+
+    scores: dict[str, float] = {}
+    items: dict[str, dict[str, Any]] = {}
+
+    for group in groups:
+        for i, r in enumerate(group):
+            key = (
+                _canonical_url(r.get("url", ""))
+                or (f"__title__:{r.get('title', '')}" if r.get("title") else "")
+                or (f"__card__:{r.get('card_type', '')}" if r.get("card_type") else "")
+                or f"__idx__:{id(r)}"
+            )
+            w = _engine_weight(r.get("_engine") or r.get("source") or "",
+                               lang=lang) if weighted else 1.0
+            scores[key] = scores.get(key, 0.0) + w / (k + i + 1)
+            eng = r.get("_engine") or r.get("source", "") or ""
+            if key not in items:
+                item = dict(r)
+                item["_rrf_score"] = 0.0
+                cons: list[str] = []
+                if eng:
+                    cons.append(eng)
+                item["consensus_engines"] = cons
+                items[key] = item
+            else:
+                cur = items[key]
+                new_txt = f"{r.get('title', '')} {r.get('snippet', '')}"
+                cur_txt = f"{cur.get('title', '')} {cur.get('snippet', '')}"
+                if len(new_txt) > len(cur_txt):
+                    cur["title"] = r.get("title", cur.get("title"))
+                    cur["snippet"] = r.get("snippet", cur.get("snippet"))
+                sources = {cur.get("source", ""), r.get("source", "")}
+                cur["source"] = "/".join(s for s in sources if s)
+                cons = list(cur.get("consensus_engines") or [])
+                if eng and eng not in cons:
+                    cons.append(eng)
+                cur["consensus_engines"] = cons
+
+    ranked = sorted(scores.items(), key=lambda x: -x[1])
+    out = []
+    for key, _ in ranked:
+        item = items[key]
+        item["_rrf_score"] = round(scores[key], 6)
+        out.append(item)
+    return out
+
+
+def merge_by_caliber(
+    ranked_lists: list[list[dict[str, Any]]],
+    k: int = 60,
+    weighted: bool = True,
+    lang: str | None = None,
+    _canonical_url=None,
+    _engine_weight=None,
+) -> list[dict[str, Any]]:
+    """口径隔离的 RRF 融合：同口径内 merge，不同口径按优先级并列输出。
+
+    这是 rrf_merge 的口径隔离版本，调用方（search_rank.rrf_merge）传入
+    私有函数引用，本模块不反向依赖 search_rank。
+
+    当 weighted=False 时，不分组，走全局 RRF（经典 RRF 行为）。
+    """
+    if not weighted:
+        # 经典 RRF：不分组，但**保留每个引擎列表的独立名次**——把所有结果
+        # 拍平成一个列表会让后一个引擎的 rank 接着前一个数，名次衰减错位。
+        return merge_within_caliber(
+            ranked_lists,
+            k=k, weighted=False, lang=lang,
+            _canonical_url=_canonical_url, _engine_weight=_engine_weight,
+        )
+
+    caliber_groups = group_by_caliber(ranked_lists)
+
+    merged_by_caliber: dict[str, list[dict[str, Any]]] = {}
+    for cal, groups in caliber_groups.items():
+        merged_by_caliber[cal] = merge_within_caliber(
+            groups, k=k, weighted=weighted, lang=lang,
+            _canonical_url=_canonical_url, _engine_weight=_engine_weight,
+        )
+
+    # 不同口径按优先级排序后并列输出
+    out = []
+    for cal in sorted(merged_by_caliber.keys(),
+                     key=lambda c: _CALIBER_PRIORITY.get(c, 8)):
+        out.extend(merged_by_caliber[cal])
+    return out

package/scripts/calibrate_lowquality.py

@@ -0,0 +1,210 @@
+#!/usr/bin/env python3
+"""calibrate_lowquality.py — 低质内容信号的区分度标定（2026-09-27 新增）。
+
+存在的理由：clickbait / title_body / template 三道门槛（0.5 / 0.5 / 0.75）
+原先都是拍脑袋的常数。手工构造两个样本时发现两个问题：
+  - 精心打磨的现代农场页 clickbait 得 0.0（PACLIC 那套悬念/夸张词表对
+    2026 年的 SEO 写法已失效）；
+  - 正常技术长文 template 得 0.833，农场页 0.875 —— 信号基本没有判别力。
+两者都是「看几个例子觉得还行」暴露不出来的，只有跑成批样本才知道。
+
+本脚本输出三个量，都是**与阈值无关**的排序性质，故能先于调参使用：
+  - AUC        : 该信号单独作为判别器的区分度（0.5 = 无判别力）
+  - 最好阈值   : 扫全阈值区间取 F1 最优点，顺带给出现行阈值的位置
+  - 混淆       : 现行阈值下 good 被误杀 / low 被漏放的具体样本 id
+
+用法：
+  python3 scripts/calibrate_lowquality.py            # 人读表格
+  python3 scripts/calibrate_lowquality.py --json     # 机读（供门禁对比）
+
+改任何阈值前后各跑一次，把数字贴进 commit —— 不要再写「感觉变好了」。
+"""
+from __future__ import annotations
+
+import json
+import os
+import sys
+from pathlib import Path
+
+REPO = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(REPO / "scripts"))
+
+FIXTURE = REPO / "tests" / "golden" / "lowquality_calibration.json"
+
+
+def _signals(sample: dict) -> dict:
+    """取一条样本的三个低质信号（与 content_signals 同源，避免口径分叉）。"""
+    from content_signals import (score_clickbait, score_template_repetition,
+                                 score_title_body_consistency)
+    title = sample.get("title", "")
+    content = sample.get("content", "")
+    cb = score_clickbait(title)
+    tb = score_title_body_consistency(title, content)
+    tp = score_template_repetition(content)
+    return {
+        # clickbait: 越大越像农场
+        "clickbait": float(cb.get("score", 0.0)),
+        # template: 越大越像模板批量生成
+        "template": float(tp.get("repetition", 0.0)),
+        # title_body: 越小越文不对题 → 取反，使三个信号同向（越大越低质）
+        "title_body_gap": 1.0 - float(tb.get("coverage", 0.5)),
+        # keyword_density: 越大越是关键词堆砌（排序层信号，见 rank_signals）
+        "keyword_density": _density(sample),
+    }
+
+
+def _density(sample: dict) -> float:
+    """整体关键词密度（title+snippet 合并）。延迟导入避免顶层拖慢 --help。"""
+    try:
+        from rank_signals import keyword_density, relevance_units
+    except ImportError:
+        return 0.0
+    uq = set(relevance_units(sample.get("query", "")))
+    return keyword_density(
+        f"{sample.get('title', '')} {sample.get('content', '')}", uq)
+
+
+# 现行 evidence_loop.py 里的实际门槛，改动这里必须同步改那里
+CURRENT_THRESHOLDS = {
+    "clickbait": ("ge", 0.5),
+    "title_body_gap": ("mismatch", 0.5),   # 语义阈值：coverage < 0.5
+    "template": ("ge", 0.75),
+    # 排序层：rank_signals.stuffing_density_penalty 的门槛
+    "keyword_density": ("ge", 0.22),
+}
+
+
+def _auc(pairs: list[tuple[float, int]]) -> float:
+    """AUC（越大越 low 方向）：正类（low=1）得分高于负类（good=0）的概率。
+
+    约定：所有进入本函数的信号都**同向**——score 越大越像低质
+    （clickbait / template / title_body_gap / keyword_density 皆如此，
+    前两个本来就是，title_body_gap 在取反时已转正）。故 AUC 可直接跨信号
+    比较，0.5 = 无判别力，>0.5 = 有判别力。ties 记 0.5。
+
+    实现用排序累加而非 sklearn：零依赖，样本量小，O(n²) 完全够。
+    """
+    pos = [s for s, y in pairs if y == 1]
+    neg = [s for s, y in pairs if y == 0]
+    if not pos or not neg:
+        return float("nan")
+    total = 0.0
+    for p in pos:
+        for n in neg:
+            total += 1.0 if p > n else (0.5 if p == n else 0.0)
+    return total / (len(pos) * len(neg))
+
+
+def _best_threshold(pairs: list[tuple[float, int]]) -> tuple[float, float]:
+    """扫候选阈值，返回 (最佳 F1 的阈值, 该 F1)。
+
+    候选取所有相邻分数的中点——对连续分数而言，判别边界只可能在数据点之间。
+    分数完全相同的一组无法分开，此时 F1 反映的是「按分数盲猜」的上限。
+    """
+    vals = sorted({s for s, _ in pairs})
+    if len(vals) < 2:
+        return (vals[0] if vals else 0.0, 0.0)
+    cands = [(vals[i] + vals[i + 1]) / 2 for i in range(len(vals) - 1)]
+    best = (0.0, 0.0)
+    for t in cands:
+        tp = sum(1 for s, y in pairs if s >= t and y == 1)
+        fp = sum(1 for s, y in pairs if s >= t and y == 0)
+        fn = sum(1 for s, y in pairs if s < t and y == 1)
+        prec = tp / (tp + fp) if (tp + fp) else 0.0
+        rec = tp / (tp + fn) if (tp + fn) else 0.0
+        f1 = 2 * prec * rec / (prec + rec) if (prec + rec) else 0.0
+        if f1 > best[1]:
+            best = (t, f1)
+    return best
+
+
+def run() -> dict:
+    data = json.loads(FIXTURE.read_text(encoding="utf-8"))
+    samples = data["samples"]
+    for s in samples:
+        s["_sig"] = _signals(s)
+
+    report: dict = {"n": len(samples), "signals": {}}
+    for name in ("clickbait", "template", "title_body_gap", "keyword_density"):
+        pairs = [(s["_sig"][name], 1 if s["quality"] == "low" else 0) for s in samples]
+        auc = _auc(pairs)
+        thr, f1 = _best_threshold(pairs)
+        cur_t = CURRENT_THRESHOLDS[name][1]
+        fp = [s["id"] for s in samples
+              if s["quality"] == "good" and s["_sig"][name] >= cur_t]
+        fn = [s["id"] for s in samples
+              if s["quality"] == "low" and s["_sig"][name] < cur_t]
+        report["signals"][name] = {
+            "auc": round(auc, 4),
+            "best_threshold": round(thr, 4),
+            "best_f1": round(f1, 4),
+            "current_threshold": cur_t,
+            "false_positive_ids": fp,   # good 被判低质
+            "false_negative_ids": fn,   # low 被放过
+            # 分低质类别的 AUC：一个信号通常只对某一类低质负责，把它不负责的
+            # 类别也塞进同一个 AUC 会得出「越改越差」的反直觉结论。
+            # 2026-09-27 踩过：堆砌密度信号上线后总体 AUC 0.571→0.370，
+            # 实际是因为「文不对题」「标题党」这些**密度天然低**的样本被算进
+            # 了分母——它们 relevance 高是**正确**的（确实高度相关，只是别的
+            # 问题），不该由密度信号压。按类拆开后该信号在 stuffing 类上
+            # AUC=0.964（越大越低质方向），即近乎完美。
+            "auc_by_lowclass": _auc_by_lowclass(samples, name),
+        }
+    return report
+
+
+def _auc_by_lowclass(samples: list[dict], name: str) -> dict:
+    """按低质类别（用 id 前缀归组）分别算 AUC。
+
+    类别取 id 的第一个下划线段：farm_* / stuff_* / seo_good_*。
+    """
+    out: dict = {}
+    groups = sorted({s["id"].split("_")[0] for s in samples
+                     if s["quality"] == "low"})
+    for g in groups:
+        sub_low = [s for s in samples
+                   if s["quality"] == "low" and s["id"].startswith(g + "_")]
+        if not sub_low:
+            continue
+        neg = [s for s in samples if s["quality"] == "good"]
+        if not neg:
+            continue
+        pairs = [(s["_sig"][name], 1) for s in sub_low] + \
+                [(s["_sig"][name], 0) for s in neg]
+        out[g] = round(_auc(pairs), 4)
+    return out
+
+
+def main() -> int:
+    as_json = "--json" in sys.argv[1:]
+    rep = run()
+    if as_json:
+        # 走统一入口而非 json.dumps(indent=2)：stdout 不出缩进 JSON
+        # （tests/test_context_budget.py 门禁；人读格式在下方非 --json 分支）
+        from cli_io import dumps
+        print(dumps(rep))
+        return 0
+
+    print(f"低质信号区分度标定（{rep['n']} 条样本，好/坏对半）\n")
+    print(f"{'信号':<18}{'总体AUC':>9}{'现门槛':>9}{'最佳门槛':>10}{'最佳F1':>9}")
+    print("-" * 55)
+    for name, m in rep["signals"].items():
+        print(f"{name:<18}{m['auc']:>9.3f}{m['current_threshold']:>9.2f}"
+              f"{m['best_threshold']:>10.3f}{m['best_f1']:>9.3f}")
+    print("\n总体 AUC：0.5=无判别力，0.75=可用，0.9+=强。")
+    print("⚠️ 总体 AUC 会误导——一个信号通常只对某一类低质负责。**按类看**：")
+    for name, m in rep["signals"].items():
+        byc = m.get("auc_by_lowclass") or {}
+        if not byc:
+            continue
+        cells = "  ".join(f"{k}={v:.3f}" for k, v in sorted(byc.items()))
+        print(f"  {name:<18} {cells}")
+    print("\n误杀(good 判成 low) / 漏放(low 判成 good)：")
+    for name, m in rep["signals"].items():
+        print(f"  {name:<18} 误杀={m['false_positive_ids'] or '无'}")
+        print(f"  {'':<18} 漏放={m['false_negative_ids'] or '无'}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())