@furongjun1999/dsh-memory 0.4.6 → 0.4.8

package/md_cg/bench6_arms.py

@@ -0,0 +1,384 @@
+# -*- coding: utf-8 -*-
+"""六系统横评（bench6）· 主进程臂：灵枢 / 纯向量 RAG + 统一评分。
+
+四家 LLM 竞品（mem0/Graphiti/GraphRAG/Letta）依赖互斥，跑在各自 venv 的独立
+脚本里（外部评测工作区的 bench6 适配脚本），只回吐 {qid: [id...]} 的 JSON；本模块
+读回后与灵枢/向量臂走**同一个** bc.rows_from_hits → ec.summarize，保证六家同口径
+（排名判据复用 ec.first_evidence_rank，指标复用 ec.summarize，均不重造）。
+
+灵枢三种检索口径（同一份中文层 + 英文原文，只变检索路）：
+  lex        : 单词法路，且建库不加 tags/condition_space——复现公开参考量级
+               （词法 hit@1≈0.936）的**回归锚**，用于确认本次口径没有漂移
+  rrf4       : 四路 RRF（lexical,bucket,entity,graph）+ 传 context——用户选定口径
+  rrf4_noref : 四路 RRF 不传 context——名义四路，量化条件约束路的净效应
+
+## 为什么灵枢要建两个库而不是一个
+lex 臂必须与 `bench_locomo_zh_public.build_pool` **逐字同口径**（无 tags、无
+condition_space）才能作回归锚；而 rrf4 臂按用户口径要喂 entity 路（tags）与
+bucket 路（condition_space）。建库参数不同 → 目录不同 → 拆成两个 ROOT，避免
+"复用已建库"逻辑把两种口径混成一个。
+
+## 诚实边界
+locomo-zh-500 是**单域**会话语料，且查询侧是**无情境标注**的关键词串。bucket 路
+要求读写两侧 `route_key` 同构（routing.py 模块头「风险2」）——写侧有域、读侧
+没有，故条件路由在本语料上预期无区分力。这一点由实测的桶健康度取证，不预设。
+"""
+from __future__ import annotations
+
+import json
+import math
+import os
+import shutil
+import sys
+import time
+import urllib.error
+import urllib.request
+
+HERE = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+for _p in (HERE, os.path.join(HERE, "md_cg")):
+    if _p not in sys.path:
+        sys.path.insert(0, _p)
+
+from md_cg import bench6_common as bc      # noqa: E402
+from md_cg import eval_common as ec        # noqa: E402
+
+ROOT_BASE = os.path.join(HERE, "_md_cg_eval_bench6")   # .gitignore 的 /_md_cg_eval_*/ 已覆盖
+
+# 条件约束路的查询侧情境：所有题共用同一份默认情境（查询串本身无情境标注）。
+# 与建库侧 condition_space 同构 → route_key 两侧落在同一键上。这是 bucket 路能被
+# 开启的唯一合法构造方式；若查询侧能精确给出每题的场景/域，那已是使用 qrels 作弊。
+QUERY_CONTEXT = {
+    "observation_position": "会话陈述",
+    "observation_tool": "会话记录",
+    "existence_constraint": "公开",
+}
+
+
+def cond_of(c):
+    """zh_fields.condition（中文三槽）→ mdcos 期望的英文槽名。
+
+    只映射 zh_fields 里**真实存在**的三槽，不臆造 time_window（写入侧会自行补）。
+    """
+    cond = (c.get("zh_fields") or {}).get("condition") or {}
+    return {"observation_position": cond.get("观测位置", ""),
+            "observation_tool": cond.get("观测工具", ""),
+            "existence_constraint": cond.get("存在约束", "")}
+
+
+def tags_of(c):
+    """喂 entity 路的裸词 tags = identity + terms。
+
+    刻意不加 `ent:` 前缀：`mdcos._path_entity` 的判据是 `t in query or query in t`，
+    带前缀的标签在自然查询下永不命中（既有实测教训）。也不加 `domain:`——那会
+    劫持 route_key 的 domain 分支，把 bucket 路变成按场景分桶，而查询侧无法复现
+    同一场景键（见模块头「诚实边界」）。
+    """
+    f = c.get("zh_fields") or {}
+    return [str(f.get("identity") or "")] + [str(t) for t in (f.get("terms") or [])]
+
+
+class Adapter:
+    """统一适配接口。六家一律只回吐 id 列表，指标计算交回 bench6_common。"""
+    name = "adapter"
+
+    def reset(self):
+        raise NotImplementedError
+
+    def add(self, nid, text):
+        raise NotImplementedError
+
+    def search(self, query, k=5):
+        raise NotImplementedError
+
+    def describe_ingest(self, nids):
+        """回读真实存储内容样本，用于取证「入库语言与入库形态」。"""
+        return []
+
+
+class LingshuAdapter(Adapter):
+    """灵枢臂：进程内 MdCGOS。
+
+    with_meta=True 时三口径共用一库（lex 需与公开脚本同库，故 lex 单独建）。
+    """
+
+    name = "lingshu"
+
+    def __init__(self, rows, variant, paths, context=None, with_meta=False,
+                 rebuild=False):
+        from md_cg.bench_locomo_zh_public import body_of
+        from md_cg.mdcos import MdCGOS
+
+        self.variant = variant
+        self.paths = tuple(paths)
+        self.context = context
+        self._body_of = body_of
+        root = os.path.join(ROOT_BASE, variant)
+        self.root = root
+        if rebuild and os.path.isdir(root):
+            shutil.rmtree(root, ignore_errors=True)
+        self.cg = MdCGOS(root, autoflush=500)
+        if len(self.cg.index["nodes"]) >= len(rows):
+            return                                  # 幂等复用
+        t0 = time.time()
+        for c in rows:
+            kw = {}
+            if with_meta:
+                kw["tags"] = tags_of(c)
+                kw["condition_space"] = cond_of(c)
+            self.cg.add(c["id"], body_of(c), layer="knowledge",
+                        eval_src="bench6:locomo-zh-500",
+                        verification_basis="data", **kw)
+        self.cg.flush()
+        print("  [lingshu/%s] 建库 %d 节点 %.1fs"
+              % (variant, len(self.cg.index["nodes"]), time.time() - t0))
+
+    def search(self, query, k=5):
+        kw = {"context": self.context} if self.context is not None else {}
+        res, _meta = self.cg.search_rrf(query, k=k, paths=self.paths,
+                                        judge=False, record=False, **kw)
+        return [r[0]["id"] for r in res]
+
+    def describe_ingest(self, nids):
+        """回读真实落盘内容（索引元数据 + 文件头），取证入库语言与形态。
+
+        不走 `cg.get`：其返回形状随层不同，且读缓存会介入；直接按索引 path 读
+        文件是最贴近"磁盘上真实存了什么"的取证方式。
+        """
+        out = []
+        for nid in nids[:2]:
+            e = self.cg.index["nodes"].get(nid)
+            if not e:
+                continue
+            head = ""
+            try:
+                with open(os.path.join(self.cg.root, e["path"]), encoding="utf-8") as f:
+                    head = f.read()[:300]
+            except OSError:
+                pass
+            out.append({"id": nid, "layer": e.get("layer"), "tags": e.get("tags"),
+                        "bucket": e.get("bucket"),
+                        "condition_space": e.get("condition_space"),
+                        "file_head": head})
+        return out
+
+    def bucket_health(self):
+        """桶健康度自检（routing.bucket_health）：取证条件路由是否有区分力。"""
+        from md_cg import routing
+        counts = {}
+        for e in self.cg.index["nodes"].values():
+            b = e.get("bucket") or "<none>"
+            counts[b] = counts.get(b, 0) + 1
+        return routing.bucket_health(counts)
+
+
+class VectorRagAdapter(Adapter):
+    """纯向量 RAG 基线（零 LLM）：英文原文直嵌入 + 余弦 Top-k。
+
+    作为**下界锚**：无结构化、无图、无实体/条件路。embedding 走本地
+    127.0.0.1:1234（OpenAI 兼容），与既有竞品探针同源，不引入新依赖。
+    """
+
+    name = "vector_rag"
+
+    def __init__(self, base=None, model=None, timeout=30):
+        self.base = (base or os.environ.get("BENCH6_EMBED_BASE")
+                     or "http://127.0.0.1:1234/v1").rstrip("/")
+        self.model = model or os.environ.get("BENCH6_EMBED_MODEL") or ""
+        self.timeout = timeout
+        self.ids = []
+        self.vecs = []
+        self._dim = 0
+
+    def _post(self, path, payload):
+        req = urllib.request.Request(
+            self.base + path, method="POST",
+            data=json.dumps(payload).encode("utf-8"),
+            headers={"Content-Type": "application/json"})
+        with urllib.request.urlopen(req, timeout=self.timeout) as r:
+            return json.loads(r.read().decode("utf-8"))
+
+    def resolve_model(self):
+        """未显式指定时，从 /models 取第一个 embedding 模型。"""
+        if self.model:
+            return self.model
+        req = urllib.request.Request(self.base + "/models")
+        with urllib.request.urlopen(req, timeout=15) as r:
+            data = json.loads(r.read().decode("utf-8"))
+        ids = [m.get("id") for m in (data.get("data") or []) if m.get("id")]
+        if not ids:
+            raise RuntimeError("embedding 服务未返回任何模型：%s" % self.base)
+        # 优先明显是 embedding 的模型名
+        for mid in ids:
+            if any(k in mid.lower() for k in ("embed", "bge", "gte", "m3")):
+                self.model = mid
+                break
+        else:
+            self.model = ids[0]
+        return self.model
+
+    def embed(self, text):
+        d = self._post("/embeddings", {"model": self.model, "input": text})
+        v = d["data"][0]["embedding"]
+        self._dim = len(v)
+        return v
+
+    def embed_many(self, texts, batch=32):
+        """批量嵌入（服务不支持 batch 时逐条回退）。"""
+        out = []
+        for i in range(0, len(texts), batch):
+            chunk = texts[i:i + batch]
+            try:
+                d = self._post("/embeddings", {"model": self.model, "input": chunk})
+                rows = sorted(d["data"], key=lambda x: x.get("index", 0))
+                out.extend([r["embedding"] for r in rows])
+            except Exception:
+                out.extend([self.embed(t) for t in chunk])
+            if self._dim == 0 and out:
+                self._dim = len(out[-1])
+        return out
+
+    def reset(self):
+        self.ids, self.vecs = [], []
+
+    def add(self, nid, text):
+        self.ids.append(nid)
+        self.vecs.append(self.embed(text))
+
+    def search(self, query, k=5):
+        if not self.vecs:
+            return []
+        qv = self.embed(query)
+        qn = math.sqrt(sum(x * x for x in qv)) or 1.0
+        scored = []
+        for i, v in enumerate(self.vecs):
+            vn = math.sqrt(sum(x * x for x in v)) or 1.0
+            s = sum(a * b for a, b in zip(qv, v)) / (qn * vn)
+            scored.append((s, self.ids[i]))
+        scored.sort(key=lambda x: (-x[0], x[1]))
+        return [nid for _s, nid in scored[:k]]
+
+    def describe_ingest(self, nids):
+        return [{"id": nid, "form": "原样英文文本 → embedding",
+                 "lang": "en", "dim": self._dim} for nid in nids[:2]]
+
+
+# ------------------------------------------------------------------ 套件与评分
+
+def run_suite(adapter, questions, lang, k=5, verbose=True):
+    """一套查询语言跑一遍 → {qid: [id...]}（顺序即相关性降序）。"""
+    key = "question_zh" if lang == "zh" else "question_en"
+    hits, t0 = {}, time.time()
+    for i, q in enumerate(questions, 1):
+        hits[q["qid"]] = adapter.search(q[key], k=k)
+        if verbose and i % 50 == 0:
+            print("    [%s] %d/%d（%.0fs）" % (lang, i, len(questions),
+                                              time.time() - t0))
+    return hits
+
+
+def evaluate(arm, adapter, questions, k=5, langs=("zh", "en")):
+    """一套库 × 两种查询语言 → 指标 + per-question 明细 + 回读取证。"""
+    out = {"arm": arm, "k": k, "langs": {}}
+    for lang in langs:
+        hits = run_suite(adapter, questions, lang, k=k)
+        rows = bc.rows_from_hits(questions, hits, k=k)
+        s = ec.summarize(rows, k=k)
+        out["langs"][lang] = {"summary": s, "rows": rows, "hits": hits}
+        print("  [%s/%s] hit@1=%.1f%% hit@%d=%.1f%% MRR=%.4f"
+              % (arm, lang, s["hit@1"] * 100, k, s["hit@%d" % k] * 100, s["mrr"]))
+    out["ingest_probe"] = adapter.describe_ingest([q["qid"] for q in questions[:2]])
+    return out
+
+
+def build_vector_rag(pool, model=None):
+    """向量基线：池内英文原文批量嵌入（零 LLM、无结构化）。"""
+    vec = VectorRagAdapter(model=model or os.environ.get("BENCH6_EMBED_MODEL")
+                           or "text-embedding-bge-m3")
+    vec.resolve_model()
+    t0 = time.time()
+    vec.ids = [t["id"] for t in pool]
+    vec.vecs = vec.embed_many([t["ingest"] for t in pool])
+    print("  [vector_rag] 模型=%s dim=%d 嵌入 %d 条 %.0fs"
+          % (vec.model, vec._dim, len(vec.vecs), time.time() - t0))
+    return vec
+
+
+def main(argv=None):
+    argv = sys.argv[1:] if argv is None else argv
+    rebuild = "--rebuild" in argv
+    only = [a for a in argv if not a.startswith("-")]
+    t_all = time.time()
+
+    data = bc.load()
+    questions, pool, man = data["questions"], data["pool"], data["manifest"]
+    print("[bench6] 题 %d / 池 %d（口径固化于 %s）"
+          % (len(questions), len(pool), man["created_at"]))
+
+    # 评测口径：与既有公开基准一致，否则与参考量级不可比（见 eval_common 注释）
+    ec.unlock_global_cap()
+    ec.use_jaccard()
+
+    corpus_by_id = {c["id"]: c for c in ec.iter_jsonl(bc.CORPUS567)}
+    rows = [corpus_by_id[t["id"]] for t in pool]
+    results, table = {}, {}
+
+    def _want(name):
+        return not only or name in only
+
+    # (1) 灵枢单词法路 = 回归锚。建库刻意**不加** tags/condition_space，与
+    #     bench_locomo_zh_public.build_pool 逐字同口径；须复现 hit@1≈0.936，
+    #     否则说明本次口径已漂移，后续所有四路数字都不可信。
+    if _want("lingshu_lex"):
+        lex = LingshuAdapter(rows, "lex", paths=("lexical",), with_meta=False,
+                             rebuild=rebuild)
+        ec.install_read_cache(lex.cg)
+        results["lingshu_lex"] = evaluate("lingshu_lex", lex, questions)
+        table["灵枢·单词法"] = results["lingshu_lex"]["langs"]["zh"]["summary"]
+
+    # (2) 灵枢四路：真开 bucket（传 context）vs 名义四路（不传），单变量对照。
+    if _want("lingshu_rrf4"):
+        rrf = LingshuAdapter(rows, "rrf4", paths=ec.PATHS, with_meta=True,
+                             rebuild=rebuild)
+        ec.install_read_cache(rrf.cg)
+        health = rrf.bucket_health()
+        print("  [lingshu] 桶健康度 %s" % json.dumps(health, ensure_ascii=False))
+        rrf.context = QUERY_CONTEXT
+        results["lingshu_rrf4"] = evaluate("lingshu_rrf4", rrf, questions)
+        rrf.context = None
+        results["lingshu_rrf4_noref"] = evaluate("lingshu_rrf4_noref", rrf, questions)
+        table["灵枢·四路(真开bucket)"] = results["lingshu_rrf4"]["langs"]["zh"]["summary"]
+        table["灵枢·四路(名义)"] = results["lingshu_rrf4_noref"]["langs"]["zh"]["summary"]
+        results["bucket_health"] = health
+
+    # (2b) 2×2 消融：隔离「建库补 meta(tags/condition_space)」与「检索从 1 路变
+    #      4 路」两个变量。rrf4(81%) 与 lex(99%) 之间混了这两个变量，不隔离就
+    #      无法归因下降来自哪一侧。两个臂都复用已建好的库，只改检索参数，0 成本。
+    if not only:
+        rrf.paths = ("lexical",)
+        results["lingshu_lex_meta"] = evaluate("lingshu_lex_meta", rrf, questions)
+        rrf.paths = ec.PATHS                       # 复原
+        lex.paths = ec.PATHS
+        lex.context = QUERY_CONTEXT
+        results["lingshu_rrf4_nometa"] = evaluate("lingshu_rrf4_nometa", lex, questions)
+        lex.paths, lex.context = ("lexical",), None  # 复原
+        table["灵枢·词法+meta"] = results["lingshu_lex_meta"]["langs"]["zh"]["summary"]
+        table["灵枢·四路(无meta)"] = results["lingshu_rrf4_nometa"]["langs"]["zh"]["summary"]
+
+    # (3) 纯向量 RAG 基线（零 LLM，下界锚）
+    if _want("vector_rag"):
+        vec = build_vector_rag(pool)
+        results["vector_rag"] = evaluate("vector_rag", vec, questions)
+        table["纯向量RAG"] = results["vector_rag"]["langs"]["zh"]["summary"]
+
+    payload = {"manifest": man, "results": results,
+               "elapsed_s": round(time.time() - t_all, 1)}
+    ec.save_result("bench6_arms_result.json", payload)
+    if table:
+        ec.print_table("bench6 主进程臂 · 中文查询（池 %d / 题 %d）"
+                       % (len(pool), len(questions)), table, k=5)
+    print("\n[bench6] 主进程臂完成，用时 %.0fs" % (time.time() - t_all))
+    return payload
+
+
+if __name__ == "__main__":
+    main()

package/md_cg/bench6_common.py

@@ -0,0 +1,222 @@
+# -*- coding: utf-8 -*-
+"""六系统横评（bench6）口径层：唯一真源。
+
+六家（灵枢/mem0/Graphiti/GraphRAG/Letta/纯向量RAG）输入同一份英文原始语料；
+灵枢按用户指定"跳过结构化"，复用题库现成中文层叠加英文原文。
+本模块只固化口径（抽样/池/双查询词面），指标与排名一律复用 eval_common。
+
+诚实边界（报告须原样引用）：
+  * 池全为 gold 证据 → 池内零干扰，指标是**上界**，非端到端记忆能力。
+  * question_zh 是中文关键词串而非自然语言问句 → 评检索命中，非 QA。
+  * n=100 时题级 CV≈8.3%（VERSION.json 记 n=500 时 CV 3.7%，容量减半 √5 放大）
+    → 家间差距小于约 16% 不可下结论。
+
+跑法：
+    python -m md_cg.bench6_common [--force]
+"""
+from __future__ import annotations
+
+import json
+import os
+import random
+import sys
+import time
+
+HERE = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+for _p in (HERE, os.path.join(HERE, "md_cg")):
+    if _p not in sys.path:
+        sys.path.insert(0, _p)
+
+from md_cg import eval_common as ec       # noqa: E402
+
+DATA = os.path.join(HERE, "data", "benchmarks", "locomo-zh-500")
+CORPUS567 = os.path.join(DATA, "corpus567.jsonl")
+QUESTIONS500 = os.path.join(DATA, "questions500.jsonl")
+# 英文原问句（上游 LoCoMo 派生副本，不入库）；可用 BENCH6_EN_QUESTIONS 覆盖
+EN_QUESTIONS = (os.environ.get("BENCH6_EN_QUESTIONS")
+                or os.path.join(HERE, "data", "external", "locomo_zh", "raw_questions.jsonl"))
+
+OUT = os.path.join(ec.RESULTS, "bench6")
+POOL_JSONL = os.path.join(OUT, "pool.jsonl")
+QUESTIONS_JSONL = os.path.join(OUT, "questions.jsonl")
+MANIFEST = os.path.join(OUT, "manifest.json")
+
+N_QUESTIONS = 100
+SEED = 7007          # 二级抽样专用；与 VERSION.json 的 500 题 seed=7 区分开
+K = 5
+
+BOUNDARIES = [
+    "池全为 gold 证据 → 池内零干扰，指标是上界而非端到端记忆能力",
+    "question_zh 是中文关键词串而非自然语言问句 → 评检索命中，非 QA",
+    "无答案文本，无法评答案质量",
+    "n=100 时题级命中率 CV≈8.3%（n=500 时 3.7%，容量减半 √5 放大）→ 家间差距 <16% 不可下结论",
+]
+
+
+def stratified_sample(rows, n=N_QUESTIONS, seed=SEED):
+    """按 qtype 比例分层抽样（最大余额法）。
+
+    不用 random.sample(rows, n)：小比例题型（open_domain 4.4%）在 n=100 时
+    期望仅 4.4 题，随机抽会在 0~8 间抖动，题型构成随种子漂移，横向对比的
+    题型归因就不稳。最大余额法锁定每型题数，使「六家 × 同题型」可比。
+    """
+    by = {}
+    for r in rows:
+        by.setdefault(r["qtype"], []).append(r)
+    total = len(rows)
+    exact = {qt: n * len(g) / total for qt, g in by.items()}
+    quota = {qt: int(v) for qt, v in exact.items()}
+    rest = n - sum(quota.values())
+    order = sorted(exact.items(), key=lambda kv: (-(kv[1] - int(kv[1])), kv[0]))
+    for qt, _ in order[:rest]:
+        quota[qt] += 1
+    out = []
+    for qt in sorted(by):
+        g, take = by[qt], min(quota[qt], len(by[qt]))
+        out.extend(random.Random("%d:%s" % (seed, qt)).sample(g, take))
+    out.sort(key=lambda r: r["qid"])
+    return out, quota
+
+
+def build_pool(sample, corpus_by_id):
+    """池 = 抽中题的 gold 引用 turn 去重。返回 (pool, dangling)。
+
+    悬空引用是数据集已知现象（VERSION.json 记 scene_3_q_58 →
+    scene_3_session_10_turn_19 不在池中）。逐条记录而非静默丢弃，
+    使第三方可按同一规则复算池条数。
+    """
+    pool_ids, dangling = set(), []
+    for q in sample:
+        miss = [t for t in q["evidence_turns"] if t not in corpus_by_id]
+        if miss:
+            dangling.append({"qid": q["qid"], "missing": miss})
+        pool_ids.update(t for t in q["evidence_turns"] if t in corpus_by_id)
+    return [corpus_by_id[i] for i in sorted(pool_ids)], dangling
+
+
+def ingest_text_en(c):
+    """六家共用的英文写入文本：原始陈述 + 会话时间。
+
+    与 ec.lc_turn_text 同构（text + [时间]，使时间类问题的词法路可召回日期词），
+    只是字段名换成 date。刻意**不**套用 ec.lc_turn_text：后者读 r["title"]，
+    而 locomo-zh-500 语料行用的是 r["date"]（"01:14 PM on Thursday 25 May, 2023"），
+    直接套用会静默丢掉全部时间信息。
+    """
+    text = str(c.get("text") or "").strip()
+    date = str(c.get("date") or "").strip()
+    return "%s [%s]" % (text, date) if date else text
+
+
+def turn_rec(c):
+    """池行 → 落盘行（六家共用的英文侧视图）。"""
+    return {"id": c["id"], "speaker": c.get("speaker") or "",
+            "date": c.get("date") or "", "text": c.get("text") or "",
+            "ingest": ingest_text_en(c)}
+
+
+def align_queries(sample, en_by_qid):
+    """中英双查询词面按 qid join，断言 evidence_turns 逐字一致。
+
+    断言不是形式主义：两套查询若 qrels 不一致，「中英对照」就退化成
+    「两套题的对照」，任何差异都无法归因到语言。缺英问句直接报错不跳过。
+    """
+    out = []
+    for q in sample:
+        en = en_by_qid.get(q["qid"])
+        if en is None:
+            raise KeyError("英文侧缺 qid=%s（中英题集不同源，评测作废）" % q["qid"])
+        if list(en["evidence_turns"]) != list(q["evidence_turns"]):
+            raise AssertionError("qid=%s 中英 qrels 不一致" % q["qid"])
+        out.append({"qid": q["qid"], "qtype": q["qtype"],
+                    "question_zh": q["question"], "question_en": en["question"],
+                    "evidence_turns": list(q["evidence_turns"])})
+    return out
+
+
+def rank_of(ids, evidence):
+    """首个证据 id 的排名（1-based；未命中 0）。
+
+    复用 ec.first_evidence_rank：把 id 列表适配成其结果四元组形状
+    （r[0]["id"]），而非重写判据——六家排名与公开基准逐字同源。
+    """
+    res = [({"id": x}, 0.0) for x in ids]
+    return ec.first_evidence_rank(res, set(evidence))
+
+
+def rows_from_hits(questions, hits_by_qid, k=K):
+    """把任一家的 id 列表统一转成 ec.summarize 的明细行。
+
+    hits_by_qid: {qid: [id, ...]}（按相关性降序；不足 k 条亦合法）。
+    top1_score 仅用于占位——竞品分数与 cg 分数不同量纲、不可比，
+    故本次不引用 summarize 的 score_p10/p50（拒答线），只取命中类指标。
+    """
+    rows = []
+    for q in questions:
+        ids = list(hits_by_qid.get(q["qid"]) or [])[:k]
+        rows.append({"qid": q["qid"], "qtype": q["qtype"],
+                     "rank": rank_of(ids, q["evidence_turns"]),
+                     "top1_score": 1.0 if ids else 0.0, "n_res": len(ids)})
+    return rows
+
+
+def prepare(force=False, verbose=True):
+    """固化口径层产物（幂等）。返回 manifest。"""
+    if not force and os.path.exists(MANIFEST) and os.path.exists(QUESTIONS_JSONL):
+        with open(MANIFEST, encoding="utf-8") as f:
+            man = json.load(f)
+        if verbose:
+            print("[bench6] 复用已固化口径：%s（题 %d / 池 %d）"
+                  % (OUT, man["queries"]["n"], man["pool"]["n"]))
+        return man
+
+    os.makedirs(OUT, exist_ok=True)
+    rows = list(ec.iter_jsonl(QUESTIONS500))
+    corpus = list(ec.iter_jsonl(CORPUS567))
+    corpus_by_id = {c["id"]: c for c in corpus}
+    en_by_qid = {r["qid"]: r for r in ec.iter_jsonl(EN_QUESTIONS)}
+
+    sample, quota = stratified_sample(rows)
+    pool, dangling = build_pool(sample, corpus_by_id)
+    questions = align_queries(sample, en_by_qid)
+
+    with open(POOL_JSONL, "w", encoding="utf-8") as f:
+        for c in pool:
+            f.write(json.dumps(turn_rec(c), ensure_ascii=False) + "\n")
+    with open(QUESTIONS_JSONL, "w", encoding="utf-8") as f:
+        for q in questions:
+            f.write(json.dumps(q, ensure_ascii=False) + "\n")
+
+    actual = {}
+    for q in questions:
+        actual[q["qtype"]] = actual.get(q["qtype"], 0) + 1
+    man = {
+        "name": "bench6", "created_at": time.strftime("%Y-%m-%dT%H:%M:%S"),
+        "arms": ["lingshu", "mem0", "graphiti", "graphrag", "letta", "vector_rag"],
+        "source": {"questions": QUESTIONS500, "corpus": CORPUS567, "en": EN_QUESTIONS},
+        "sampling": {"method": "stratified_proportional_by_qtype_max_remainder",
+                     "seed": SEED, "n_target": N_QUESTIONS, "k": K,
+                     "quota": quota, "actual": dict(sorted(actual.items()))},
+        "pool": {"n": len(pool), "dangling": dangling},
+        "queries": {"n": len(questions), "langs": ["zh", "en"]},
+        "boundaries": BOUNDARIES,
+    }
+    with open(MANIFEST, "w", encoding="utf-8") as f:
+        json.dump(man, f, ensure_ascii=False, indent=2)
+
+    if verbose:
+        print("[bench6] 口径已固化：%s" % OUT)
+        print("  题型配额 %s → 实际 %s" % (quota, man["sampling"]["actual"]))
+        print("  池条数 %d；悬空引用 %d 条 %s" % (len(pool), len(dangling), dangling))
+        print("  中英双查询 %d 题（qrels 已断言逐字一致）" % len(questions))
+    return man
+
+
+def load():
+    """读回已固化的口径层产物。六家适配器只经此取题与池。"""
+    man = json.load(open(MANIFEST, encoding="utf-8"))
+    return {"questions": list(ec.iter_jsonl(QUESTIONS_JSONL)),
+            "pool": list(ec.iter_jsonl(POOL_JSONL)), "manifest": man}
+
+
+if __name__ == "__main__":
+    prepare(force="--force" in sys.argv)