@furongjun1999/dsh-memory 0.4.11 → 0.5.1

package/md_cg/bench_e2e_locomo_qa.py

@@ -0,0 +1,368 @@
+# -*- coding: utf-8 -*-
+"""LoCoMo QA 同口径对照评测：完整对话做记忆 · 自然问句做查询（2026-09-23）
+
+目的（使用者口径）：与 Mem0/Letta/Zep 等同设定直接比数字——**对话做记忆、自然问句做
+查询、LLM reader + LLM judge**，不再以「摘要卡/关键词串」口径差异回避对照。
+
+与同行的对齐面：
+  · 记忆面：LoCoMo 全量 5882 turn 完整对话（mteb/LoCoMo BEIR corpus，id 与仓内
+    evidence_turns 逐字对齐已验证）→ LLM 译为中文（人名保留）→ 逐 turn 写入灵枢
+    （对话原文，不做摘要加工——写入侧加工差异正是各记忆系统的被测对象本身）
+  · 查询面：上游英文自然问句 → LLM 译为中文自然问句
+  · 答题面：reader 只据注入记忆作答（不足答「无法确定」）→ judge 对上游 gold
+    语义判等（correct/incorrect/refused；adversarial gold 空=正确行为拒答）
+  · 两臂：
+      retrieval     灵枢检索注入（cg.search 阶梯路 top-10，与生产主形态同路）
+      full_context  整段对话全量注入（对标 Mem0 论文 Table 2 的 Full-context 72.90）
+
+标尺（Mem0 论文 arXiv:2504.19413 Table 2，agent=GPT-4o-mini）：
+  Full-context 72.90 · Mem0ᵍ 68.44 · Mem0 66.88 · Zep 65.99 · LangMem 58.10 ·
+  OpenAI memory 52.90 · A-Mem 48.38；Letta Filesystem 复测 74.0（letta.com 2025-08）。
+  本评测 reader/judge=deepseek-chat（与同行 GPT-4o-mini 不同档，绝对分含模型差，
+  对照时声明；同标尺内两臂之差是检索质量的净效应，与模型无关）。
+
+跑法：
+  python -X utf8 -m md_cg.bench_e2e_locomo_qa --translate   # 翻译 5882 turn + 500 问句
+  python -X utf8 -m md_cg.bench_e2e_locomo_qa               # 全量 500 题 × 2 臂
+  python -X utf8 -m md_cg.bench_e2e_locomo_qa --quick       # 冒烟 3 题（须先完成翻译）
+环境：DEEPSEEK_API_KEY；建议 MDCG_UNIFY_QUERY=0（中文问句含英文名时批次15归一有
+形态缺陷，见 docs/eval/端到端干扰池评测_v1.1 §4）。
+"""
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import os
+import re
+import sys
+import time
+import urllib.request
+from concurrent.futures import ThreadPoolExecutor
+
+HERE = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+if HERE not in sys.path:
+    sys.path.insert(0, HERE)
+
+from md_cg.bench_e2e_judge import DEFAULT_ROOT, llm_chat       # noqa: E402
+from md_cg.bench_e2e_qa import (                              # noqa: E402
+    JUDGE_SYS, READER_SYS, _cache_call, parse_json_field)
+
+UP = os.path.join(DEFAULT_ROOT, "upstream")
+CORPUS_PQ = os.path.join(UP, "corpus.parquet")
+ANSWERS = os.path.join(UP, "answers_map.json")
+ZH_TURNS = os.path.join(UP, "zh_turns.json")
+ZH_QUERIES = os.path.join(UP, "zh_queries.json")
+T_CACHE = os.path.join(UP, "translate_cache")
+
+TRANS_TURN_SYS = (
+    "你是对话翻译器。把英文对话 turn 逐条译成中文：忠实原义、不压缩不增删；"
+    "人名与专有名词保留英文原文（如 Caroline / Ed Sheeran）；说话人前缀保留"
+    "（格式「名字： 译文」）。\n只输出一行 JSON：{\"x1\": \"名字： 译文\", ...}"
+)
+TRANS_Q_SYS = (
+    "你是问题翻译器。把英文问句译成自然的中文问句：语气自然、不逐词直译；"
+    "人名与专有名词保留英文原文。\n只输出一行 JSON：{\"x1\": \"中文问句\", ...}"
+)
+
+
+def load_corpus():
+    import pyarrow.parquet as pq
+    t = pq.read_table(CORPUS_PQ)
+    ids = t.column("id").to_pylist()
+    txts = t.column("text").to_pylist()
+    return ids, txts
+
+
+def _batch(lst, n):
+    for i in range(0, len(lst), n):
+        yield lst[i:i + n]
+
+
+def translate_all(key, model_cfg, items, batch_n, sys_prompt, out_path,
+                  max_tokens=4000):
+    """items=[(k, en_text)] → {k: zh}，分片缓存、断点续跑。"""
+    os.makedirs(T_CACHE, exist_ok=True)
+    out = {}
+    if os.path.isfile(out_path):
+        out = json.load(open(out_path, encoding="utf-8"))
+    todo = [(k, t) for k, t in items if k not in out]
+    print(f"[translate:{key}] 已有 {len(out)} / 待译 {len(todo)}"
+          f" · engine={model_cfg['model']}@{model_cfg['base']}")
+    jobs = list(enumerate(_batch(todo, batch_n)))
+
+    def call_one(items, mt):
+        # 行前缀从 x1 起（与系统提示示例 {"x1": …} 一致）：模型对 x0 起头
+        # 的批量会整体偏移成 x1 起，导致键错位解析失败（214 条顽固缺口的根因）
+        mapping = {f"x{i + 1}": k for i, (k, _t) in enumerate(items)}
+        user = "\n".join(f"x{i + 1}. {t}" for i, (_k, t) in enumerate(items))
+        try:
+            raw, _u, _d = llm_chat(
+                model_cfg["model"], model_cfg["base"], model_cfg["key"],
+                [{"role": "system", "content": sys_prompt},
+                 {"role": "user", "content": user}],
+                timeout=model_cfg["timeout"], max_tokens=mt,
+                extra_payload=model_cfg.get("extra"))
+        except Exception:                                   # noqa: BLE001
+            return {}
+        s = re.sub(r"<think>.*?</think>", "", str(raw), flags=re.S)
+        obj = {}
+        _i, _j = s.find("{"), s.rfind("}")
+        if _i >= 0 and _j > _i:
+            try:
+                obj = json.loads(s[_i:_j + 1])
+            except ValueError:
+                obj = {}
+        res = {}
+        for tag, zh in obj.items():
+            if tag in mapping and isinstance(zh, str) and zh.strip():
+                res[mapping[tag]] = zh.strip()
+        return res
+
+    def work(item):
+        idx, job = item
+        # 缓存按**内容寻址**（分片键集 hash）：早前按 todo 重排 idx 命名，
+        # 续传轮 idx 语义漂移会命中**旧内容的分片**（返回已在 out 的旧键，
+        # n_ok 虚涨而总数停滞——180 条顽固缺口的第二重根因）。内容寻址
+        # 跨轮稳定，同内容分片天然命中、不同内容互不误撞。
+        chash = hashlib.md5(",".join(k for k, _ in job)
+                            .encode("utf-8")).hexdigest()[:12]
+        cpath = os.path.join(T_CACHE, f"{key}_{chash}.json")
+        if os.path.isfile(cpath):
+            return json.load(open(cpath, encoding="utf-8"))
+        res = call_one(job, max_tokens)
+        if len(res) < len(job):
+            # 分片级失败兜底：逐条重译（长 turn 挤爆批量输出导致 JSON 截断
+            # 的形态，单条请求给足生成空间即可收敛）
+            for one in job:
+                if one[0] not in res:
+                    res.update(call_one([one], 1000))
+        if len(res) == len(job):
+            json.dump(res, open(cpath, "w", encoding="utf-8"),
+                      ensure_ascii=False)
+        return res
+
+    n_ok = 0
+    with ThreadPoolExecutor(max_workers=model_cfg["workers"]) as ex:
+        for res in ex.map(work, jobs):
+            out.update(res)
+            n_ok += len(res)
+    json.dump(out, open(out_path, "w", encoding="utf-8"), ensure_ascii=False)
+    print(f"[translate:{key}] 本轮完成 {n_ok}，总 {len(out)}/{len(items)}")
+    return out
+
+
+def build_dialog_pool(ids, zh_turns, root_dir):
+    """5882 中文 turn 逐条写入灵枢（对话原文做记忆，无摘要加工）。"""
+    root = os.path.join(root_dir, "pools", "dialog_zh")
+    if os.path.isfile(os.path.join(root, "_manifest.json")):
+        try:
+            m = json.load(open(os.path.join(root, "_manifest.json"),
+                               encoding="utf-8"))
+            if m == {"turns": len(ids)}:
+                from md_cg.mdcos import MdCGOS
+                return MdCGOS(os.path.join(root, "mem"))
+        except Exception:
+            pass
+    import shutil
+    from md_cg.mdcos import MdCGOS
+    if os.path.isdir(root):
+        shutil.rmtree(root)
+    os.makedirs(root, exist_ok=True)
+    cg = MdCGOS(os.path.join(root, "mem"))
+    for nid in ids:
+        cg.add(nid, zh_turns.get(nid) or "", layer="knowledge",
+               verification_basis="data")
+    json.dump({"turns": len(ids)},
+              open(os.path.join(root, "_manifest.json"), "w", encoding="utf-8"))
+    return cg
+
+
+READER_USER = "问题：{q}\n\n候选记忆：\n{cards}"
+
+
+def run_one(cfg, cache_dir, qid, qtype, zh_q, en_q, gold, cards_text, arm):
+    row = {"qid": qid, "qtype": qtype, "arm": arm, "gold": gold}
+
+    def call_reader():
+        return llm_chat(cfg["model"], cfg["base"], cfg["key"],
+                        [{"role": "system", "content": READER_SYS},
+                         {"role": "user", "content": READER_USER.format(
+                             q=zh_q, cards=cards_text)}],
+                        timeout=cfg["timeout"], max_tokens=300)
+
+    def call_judge():
+        return llm_chat(cfg["model"], cfg["base"], cfg["key"],
+                        [{"role": "system", "content": JUDGE_SYS},
+                         {"role": "user", "content":
+                          f"问题：{en_q}\ngold：{gold or ''}\n"
+                          f"模型回答：{row['answer']}"}],
+                        timeout=cfg["timeout"], max_tokens=200)
+
+    try:
+        pkey = f"{cfg['model']}|{arm}|{qid}|{cards_text[:200]}"
+        raw, _t1, _d1, _c1 = _cache_call(cache_dir, "reader",
+                                         hashlib.md5(
+                                             pkey.encode()).hexdigest(),
+                                         call_reader)
+        a = parse_json_field(raw, "answer")
+        row["answer"] = a if a is not None else (raw or "").strip()[:80]
+        jkey = f"{cfg['model']}|{en_q}|{gold}|{row['answer']}"
+        raw2, _t2, _d2, _c2 = _cache_call(
+            cache_dir, "judge", hashlib.md5(jkey.encode()).hexdigest(),
+            call_judge)
+        v = parse_json_field(raw2, "verdict")
+        row["verdict"] = v if v in ("correct", "incorrect", "refused") else None
+        if row["verdict"] is None:
+            row["error"] = f"judge 不可解析 {(raw2 or '')[:60]}"
+    except Exception as exc:                                # noqa: BLE001
+        row["error"] = f"{type(exc).__name__}: {exc}"
+        row.setdefault("answer", "")
+        row["verdict"] = None
+    return row
+
+
+def report(rows, title):
+    ok = [r for r in rows if r.get("verdict")]
+    n = max(len(ok), 1)
+    acc = sum(1 for r in ok if r["verdict"] == "correct")
+    out = {"n": len(rows), "judged": len(ok), "acc": acc,
+           "acc_pct": 100.0 * acc / n, "refused": sum(
+               1 for r in ok if r["verdict"] == "refused")}
+    by = {}
+    for qt in ("single_hop", "multi_hop", "temporal_reasoning",
+               "adversarial", "open_domain"):
+        rs = [r for r in ok if r["qtype"] == qt]
+        if rs:
+            c = sum(1 for r in rs if r["verdict"] == "correct")
+            by[qt] = f"{100.0*c/len(rs):.1f}%({c}/{len(rs)})"
+    print(f"  {title:<14} QA准确率={out['acc_pct']:5.1f}%"
+          f"（{acc}/{len(ok)}）  拒答={out['refused']}  分题型: {by}")
+    return out
+
+
+def main(argv=None):
+    ap = argparse.ArgumentParser(description="LoCoMo QA 同口径对照（中文全对话）")
+    ap.add_argument("--data-root", default=DEFAULT_ROOT)
+    ap.add_argument("--n", type=int, default=0, help="题数（0=全量500）")
+    ap.add_argument("--model", default="deepseek-chat")
+    ap.add_argument("--base-url", default="https://api.deepseek.com/v1")
+    ap.add_argument("--key", default="")
+    ap.add_argument("--workers", type=int, default=6)
+    ap.add_argument("--timeout", type=int, default=180)
+    ap.add_argument("--k", type=int, default=10)
+    ap.add_argument("--translate", action="store_true",
+                    help="只做翻译阶段（5882 turn + 500 问句）")
+    ap.add_argument("--lm-studio", action="store_true",
+                    help="翻译走 LM Studio 本地模型（localhost:1234/v1，"
+                         "自动探测模型名；reader/judge 仍用 --model）")
+    ap.add_argument("--batch-turns", type=int, default=8,
+                    help="翻译每请求 turn 数（LM Studio 建议 5）")
+    ap.add_argument("--batch-q", type=int, default=10,
+                    help="问句翻译每请求数（LM Studio 建议 6）")
+    ap.add_argument("--quick", action="store_true")
+    a = ap.parse_args(argv)
+    if a.quick:
+        a.n = 3
+    key = a.key or os.environ.get("DEEPSEEK_API_KEY") or ""
+    if not key:
+        print("[error] 需要 DEEPSEEK_API_KEY 或 --key")
+        return 2
+    cfg = {"model": a.model, "base": a.base_url, "key": key,
+           "timeout": a.timeout, "workers": a.workers}
+    print(f"[locomo_qa] 归一层开关 MDCG_UNIFY_QUERY="
+          f"{os.environ.get('MDCG_UNIFY_QUERY', '1')}（建议 0，见模块头注）")
+
+    ids, txts = load_corpus()
+    answers = json.load(open(ANSWERS, encoding="utf-8"))
+    ours = [json.loads(l) for l in open(os.path.join(
+        HERE, "data", "benchmarks", "locomo-zh-500", "questions500.jsonl"),
+        encoding="utf-8") if l.strip()]
+    qs = ours[:a.n] if a.n else ours
+
+    if a.translate or not (os.path.isfile(ZH_TURNS)
+                           and os.path.isfile(ZH_QUERIES)):
+        tcfg = cfg
+        if a.lm_studio:
+            with urllib.request.urlopen(
+                    "http://localhost:1234/v1/models", timeout=8) as r:
+                mids = [m["id"] for m in
+                        json.loads(r.read().decode("utf-8"))["data"]
+                        if "embed" not in m["id"].lower()]
+            if not mids:
+                print("[error] LM Studio 无已加载 LLM（仅有 embedding 模型）")
+                return 2
+            tcfg = {"model": mids[0], "base": "http://localhost:1234/v1",
+                    "key": "lm-studio", "timeout": 600,
+                    "workers": min(a.workers, 3),
+                    # 顶层 reasoning_effort=none 是实测唯一能压住思考的传法
+                    # （chat_template_kwargs/enable_thinking 均无效，思考仍吃满
+                    # max_tokens；矩阵实验 2026-09-23：none → 0 reasoning 4s/批）
+                    "extra": {"reasoning_effort": "none"}}
+            print(f"[lm-studio] 使用本地模型 {mids[0]}（并发 {tcfg['workers']}）")
+        items_t = list(zip(ids, txts))
+        zh_turns = translate_all("turn", tcfg, items_t, a.batch_turns,
+                                 TRANS_TURN_SYS, ZH_TURNS,
+                                 max_tokens=2000 if a.lm_studio else 4000)
+        items_q = [(q["qid"], answers[q["qid"]]["en_q"]) for q in ours
+                   if q["qid"] in answers]
+        zh_qs = translate_all("q", tcfg, items_q, a.batch_q,
+                              TRANS_Q_SYS, ZH_QUERIES,
+                              max_tokens=2000 if a.lm_studio else 4000)
+        miss_t = len(ids) - len(zh_turns)
+        miss_q = len(items_q) - len(zh_qs)
+        print(f"[translate] 缺口 turn={miss_t} q={miss_q}"
+              f"（缺口>0 可重跑 --translate 续传）")
+        if a.translate:
+            return 0 if (miss_t == 0 and miss_q == 0) else 1
+
+    zh_turns = json.load(open(ZH_TURNS, encoding="utf-8"))
+    zh_qs = json.load(open(ZH_QUERIES, encoding="utf-8"))
+    cache_dir = os.path.join(a.data_root, "lq_cache")
+    os.makedirs(cache_dir, exist_ok=True)
+    from md_cg.mdcos import MdCGOS
+    cg = build_dialog_pool(ids, zh_turns, a.data_root)
+    # scene → 全对话中文（full-context 臂用）
+    scene_turns = {}
+    for nid in ids:
+        scene_turns.setdefault(nid.split("_session")[0], []).append(nid)
+
+    tasks = []
+    for q in qs:
+        qid = q["qid"]
+        if qid not in answers or qid not in zh_qs:
+            continue
+        gold = answers[qid]["answer"] or ""
+        zh_q = zh_qs[qid]
+        en_q = answers[qid]["en_q"]
+        # 臂1：灵枢检索注入（生产主形态阶梯路；judge 关——对话原文无 CCG 面）
+        res, _m = cg.search(zh_q, k=a.k, judge=False, record=False)
+        cards = "\n\n".join(f"【{i}】{zh_turns.get(r[0]['id'], '')}"
+                            for i, r in enumerate(res, 1))
+        tasks.append((qid, q.get("qtype"), zh_q, en_q, gold, cards,
+                      "retrieval"))
+        # 臂2：full-context（该 scene 全对话）
+        scene = qid.split("_q_")[0]
+        full = "\n".join(zh_turns.get(t, "") for t in scene_turns[scene])
+        tasks.append((qid, q.get("qtype"), zh_q, en_q, gold, full[:110000],
+                      "full_context"))
+    t0 = time.time()
+    with ThreadPoolExecutor(max_workers=a.workers) as ex:
+        rows = list(ex.map(lambda t: run_one(cfg, cache_dir, *t), tasks))
+    print(f"[locomo_qa] {len(tasks)} 次 reader+judge 耗时 {time.time()-t0:.0f}s")
+    errs = sum(1 for r in rows if r.get("error"))
+    print(f"判分失败 {errs} 行\n")
+    summary = {"meta": {"model": a.model, "n_q": len(qs), "k": a.k,
+                        "unify": os.environ.get("MDCG_UNIFY_QUERY", "1")}}
+    for arm in ("retrieval", "full_context"):
+        summary[arm] = report([r for r in rows if r["arm"] == arm], arm)
+    out = os.path.join(a.data_root, "results",
+                       f"locomo_qa_{time.strftime('%Y%m%d_%H%M%S')}.json")
+    json.dump({"summary": summary, "rows": rows},
+              open(out, "w", encoding="utf-8"), ensure_ascii=False, indent=1)
+    print(f"\n[locomo_qa] 明细 → {out}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/md_cg/bench_e2e_qa.py

@@ -0,0 +1,256 @@
+# -*- coding: utf-8 -*-
+"""端到端 QA 评测：pinpoint / answerability / decision-making（2026-09-23 · 外部建议）
+
+评测问题（外部建议口径：hit@k 是上界，端到端要答对题）：
+  pinpoint        干扰浓度升高（level 0 → 4），reader 从候选记忆中抽取正确事实的
+                  QA 准确率退化多少？
+  answerability   adversarial 题（上游设计 gold 为空、正确行为=拒答）：reader 会不会
+                  被干扰节点骗去编答案？
+  decision-making 裁决层是否改变下游行为——同一 reader，候选分别来自
+                  arm_base（裸融合 top10）与 arm_firewall（证据防火墙重排 top10），
+                  QA 准确率有无差异？
+
+管线（全部真实 LLM，deepseek-chat，temperature=0，带缓存）：
+  检索（与 bench_e2e_judge 同池同采样同臂）→ reader 只据候选卡作答
+  （不足则答「无法确定」）→ judge 对 gold 语义判等（correct/incorrect/refused）。
+  gold 答案来自上游 LoCoMo 原始标注（mteb/LoCoMo 1976 题英文题面 ↔
+  snap-research/locomo10.json 逐题文本精确匹配，500/500 覆盖；
+  113 题 adversarial 的 gold 为空 = 拒答语义）。
+
+诚实边界：
+  · reader 同时拿中文关键词串与上游英文原题（真实用户问题面），记忆卡为中文，
+    judge 对英文 gold 做跨语语义判等——这是双语口径，非纯单语评测。
+  · judge 亦为 LLM（LLM-as-a-judge），其判分自身有方差；refused 由 judge 判
+    reader 文本，边界例（半猜测半拒答）按 judge 裁决计入，不人工复核（v1）。
+  · level 2 不入 QA 层（0 vs 4 已给退化区间，节省调用）。
+
+跑法：
+  python -X utf8 -m md_cg.bench_e2e_qa --quick          # 冒烟 3 题
+  python -X utf8 -m md_cg.bench_e2e_qa                  # 全量 120 题 × 4 组
+依赖：DEEPSEEK_API_KEY；answers_map 由上游对齐生成（见 --answers 默认路径）。
+"""
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import os
+import sys
+import time
+from concurrent.futures import ThreadPoolExecutor
+
+HERE = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+if HERE not in sys.path:
+    sys.path.insert(0, HERE)
+
+from md_cg.bench_e2e_judge import (                     # noqa: E402
+    DEFAULT_ROOT, load_src, speakers_by_scene, build_pool, run_det_arms,
+    llm_chat)
+
+READER_SYS = (
+    "你是记忆问答器。只根据给定的记忆卡回答问题，不要用记忆卡以外的知识。"
+    "记忆卡不足以确定答案时，回答「无法确定」。答案尽量简短（不超过20字），"
+    "不要解释。\n只输出一行 JSON：{\"answer\": \"…\"}"
+)
+
+JUDGE_SYS = (
+    "你是 QA 评分器。给定问题、参考答案（gold）与模型回答，判一个结论：\n"
+    "- correct：回答与 gold 语义等价（中英跨语判等；日期/人名/实体须一致，"
+    "表述差异可容忍）\n"
+    "- incorrect：回答了但语义与 gold 不等价（含被错误记忆误导的答案）\n"
+    "- refused：回答表示无法确定/不知道/记忆未提及\n"
+    "gold 为空字符串时（对抗题）：回答「无法确定」= correct，给出任何具体"
+    "答案 = incorrect。\n只输出一行 JSON：{\"verdict\": \"correct\"}"
+)
+
+
+def _cache_call(cache_dir, kind, payload_key, fn):
+    """LLM 调用缓存：payload_key 相同直接回放（qa 层可复现性与省钱）。"""
+    hexk = hashlib.md5(payload_key.encode("utf-8")).hexdigest()
+    path = os.path.join(cache_dir, f"{kind}_{hexk}.json")
+    if os.path.isfile(path):
+        d = json.load(open(path, encoding="utf-8"))
+        return d["out"], d.get("tokens", 0), 0.0, True
+    out, usage, dt = fn()
+    json.dump({"out": out, "tokens": int(usage.get("total_tokens") or 0)},
+              open(path, "w", encoding="utf-8"), ensure_ascii=False)
+    return out, int(usage.get("total_tokens") or 0), dt, False
+
+
+def parse_json_field(raw, field):
+    s = raw or ""
+    i, j = s.find("{"), s.rfind("}")
+    if i >= 0 and j > i:
+        try:
+            return json.loads(s[i:j + 1]).get(field)
+        except ValueError:
+            pass
+    return None
+
+
+def run_qa_one(cfg, cache_dir, q, ans, cand_nodes, arm, level):
+    """一题一组：reader 作答 → judge 判分。返回行（异常行 error 字段）。"""
+    cards = "\n\n".join(
+        f"【{i}】{str(nd.get('content') or '')[:400]}"
+        for i, nd in enumerate(cand_nodes, 1))
+    r_user = (f"问题（中文关键词）：{q['question']}\n"
+              f"问题（英文原题）：{ans.get('en_q') or ''}\n\n候选记忆卡：\n{cards}")
+    tag = f"{q['qid']}|{arm}|L{level}"
+
+    def call_reader():
+        return llm_chat(cfg["model"], cfg["base"], cfg["key"],
+                        [{"role": "system", "content": READER_SYS},
+                         {"role": "user", "content": r_user}],
+                        timeout=cfg["timeout"], max_tokens=300)
+
+    def call_judge():
+        return llm_chat(cfg["model"], cfg["base"], cfg["key"],
+                        [{"role": "system", "content": JUDGE_SYS},
+                         {"role": "user", "content":
+                          f"问题：{ans.get('en_q') or q['question']}\n"
+                          f"gold：{ans.get('answer') or ''}\n"
+                          f"模型回答：{row['answer']}"}],
+                        timeout=cfg["timeout"], max_tokens=200)
+
+    row = {"qid": q["qid"], "qtype": q.get("qtype"), "arm": arm,
+           "level": level, "gold": ans.get("answer") or ""}
+    try:
+        raw, tok_r, dt_r, _c = _cache_call(
+            cache_dir, "reader", cfg["model"] + "|" + r_user, call_reader)
+        ans_txt = parse_json_field(raw, "answer")
+        if ans_txt is None:
+            ans_txt = (raw or "").strip()[:80]
+        row["answer"] = ans_txt
+        raw2, tok_j, dt_j, _c2 = _cache_call(
+            cache_dir, "judge",
+            cfg["model"] + "|" + row["gold"] + "|" + ans_txt + "|"
+            + (ans.get("en_q") or ""), call_judge)
+        verdict = parse_json_field(raw2, "verdict")
+        row["verdict"] = verdict if verdict in ("correct", "incorrect",
+                                                "refused") else None
+        row["tokens"] = tok_r + tok_j
+        row["latency"] = dt_r + dt_j
+        if row["verdict"] is None:
+            row["error"] = f"judge 不可解析：{(raw2 or '')[:80]}"
+    except Exception as exc:                            # noqa: BLE001
+        row["error"] = f"{type(exc).__name__}: {exc}"
+        row.setdefault("answer", "")
+        row["verdict"] = None
+        row["tokens"] = row.get("tokens", 0)
+        row["latency"] = row.get("latency", 0.0)
+    row["_tag"] = tag
+    return row
+
+
+def qa_report(rows):
+    n = len(rows)
+    ok = [r for r in rows if r.get("verdict")]
+    adv = [r for r in ok if r.get("qtype") == "adversarial"]
+    norm = [r for r in ok if r.get("qtype") != "adversarial"]
+    c = lambda rs, v: sum(1 for r in rs if r["verdict"] == v)   # noqa: E731
+    st = {
+        "n": n, "judged": len(ok), "errors": n - len(ok),
+        "acc": c(ok, "correct") / max(len(ok), 1),
+        "incorrect": c(ok, "incorrect"), "refused": c(ok, "refused"),
+        "norm_n": len(norm),
+        "norm_acc": c(norm, "correct") / max(len(norm), 1),
+        "norm_refused": c(norm, "refused"),
+        "adv_n": len(adv),
+        "adv_correct_refusal": c(adv, "correct"),
+        "adv_fooled": c(adv, "incorrect"),
+        "tokens": sum(r.get("tokens", 0) for r in rows),
+    }
+    return st
+
+
+def main(argv=None):
+    ap = argparse.ArgumentParser(description="端到端 QA：pinpoint/answerability")
+    ap.add_argument("--data-root", default=DEFAULT_ROOT)
+    ap.add_argument("--answers",
+                    default=os.path.join(DEFAULT_ROOT, "upstream",
+                                         "answers_map.json"))
+    ap.add_argument("--n", type=int, default=120)
+    ap.add_argument("--seed", type=int, default=7)
+    ap.add_argument("--levels", default="0,4")
+    ap.add_argument("--model", default="deepseek-chat")
+    ap.add_argument("--base-url", default="https://api.deepseek.com/v1")
+    ap.add_argument("--key", default="")
+    ap.add_argument("--workers", type=int, default=6)
+    ap.add_argument("--timeout", type=int, default=120)
+    ap.add_argument("--k", type=int, default=10)
+    ap.add_argument("--quick", action="store_true")
+    a = ap.parse_args(argv)
+    if a.quick:
+        a.n = 3
+    key = a.key or os.environ.get("DEEPSEEK_API_KEY") or ""
+    if not key:
+        print("[error] 需要 DEEPSEEK_API_KEY 或 --key")
+        return 2
+    if not os.path.isfile(a.answers):
+        print(f"[error] 找不到答案映射 {a.answers}（先跑上游对齐生成）")
+        return 2
+
+    levels = [int(x) for x in a.levels.split(",") if x.strip() != ""]
+    cache_dir = os.path.join(a.data_root, "qa_cache")
+    os.makedirs(cache_dir, exist_ok=True)
+    res_dir = os.path.join(a.data_root, "results")
+    os.makedirs(res_dir, exist_ok=True)
+
+    import random
+    corpus, questions = load_src()
+    corpus_map = {c["id"]: c for c in corpus}
+    spk = speakers_by_scene(corpus)
+    answers = json.load(open(a.answers, encoding="utf-8"))
+    sample = sorted(random.Random(a.seed).sample(
+        questions, min(a.n, len(questions))), key=lambda q: q["qid"])
+    print(f"[e2e_qa] 采样 {len(sample)} 题（seed={a.seed}）× levels={levels}"
+          f" × 候选臂 base/firewall × {a.model}")
+
+    cfg = {"model": a.model, "base": a.base_url, "key": key,
+           "timeout": a.timeout}
+    all_rows, summary = [], {"meta": {"seed": a.seed, "n": len(sample),
+                                      "model": a.model, "levels": levels}}
+    for lv in levels:
+        cg, n_synth = build_pool(lv, corpus, questions, corpus_map, spk,
+                                 a.data_root)
+        det = []
+        for q in sample:
+            r = run_det_arms(cg, q, k=a.k)
+            r["q"] = q
+            det.append(r)
+        tasks = []
+        for r in det:
+            for arm in ("base", "firewall"):
+                nodes = r["base_nodes"][:a.k] if arm == "base" \
+                    else r["fw_nodes"][:a.k]
+                tasks.append((r["q"], answers.get(r["q"]["qid"], {}),
+                              nodes, arm, lv))
+        t0 = time.time()
+        with ThreadPoolExecutor(max_workers=a.workers) as ex:
+            rows = list(ex.map(lambda t: run_qa_one(cfg, cache_dir, *t),
+                               tasks))
+        all_rows.extend(rows)
+        print(f"\n===== level {lv}（池={len(corpus)+n_synth}）"
+              f" · QA 耗时 {time.time()-t0:.1f}s =====")
+        print(f"{'臂':<10}{'QA准确率':>9}{'常规准确率':>10}{'常规拒答':>8}"
+              f"{'对抗正确拒答':>12}{'对抗被骗':>8}{'判分失败':>8}")
+        for arm in ("base", "firewall"):
+            st = qa_report([r for r in rows if r["arm"] == arm])
+            print(f"{arm:<10}{st['acc']*100:>8.1f}%{st['norm_acc']*100:>9.1f}%"
+                  f"{st['norm_refused']:>7d}/{st['norm_n']}"
+                  f"{st['adv_correct_refusal']:>10d}/{st['adv_n']}"
+                  f"{st['adv_fooled']:>7d}"
+                  f"{st['errors']:>7d}")
+            summary[f"L{lv}_{arm}"] = st
+    out = os.path.join(res_dir,
+                       f"qa_{time.strftime('%Y%m%d_%H%M%S')}.json")
+    json.dump({"summary": summary,
+               "rows": [{k: v for k, v in r.items() if k != "q"}
+                        for r in all_rows]},
+              open(out, "w", encoding="utf-8"), ensure_ascii=False, indent=1)
+    print(f"\n[e2e_qa] 完成。明细 → {out}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())