@furongjun1999/dsh-memory 0.4.11 → 0.5.1

package/docs/eval/bench_lingshu_self/bench_self.py

@@ -0,0 +1,140 @@
+# -*- coding: utf-8 -*-
+"""灵枢自库端到端检索评测（中文 · 自然问句 vs 关键词对照 · 生产读路径）
+
+实测方式（用户指定）：mdcg 的读检索——spawn python -m md_cg.mcp_server（stdio JSON-RPC，
+与 DSH 插件 src/lib/mdcg_client.ts 同款生产路径），逐题调 cg(op=read, query=...)，
+只收 top-10 候选的 node.id 与 score，判 gold 节点命中。
+
+gold = 已落盘（索引可见）的已知节点；审核队列 pending 提案对检索不可见（如实记录为
+发现，不进评测集）。运行环境继承 MDCG_TOKEN（designer/internal 档）。
+"""
+import json, subprocess, sys, os, time
+
+REPO = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+
+# ── 测试集：20 题 · 自然问句 → gold（已落盘节点）──────────────────────────
+CASES = [
+    # (自然问句, [gold_node_ids], 关键词对照组)
+    ("hive 的 serve 怎么判断还活着？", ["mem_1789612355948"], "serve 存活 判据 三层 心跳 pid"),
+    ("双实例互验怎么防止验证者作弊？", ["mem_1789602635975"], "互验 判据冻结 A3 验证者 指纹"),
+    ("任务派发应该按什么原则分层决策？", ["mem_1789605410646"], "任务派发 三层决策 L1 L2 L3 分层原则"),
+    ("蜂群健康评分的 integrity 因子是怎么算的？", ["swarm_v061_integrity_fix"], "蜂群 integrity 健康评分 验签 aggregate_report"),
+    ("全局截断 cap 会造成哪些检索可见性问题？", ["mem_1789511615088"], "GLOBAL_CAP 截断 检索 可见性 T3 importance"),
+    ("冷验证队列的回执什么情况下会报假成功？", ["mem_1789897251107"], "冷队列 回执 诚实 patrol_check propagate_depth"),
+    ("英文语料用中文索引检索效果怎么样？", ["mem_1789132609619"], "LongMemEval 中文层 探针 英文原文 载荷回填"),
+    ("检索扩散 reach 功能默认是开的吗？", ["mem_1789795794376"], "reach opt-in MDCG_REACH 收敛层 接线"),
+    ("记忆怎么按热度分层来加速检索？", ["mem_1789803443800"], "热温冷 分层检索 热路径缓存 冷路径 异步验证"),
+    ("LLM 批次修复第二轮还是不过怎么办？", ["mem_1789666192335"], "fix pass 二轮 REJECT held 重生成"),
+    ("子任务上下文快满了怎么交接？", ["mem_1789548798424"], "handoff 满上下文 换人续跑 progress.jsonl"),
+    ("记忆条目的过期和观测变旧怎么区分？", ["mem_1789539356385"], "stale 时间窗 失效判据 observation_aged observation_aged 信息差"),
+    ("代码提交推送到公开仓库前要做什么检查？", ["mem_1789612355948"], "提交 推送 双清单 门禁 隐私脱敏 扫描"),
+    ("多个智能体共用一份权限时怎么区分是谁干的？", ["mem_1789122153945"], "嵌套身份 权限域 归因 进程 会话"),
+    ("DSH 的会话 id 关机重启后会变吗？", ["mem_1789100285255"], "DSH 会话 id 持久 projcache resume"),
+    ("对话日志怎么自动挖出错误和修复的配对？", ["node_02fe882d_1788905562143", "doc_4a3accff94d6"], "fix pairs 自动挖掘 行为日志 错误 修复 负记忆"),
+    ("令牌放在系统环境变量里有什么安全风险？", ["mem_1789099723060"], "MDCG_TOKEN Machine 作用域 部署 按 身份授权 缺口"),
+    ("服务进程的随机会话 id 会带来什么问题？", ["mem_1789099936556"], "Principal.session 进程随机 uuid 会话身份 前置缺口"),
+    ("查询词完全命不中桶的时候检索会怎么走？", ["mem_1789511615088"], "no_key_match T3 全量兜底 importance 排序"),
+    ("白箱知识库在进程里怎么被调用？", ["code_d98d1f5aa78e", "doc_3942cfca7bdf"], "白箱 引擎 进程内 门面 engine call_tool"),
+]
+
+
+class Client:
+    def __init__(self):
+        env = dict(os.environ)
+        if not env.get("MDCG_ROOT"):
+            raise SystemExit("须设 MDCG_ROOT 指向被测记忆库根（见同目录实测报告的复现节）")
+        self.p = subprocess.Popen(
+            [sys.executable, "-X", "utf8", "-m", "md_cg.mcp_server"],
+            cwd=REPO, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+            stderr=subprocess.DEVNULL, env=env,
+        )
+        self._id = 0
+        self._call("initialize", {"protocolVersion": "2024-11-05", "capabilities": {},
+                                  "clientInfo": {"name": "self-bench", "version": "1.0"}})
+        self.p.stdin.write((json.dumps({"jsonrpc": "2.0", "method": "notifications/initialized"}) + "\n").encode())
+        self.p.stdin.flush()
+
+    def _call(self, method, params):
+        self._id += 1
+        req = {"jsonrpc": "2.0", "id": self._id, "method": method, "params": params}
+        self.p.stdin.write((json.dumps(req, ensure_ascii=False) + "\n").encode("utf-8"))
+        self.p.stdin.flush()
+        deadline = time.time() + 120
+        while time.time() < deadline:
+            line = self.p.stdout.readline().decode("utf-8", "replace").strip()
+            if not line:
+                continue
+            try:
+                return json.loads(line)
+            except json.JSONDecodeError:
+                continue  # 启动横幅等非 JSON 行
+        return {"error": "timeout-waiting-json-line"}
+
+    def read(self, query):
+        r = self._call("tools/call", {"name": "cg", "arguments": {"op": "read", "query": query}})
+        # 返回 {result:{content:[{type:text,text:json串}]}} 或 error
+        if "error" in r:
+            return {"error": str(r["error"])[:120]}
+        try:
+            txt = r["result"]["content"][0]["text"]
+            return json.loads(txt)
+        except Exception as e:
+            return {"error": f"parse:{e}"}
+
+    def close(self):
+        try:
+            self.p.stdin.close(); self.p.wait(timeout=5)
+        except Exception:
+            self.p.kill()
+
+
+def top_ids(resp, k=10):
+    if not isinstance(resp, dict) or "results" not in resp:
+        return [], (resp.get("error") if isinstance(resp, dict) else "no-results")[:80], None
+    rs = resp["results"][:k]
+    ids = [x["node"]["id"] for x in rs]
+    meta = resp.get("meta", {})
+    return ids, meta.get("tier"), [round(x.get("score", 0), 4) for x in rs]
+
+
+def hit_stats(hits_pos):
+    h1 = sum(1 for p in hits_pos if p is not None and p < 1)
+    h5 = sum(1 for p in hits_pos if p is not None and p < 5)
+    h10 = sum(1 for p in hits_pos if p is not None and p < 10)
+    return h1, h5, h10
+
+
+def main():
+    cli = Client()
+    rows = []
+    for nat_q, golds, kw_q in CASES:
+        row = {"q": nat_q, "golds": golds}
+        for arm, q in (("nat", nat_q), ("kw", kw_q)):
+            resp = cli.read(q)
+            ids, tier, scores = top_ids(resp)
+            pos = next((i for i, nid in enumerate(ids) if nid in golds), None)
+            row[arm] = {"pos": pos, "tier": tier, "top1": ids[0] if ids else None,
+                        "top1_is_gold": bool(ids and ids[0] in golds),
+                        "n_res": len(ids)}
+            time.sleep(0.1)
+        rows.append(row)
+        m = row["nat"]
+        print(f"[{m['tier'] or '?':>16}] nat pos={str(m['pos']):>4}  kw pos={str(row['kw']['pos']):>4}  {nat_q[:34]}")
+    cli.close()
+
+    for arm in ("nat", "kw"):
+        h1, h5, h10 = hit_stats([r[arm]["pos"] for r in rows])
+        n = len(rows)
+        print(f"\n===== {arm}: hit@1={h1}/{n} ({h1/n:.0%})  hit@5={h5}/{n} ({h5/n:.0%})  hit@10={h10}/{n} ({h10/n:.0%})")
+    tiers = {}
+    for r in rows:
+        tiers[r["nat"]["tier"]] = tiers.get(r["nat"]["tier"], 0) + 1
+    print("nat tier 分布:", tiers)
+    out = os.path.join(os.path.dirname(os.path.abspath(__file__)), "self_bench_result.json")
+    with open(out, "w", encoding="utf-8") as f:
+        json.dump(rows, f, ensure_ascii=False, indent=1)
+    print("明细 →", out)
+
+
+if __name__ == "__main__":
+    main()

package/docs/eval/bench_lingshu_self/self_bench_result.json

@@ -0,0 +1,404 @@
+[
+ {
+  "q": "hive 的 serve 怎么判断还活着？",
+  "golds": [
+   "mem_1789612355948"
+  ],
+  "nat": {
+   "pos": null,
+   "tier": "T2_global_like",
+   "top1": "mem_1789552857538",
+   "top1_is_gold": false,
+   "n_res": 10
+  },
+  "kw": {
+   "pos": null,
+   "tier": "T2_global_like",
+   "top1": "mem_1789612140901",
+   "top1_is_gold": false,
+   "n_res": 10
+  }
+ },
+ {
+  "q": "双实例互验怎么防止验证者作弊？",
+  "golds": [
+   "mem_1789602635975"
+  ],
+  "nat": {
+   "pos": 1,
+   "tier": "T2_global_like",
+   "top1": "mem_1789603603896",
+   "top1_is_gold": false,
+   "n_res": 10
+  },
+  "kw": {
+   "pos": 0,
+   "tier": "T3_global_scan",
+   "top1": "mem_1789602635975",
+   "top1_is_gold": true,
+   "n_res": 10
+  }
+ },
+ {
+  "q": "任务派发应该按什么原则分层决策？",
+  "golds": [
+   "mem_1789605410646"
+  ],
+  "nat": {
+   "pos": 0,
+   "tier": "T2_global_like",
+   "top1": "mem_1789605410646",
+   "top1_is_gold": true,
+   "n_res": 10
+  },
+  "kw": {
+   "pos": 0,
+   "tier": "T3_global_scan",
+   "top1": "mem_1789605410646",
+   "top1_is_gold": true,
+   "n_res": 10
+  }
+ },
+ {
+  "q": "蜂群健康评分的 integrity 因子是怎么算的？",
+  "golds": [
+   "swarm_v061_integrity_fix"
+  ],
+  "nat": {
+   "pos": 0,
+   "tier": "T3_global_scan",
+   "top1": "swarm_v061_integrity_fix",
+   "top1_is_gold": true,
+   "n_res": 10
+  },
+  "kw": {
+   "pos": 0,
+   "tier": "T3_global_scan",
+   "top1": "swarm_v061_integrity_fix",
+   "top1_is_gold": true,
+   "n_res": 10
+  }
+ },
+ {
+  "q": "全局截断 cap 会造成哪些检索可见性问题？",
+  "golds": [
+   "mem_1789511615088"
+  ],
+  "nat": {
+   "pos": 2,
+   "tier": "T3_global_scan",
+   "top1": "node_132deca4_1789537606255",
+   "top1_is_gold": false,
+   "n_res": 10
+  },
+  "kw": {
+   "pos": 1,
+   "tier": "T2_global_like",
+   "top1": "node_41abeae8_1789510901539",
+   "top1_is_gold": false,
+   "n_res": 10
+  }
+ },
+ {
+  "q": "冷验证队列的回执什么情况下会报假成功？",
+  "golds": [
+   "mem_1789897251107"
+  ],
+  "nat": {
+   "pos": 2,
+   "tier": "T2_global_like",
+   "top1": "mem_1789722858540",
+   "top1_is_gold": false,
+   "n_res": 10
+  },
+  "kw": {
+   "pos": 0,
+   "tier": "T3_global_scan",
+   "top1": "mem_1789897251107",
+   "top1_is_gold": true,
+   "n_res": 10
+  }
+ },
+ {
+  "q": "英文语料用中文索引检索效果怎么样？",
+  "golds": [
+   "mem_1789132609619"
+  ],
+  "nat": {
+   "pos": null,
+   "tier": "T2_global_like",
+   "top1": "mem_1789354548231",
+   "top1_is_gold": false,
+   "n_res": 10
+  },
+  "kw": {
+   "pos": 0,
+   "tier": "T2_global_like",
+   "top1": "mem_1789132609619",
+   "top1_is_gold": true,
+   "n_res": 10
+  }
+ },
+ {
+  "q": "检索扩散 reach 功能默认是开的吗？",
+  "golds": [
+   "mem_1789795794376"
+  ],
+  "nat": {
+   "pos": null,
+   "tier": "T3_global_scan",
+   "top1": "code_3d03ed70458d",
+   "top1_is_gold": false,
+   "n_res": 10
+  },
+  "kw": {
+   "pos": 0,
+   "tier": "T2_global_like",
+   "top1": "mem_1789795794376",
+   "top1_is_gold": true,
+   "n_res": 10
+  }
+ },
+ {
+  "q": "记忆怎么按热度分层来加速检索？",
+  "golds": [
+   "mem_1789803443800"
+  ],
+  "nat": {
+   "pos": null,
+   "tier": "T2_global_like",
+   "top1": "mem_1789274628336",
+   "top1_is_gold": false,
+   "n_res": 10
+  },
+  "kw": {
+   "pos": 1,
+   "tier": "T2_global_like",
+   "top1": "code_0602d0907acc",
+   "top1_is_gold": false,
+   "n_res": 10
+  }
+ },
+ {
+  "q": "LLM 批次修复第二轮还是不过怎么办？",
+  "golds": [
+   "mem_1789666192335"
+  ],
+  "nat": {
+   "pos": null,
+   "tier": "T2_global_like",
+   "top1": "mem_1789274628336",
+   "top1_is_gold": false,
+   "n_res": 10
+  },
+  "kw": {
+   "pos": 3,
+   "tier": "T2_global_like",
+   "top1": "mem_1789665204611",
+   "top1_is_gold": false,
+   "n_res": 10
+  }
+ },
+ {
+  "q": "子任务上下文快满了怎么交接？",
+  "golds": [
+   "mem_1789548798424"
+  ],
+  "nat": {
+   "pos": 7,
+   "tier": "T2_global_like",
+   "top1": "mem_1789599787541",
+   "top1_is_gold": false,
+   "n_res": 10
+  },
+  "kw": {
+   "pos": 0,
+   "tier": "T2_global_like",
+   "top1": "mem_1789548798424",
+   "top1_is_gold": true,
+   "n_res": 10
+  }
+ },
+ {
+  "q": "记忆条目的过期和观测变旧怎么区分？",
+  "golds": [
+   "mem_1789539356385"
+  ],
+  "nat": {
+   "pos": null,
+   "tier": "T2_global_like",
+   "top1": "node_ef7bc1ee_1786918581699",
+   "top1_is_gold": false,
+   "n_res": 10
+  },
+  "kw": {
+   "pos": null,
+   "tier": "T3_global_scan",
+   "top1": "mem_1789540113805",
+   "top1_is_gold": false,
+   "n_res": 10
+  }
+ },
+ {
+  "q": "代码提交推送到公开仓库前要做什么检查？",
+  "golds": [
+   "mem_1789612355948"
+  ],
+  "nat": {
+   "pos": 9,
+   "tier": "T2_global_like",
+   "top1": "mem_1789806369843",
+   "top1_is_gold": false,
+   "n_res": 10
+  },
+  "kw": {
+   "pos": 0,
+   "tier": "T2_global_like",
+   "top1": "mem_1789612355948",
+   "top1_is_gold": true,
+   "n_res": 10
+  }
+ },
+ {
+  "q": "多个智能体共用一份权限时怎么区分是谁干的？",
+  "golds": [
+   "mem_1789122153945"
+  ],
+  "nat": {
+   "pos": null,
+   "tier": "T2_global_like",
+   "top1": "node_76728d45_1786918560368",
+   "top1_is_gold": false,
+   "n_res": 10
+  },
+  "kw": {
+   "pos": 0,
+   "tier": "T2_global_like",
+   "top1": "mem_1789122153945",
+   "top1_is_gold": true,
+   "n_res": 10
+  }
+ },
+ {
+  "q": "DSH 的会话 id 关机重启后会变吗？",
+  "golds": [
+   "mem_1789100285255"
+  ],
+  "nat": {
+   "pos": 0,
+   "tier": "T2_global_like",
+   "top1": "mem_1789100285255",
+   "top1_is_gold": true,
+   "n_res": 10
+  },
+  "kw": {
+   "pos": 0,
+   "tier": "T2_global_like",
+   "top1": "mem_1789100285255",
+   "top1_is_gold": true,
+   "n_res": 10
+  }
+ },
+ {
+  "q": "对话日志怎么自动挖出错误和修复的配对？",
+  "golds": [
+   "node_02fe882d_1788905562143",
+   "doc_4a3accff94d6"
+  ],
+  "nat": {
+   "pos": 3,
+   "tier": "T2_global_like",
+   "top1": "node_030c84b7_1786688525595",
+   "top1_is_gold": false,
+   "n_res": 10
+  },
+  "kw": {
+   "pos": 1,
+   "tier": "T2_global_like",
+   "top1": "code_4297379afe64",
+   "top1_is_gold": false,
+   "n_res": 10
+  }
+ },
+ {
+  "q": "令牌放在系统环境变量里有什么安全风险？",
+  "golds": [
+   "mem_1789099723060"
+  ],
+  "nat": {
+   "pos": 0,
+   "tier": "T2_global_like",
+   "top1": "mem_1789099723060",
+   "top1_is_gold": true,
+   "n_res": 10
+  },
+  "kw": {
+   "pos": 0,
+   "tier": "T2_global_like",
+   "top1": "mem_1789099723060",
+   "top1_is_gold": true,
+   "n_res": 10
+  }
+ },
+ {
+  "q": "服务进程的随机会话 id 会带来什么问题？",
+  "golds": [
+   "mem_1789099936556"
+  ],
+  "nat": {
+   "pos": 3,
+   "tier": "T2_global_like",
+   "top1": "mem_1789100285255",
+   "top1_is_gold": false,
+   "n_res": 10
+  },
+  "kw": {
+   "pos": 0,
+   "tier": "T2_global_like",
+   "top1": "mem_1789099936556",
+   "top1_is_gold": true,
+   "n_res": 10
+  }
+ },
+ {
+  "q": "查询词完全命不中桶的时候检索会怎么走？",
+  "golds": [
+   "mem_1789511615088"
+  ],
+  "nat": {
+   "pos": null,
+   "tier": "T2_global_like",
+   "top1": "node_subject_4939401561_1787018087635",
+   "top1_is_gold": false,
+   "n_res": 10
+  },
+  "kw": {
+   "pos": 0,
+   "tier": "T3_global_scan",
+   "top1": "mem_1789511615088",
+   "top1_is_gold": true,
+   "n_res": 10
+  }
+ },
+ {
+  "q": "白箱知识库在进程里怎么被调用？",
+  "golds": [
+   "code_d98d1f5aa78e",
+   "doc_3942cfca7bdf"
+  ],
+  "nat": {
+   "pos": 1,
+   "tier": "T2_global_like",
+   "top1": "code_a53cb473e34f",
+   "top1_is_gold": false,
+   "n_res": 10
+  },
+  "kw": {
+   "pos": 0,
+   "tier": "T3_global_scan",
+   "top1": "code_d98d1f5aa78e",
+   "top1_is_gold": true,
+   "n_res": 10
+  }
+ }
+]

package/docs/eval/bench_lingshu_self//347/201/265/346/236/242/350/207/252/345/272/223/347/253/257/345/210/260/347/253/257/346/243/200/347/264/242/345/256/236/346/265/213_v1.0.md

@@ -0,0 +1,38 @@
+# 灵枢自库端到端检索实测 v1.0（原生中文 · 自然问句 vs 关键词 · 生产读路径）
+
+> **口径**：被测库=灵枢自身在役记忆库（14095 节点 · 原生中文 · CCG 六要素为主写入面 · audit 23.9 万事件的真实累积库）；实测方式=mdcg 读检索生产路径——spawn `python -m md_cg.mcp_server`（stdio JSON-RPC，与 DSH 插件同款链路），逐题调 `cg(op=read, query=…)`，取 top-10 候选判 gold 命中。评测令牌为专用 designer 令牌（`tk_70f633dc8398`，internal 以上可见）。
+> **题集**：20 道自然中文问句（口语化、非节点原文词），gold=16 个已落盘已知节点（多题一 gold / 一题多 gold）；对照组=同题关键词版（从 gold 六要素词面构造）。脚本：`bench_self.py`（本目录）；逐题明细：`self_bench_result.json`。
+
+## 结果
+
+| 臂 | hit@1 | hit@5 | hit@10 |
+| ---- | ---- | ---- | ---- |
+| 自然问句 | 4/20（**20%**） | 10/20（**50%**） | 12/20（**60%**） |
+| 关键词对照 | 14/20（**70%**） | 18/20（**90%**） | 18/20（**90%**） |
+
+## 三条对照解读
+
+1. **与 LoCoMo 翻译语料口径对照（自然问句 40.2% / 派生题面 92%）**：原生中文 CCG 库上自然问句 **60% > 40.2%（+19.8pp）**——CCG 六要素写入面（功能名/生效条件等结构化中文槽）确实部分缩小了问句-答案词汇差，翻译噪声也被排除。
+2. **鸿沟同构成立**：自库上自然问句与关键词之间仍有 **30pp**（60% vs 90%）——与 LoCoMo 口径的 51.8pp 鸿沟同向。词汇差瓶颈不依赖语料来源，是词法检索的固有边界；**认知图概念桥接（自然问句 → 92%±）的必要性在本口径下再次成立**，且 60% 是桥接实施的自库基线起点。
+3. **失败定性（8 题 hit@10 外）**：6 题 kw 臂能命中（典型词汇差：问句「过期/变旧」vs 节点 `stale/observation_aged`、问句「快满了怎么交接」vs 节点 `handoff 满上下文`）——词面不同构是主因；2 题双臂皆败（Q1 serve 存活、Q12 stale 区分）经直读验证 gold 在库可检索，属**排序区分度问题**——「hive/serve」词面在众多蜂巢节点同现，gold 特异词（存活判据/三层）未入问句时被主题相近节点淹没（top-3 全为蜂巢编排类节点，score 0.44-0.59）。
+
+## 过程发现（评测副产物，两条）
+
+1. **审核队列对检索不可见**：`review_pending=22`——缺陷迭代九轮的归档提案全部 pending、索引不可见，`cg(op=read)` 检索不到（例：问「MCP 如何防打崩」返回的 top-10 全为授权/身份主题无关节点，第 10 轮 MCP 入口守卫归档不在候选）。知识从 propose 到可检索依赖设计者裁决，裁决积压=知识不可用。**建议：设计者批量裁决积压提案（review_cli），或为 pending 提案建立降级可见通道（另行设计）。**
+2. **自然问句零桶收敛**：tier 分布 17/20 走 `T2_global_like`（全局扫 13550 节点）、3/20 走 `T3_global_scan`，**没有一题走桶收敛（T0/T1）**——自然问句不命中英文桶键；#33 的 `backfill_bucket_zh` 中文别名回填未在本库执行过。全局兜底路径承担了全部自然问句流量，S1b 桶收敛对真实问句形态未起作用。
+
+## 复现
+
+```bash
+MDCG_ROOT=<真实库根> MDCG_TOKEN=<designer 令牌> \
+  python -X utf8 docs/eval/bench_lingshu_self/bench_self.py
+```
+
+> 本报告不含库根绝对路径与令牌明文；评测令牌 token_id `tk_70f633dc8398` 保留复测用，可随时 `python -m md_cg.tokens revoke` 吊销。
+
+## 边界
+
+- 20 题小样本，百分比仅作量级参考（±1 题 = ±5pp）；
+- gold 由编排会话指定（已知答案节点），非全库抽样；问句风格单一（工程事实型），未覆盖时间/条件/否定类问句；
+- 关键词臂由 gold 六要素词面构造，含轻度答案侧词汇先验（与 LoCoMo 口径的派生题面同性质）；
+- 被测库持续增长（audit 在写），两次运行数字可能小幅漂移。