@furongjun1999/dsh-memory 0.9.9 → 0.9.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. package/README.md +181 -150
  2. package/lib/hooks.d.ts +16 -1
  3. package/lib/hooks.js +53 -2
  4. package/lib/index.d.ts +3 -0
  5. package/lib/index.js +9 -1
  6. package/lib/tools.d.ts +8 -0
  7. package/lib/tools.js +32 -5
  8. package/md_cg/config_registry_bulk.py +488 -483
  9. package/md_cg/config_validate.py +5 -0
  10. package/md_cg/mcp_server.py +5446 -5324
  11. package/md_cg/mdcg.py +62 -4
  12. package/md_cg/mdcos.py +7259 -7159
  13. package/md_cg/refindex.py +11 -4
  14. package/md_cg/test_c3_transient_read_negative.py +797 -797
  15. package/md_cg/test_c8_search_rrf_gates.py +511 -502
  16. package/md_cg/test_domain_granularity.py +68 -0
  17. package/md_cg/test_exact_text_cache.py +194 -194
  18. package/md_cg/test_hotcache_tool.py +121 -121
  19. package/md_cg/test_index_snapshot_fastpath.py +72 -0
  20. package/md_cg/test_issue274_read_node_id_none.py +364 -364
  21. package/md_cg/test_issue623_fullwidth_symmetry.py +66 -0
  22. package/md_cg/test_p28_refcheck.py +7 -7
  23. package/md_cg/test_p3_legacy_closure.py +441 -441
  24. package/md_cg/test_retr_gates_prodpath.py +11 -1
  25. package/md_cg/test_search_async.py +84 -0
  26. package/md_cg/test_search_bm25_default.py +60 -60
  27. package/md_cg/test_search_prefilter.py +85 -0
  28. package/md_cg/test_untrusted_audit.py +55 -55
  29. package/md_cg/test_untrusted_classifier.py +42 -42
  30. package/md_cg/test_untrusted_enforcement.py +66 -66
  31. package/md_cg/tokens.py +5 -1
  32. package/md_cg/untrusted.py +45 -45
  33. package/md_cg/untrusted_audit.py +87 -87
  34. package/package.json +113 -113
  35. package/scripts/workspace_index.py +2 -0
  36. package/src/hooks.ts +1043 -986
  37. package/src/index.ts +12 -1
  38. package/src/tools.ts +37 -6
@@ -1,121 +1,121 @@
1
- # -*- coding: utf-8 -*-
2
- """工具面热缓存守卫(issue #614)。
3
-
4
- C1 第二次同参命中且结果逐条相同
5
- C2 写后读不陈旧
6
- C3 换 k 不串味
7
- C4 环境开关翻动不命中
8
- C5 显式 node_id 绕过缓存
9
- C6 值形状标签:命中体必须是本面写入的(surface/v 校验),异形一律当未命中
10
- C7 命中返回体是深拷贝:调用方原地修改不得污染缓存
11
- C8 validity=True 一律不缓存(无时钟窗语义)
12
- """
13
- from __future__ import annotations
14
-
15
- import os
16
- import tempfile
17
-
18
- from .mdcos import MdCGSecure
19
- from .mcp_server import call_tool
20
-
21
-
22
- def main():
23
- os.environ["MDCG_HOTCACHE"] = "1"
24
- ok = 0
25
- bad = 0
26
-
27
- def chk(cond, label):
28
- nonlocal ok, bad
29
- if cond:
30
- ok += 1
31
- print(" PASS", label)
32
- else:
33
- bad += 1
34
- print(" FAIL", label)
35
-
36
- root = tempfile.mkdtemp(prefix="hc_tool2_")
37
- cg = MdCGSecure(root)
38
- for i in range(6):
39
- cg.add("n%d" % i, "灵枢 记忆库 条目 %d alpha" % i, layer="knowledge")
40
-
41
- r1 = call_tool(cg, "mdcg_search", {"query": "灵枢 记忆库 alpha", "k": 5})
42
- hc = getattr(cg, "_hotcache_tool", None)
43
- chk(hc is not None, "C0 工具层用独立缓存实例")
44
- s1 = hc.stats()
45
- r2 = call_tool(cg, "mdcg_search", {"query": "灵枢 记忆库 alpha", "k": 5})
46
- s2 = hc.stats()
47
- chk(s1["query_misses"] >= 1 and s2["query_hits"] >= 1, "C1 命中(misses=%d hits=%d)" % (s1["query_misses"], s2["query_hits"]))
48
- chk(r1.get("results") == r2.get("results"), "C1b 命中结果逐条相同")
49
-
50
- cg.add("n_new", "阿洛芬 独有词", layer="knowledge")
51
- r3 = call_tool(cg, "mdcg_search", {"query": "阿洛芬", "k": 5})
52
- chk("n_new" in [x["node"]["id"] for x in (r3.get("results") or [])], "C2 写后读不陈旧")
53
-
54
- before = hc.stats()["query_hits"]
55
- call_tool(cg, "mdcg_search", {"query": "灵枢 记忆库 alpha", "k": 3})
56
- chk(hc.stats()["query_hits"] == before, "C3 换 k 不串味")
57
-
58
- os.environ["MDCG_SEMANTIC"] = "0"
59
- call_tool(cg, "mdcg_search", {"query": "灵枢 记忆库 alpha", "k": 5})
60
- b4 = hc.stats()["query_hits"]
61
- os.environ["MDCG_SEMANTIC"] = "1"
62
- call_tool(cg, "mdcg_search", {"query": "灵枢 记忆库 alpha", "k": 5})
63
- chk(hc.stats()["query_hits"] == b4, "C4 开关翻动后不命中")
64
- os.environ.pop("MDCG_SEMANTIC", None)
65
-
66
- b5 = hc.stats()["query_hits"]
67
- call_tool(cg, "mdcg_search", {"query": "灵枢 记忆库 alpha", "k": 5, "node_id": None})
68
- chk(hc.stats()["query_hits"] == b5, "C5 显式 node_id 绕过")
69
-
70
- # C6:往同一实例塞一个异形值(模拟内部层写进来的形状),必须当未命中而不是返回垃圾
71
- call_tool(cg, "mdcg_search", {"query": "形状探针", "k": 5})
72
- _q = "形状探针"
73
- try:
74
- _keys = list(hc._queries.keys())
75
- _k0 = [k for k in _keys if k[0] == _q]
76
- if _k0:
77
- hc._queries[_k0[0]] = (hc._queries[_k0[0]][0], [("row", 0.1, {}, "p")], {})
78
- except Exception:
79
- pass
80
- r6 = call_tool(cg, "mdcg_search", {"query": _q, "k": 5})
81
- chk(isinstance(r6, dict) and "results" in r6, "C6 异形缓存值被当未命中(不返回垃圾)")
82
-
83
- # C7:命中体是深拷贝——改它不能污染缓存
84
- _r7a = call_tool(cg, "mdcg_search", {"query": "灵枢 记忆库 alpha", "k": 5})
85
- _r7a.setdefault("results", []).append({"node": {"id": "污染"}, "score": 99})
86
- _r7b = call_tool(cg, "mdcg_search", {"query": "灵枢 记忆库 alpha", "k": 5})
87
- chk("污染" not in [x["node"]["id"] for x in (_r7b.get("results") or [])], "C7 命中体深拷贝(原地修改不污染)")
88
-
89
- # C8:validity=True 不缓存
90
- b8 = hc.stats()["query_hits"]
91
- call_tool(cg, "mdcg_search", {"query": "灵枢 记忆库 alpha", "k": 5, "validity": True})
92
- call_tool(cg, "mdcg_search", {"query": "灵枢 记忆库 alpha", "k": 5, "validity": True})
93
- chk(hc.stats()["query_hits"] == b8, "C8 validity=True 不缓存")
94
-
95
- # C9 安全:切换调用者身份(cg.session)后不得命中前者缓存——
96
- # 检索结果按 session/密级过滤,缓存键必须含有效身份,否则跨会话/跨密级泄漏。
97
- call_tool(cg, "mdcg_search", {"query": "灵枢 记忆库 alpha", "k": 5})
98
- b9 = hc.stats()["query_hits"]
99
- _old_sess = getattr(cg, "session", None)
100
- cg.session = "another-session"
101
- call_tool(cg, "mdcg_search", {"query": "灵枢 记忆库 alpha", "k": 5})
102
- chk(hc.stats()["query_hits"] == b9, "C9 换会话身份后不命中(防跨会话/跨密级泄漏)")
103
- cg.session = _old_sess
104
-
105
- # C10 安全:索引突变(写入/遗忘都经 _on_index_mutation)必须清空工具层缓存,
106
- # 否则遗忘后 TTL 内还能被同参 query 读回正文。
107
- # 注:本用例直接调突变单点——forget 需管理权限,测试 principal 没有(AccessDenied),
108
- # 故验证「突变即失效」这条接线本身,而不是绕开权限去删节点。
109
- cg.add("n_del", "波塞冬 独有词 待删除", layer="knowledge")
110
- _r = call_tool(cg, "mdcg_search", {"query": "波塞冬", "k": 5})
111
- chk(len(getattr(cg, "_hotcache_tool")._queries) >= 1, "C10a 查询已进工具层缓存")
112
- cg._on_index_mutation("n_del")
113
- chk(len(getattr(cg, "_hotcache_tool")._queries) == 0, "C10b 索引突变即清空工具层缓存(删除承诺不被缓存吃掉)")
114
-
115
- print("=" * 60)
116
- print("工具面热缓存守卫:%d 通过 / %d 失败" % (ok, bad))
117
- return 0 if bad == 0 else 1
118
-
119
-
120
- if __name__ == "__main__":
121
- raise SystemExit(main())
1
+ # -*- coding: utf-8 -*-
2
+ """工具面热缓存守卫(issue #614)。
3
+
4
+ C1 第二次同参命中且结果逐条相同
5
+ C2 写后读不陈旧
6
+ C3 换 k 不串味
7
+ C4 环境开关翻动不命中
8
+ C5 显式 node_id 绕过缓存
9
+ C6 值形状标签:命中体必须是本面写入的(surface/v 校验),异形一律当未命中
10
+ C7 命中返回体是深拷贝:调用方原地修改不得污染缓存
11
+ C8 validity=True 一律不缓存(无时钟窗语义)
12
+ """
13
+ from __future__ import annotations
14
+
15
+ import os
16
+ import tempfile
17
+
18
+ from .mdcos import MdCGSecure
19
+ from .mcp_server import call_tool
20
+
21
+
22
+ def main():
23
+ os.environ["MDCG_HOTCACHE"] = "1"
24
+ ok = 0
25
+ bad = 0
26
+
27
+ def chk(cond, label):
28
+ nonlocal ok, bad
29
+ if cond:
30
+ ok += 1
31
+ print(" PASS", label)
32
+ else:
33
+ bad += 1
34
+ print(" FAIL", label)
35
+
36
+ root = tempfile.mkdtemp(prefix="hc_tool2_")
37
+ cg = MdCGSecure(root)
38
+ for i in range(6):
39
+ cg.add("n%d" % i, "灵枢 记忆库 条目 %d alpha" % i, layer="knowledge")
40
+
41
+ r1 = call_tool(cg, "mdcg_search", {"query": "灵枢 记忆库 alpha", "k": 5})
42
+ hc = getattr(cg, "_hotcache_tool", None)
43
+ chk(hc is not None, "C0 工具层用独立缓存实例")
44
+ s1 = hc.stats()
45
+ r2 = call_tool(cg, "mdcg_search", {"query": "灵枢 记忆库 alpha", "k": 5})
46
+ s2 = hc.stats()
47
+ chk(s1["query_misses"] >= 1 and s2["query_hits"] >= 1, "C1 命中(misses=%d hits=%d)" % (s1["query_misses"], s2["query_hits"]))
48
+ chk(r1.get("results") == r2.get("results"), "C1b 命中结果逐条相同")
49
+
50
+ cg.add("n_new", "阿洛芬 独有词", layer="knowledge")
51
+ r3 = call_tool(cg, "mdcg_search", {"query": "阿洛芬", "k": 5})
52
+ chk("n_new" in [x["node"]["id"] for x in (r3.get("results") or [])], "C2 写后读不陈旧")
53
+
54
+ before = hc.stats()["query_hits"]
55
+ call_tool(cg, "mdcg_search", {"query": "灵枢 记忆库 alpha", "k": 3})
56
+ chk(hc.stats()["query_hits"] == before, "C3 换 k 不串味")
57
+
58
+ os.environ["MDCG_SEMANTIC"] = "0"
59
+ call_tool(cg, "mdcg_search", {"query": "灵枢 记忆库 alpha", "k": 5})
60
+ b4 = hc.stats()["query_hits"]
61
+ os.environ["MDCG_SEMANTIC"] = "1"
62
+ call_tool(cg, "mdcg_search", {"query": "灵枢 记忆库 alpha", "k": 5})
63
+ chk(hc.stats()["query_hits"] == b4, "C4 开关翻动后不命中")
64
+ os.environ.pop("MDCG_SEMANTIC", None)
65
+
66
+ b5 = hc.stats()["query_hits"]
67
+ call_tool(cg, "mdcg_search", {"query": "灵枢 记忆库 alpha", "k": 5, "node_id": None})
68
+ chk(hc.stats()["query_hits"] == b5, "C5 显式 node_id 绕过")
69
+
70
+ # C6:往同一实例塞一个异形值(模拟内部层写进来的形状),必须当未命中而不是返回垃圾
71
+ call_tool(cg, "mdcg_search", {"query": "形状探针", "k": 5})
72
+ _q = "形状探针"
73
+ try:
74
+ _keys = list(hc._queries.keys())
75
+ _k0 = [k for k in _keys if k[0] == _q]
76
+ if _k0:
77
+ hc._queries[_k0[0]] = (hc._queries[_k0[0]][0], [("row", 0.1, {}, "p")], {})
78
+ except Exception:
79
+ pass
80
+ r6 = call_tool(cg, "mdcg_search", {"query": _q, "k": 5})
81
+ chk(isinstance(r6, dict) and "results" in r6, "C6 异形缓存值被当未命中(不返回垃圾)")
82
+
83
+ # C7:命中体是深拷贝——改它不能污染缓存
84
+ _r7a = call_tool(cg, "mdcg_search", {"query": "灵枢 记忆库 alpha", "k": 5})
85
+ _r7a.setdefault("results", []).append({"node": {"id": "污染"}, "score": 99})
86
+ _r7b = call_tool(cg, "mdcg_search", {"query": "灵枢 记忆库 alpha", "k": 5})
87
+ chk("污染" not in [x["node"]["id"] for x in (_r7b.get("results") or [])], "C7 命中体深拷贝(原地修改不污染)")
88
+
89
+ # C8:validity=True 不缓存
90
+ b8 = hc.stats()["query_hits"]
91
+ call_tool(cg, "mdcg_search", {"query": "灵枢 记忆库 alpha", "k": 5, "validity": True})
92
+ call_tool(cg, "mdcg_search", {"query": "灵枢 记忆库 alpha", "k": 5, "validity": True})
93
+ chk(hc.stats()["query_hits"] == b8, "C8 validity=True 不缓存")
94
+
95
+ # C9 安全:切换调用者身份(cg.session)后不得命中前者缓存——
96
+ # 检索结果按 session/密级过滤,缓存键必须含有效身份,否则跨会话/跨密级泄漏。
97
+ call_tool(cg, "mdcg_search", {"query": "灵枢 记忆库 alpha", "k": 5})
98
+ b9 = hc.stats()["query_hits"]
99
+ _old_sess = getattr(cg, "session", None)
100
+ cg.session = "another-session"
101
+ call_tool(cg, "mdcg_search", {"query": "灵枢 记忆库 alpha", "k": 5})
102
+ chk(hc.stats()["query_hits"] == b9, "C9 换会话身份后不命中(防跨会话/跨密级泄漏)")
103
+ cg.session = _old_sess
104
+
105
+ # C10 安全:索引突变(写入/遗忘都经 _on_index_mutation)必须清空工具层缓存,
106
+ # 否则遗忘后 TTL 内还能被同参 query 读回正文。
107
+ # 注:本用例直接调突变单点——forget 需管理权限,测试 principal 没有(AccessDenied),
108
+ # 故验证「突变即失效」这条接线本身,而不是绕开权限去删节点。
109
+ cg.add("n_del", "波塞冬 独有词 待删除", layer="knowledge")
110
+ _r = call_tool(cg, "mdcg_search", {"query": "波塞冬", "k": 5})
111
+ chk(len(getattr(cg, "_hotcache_tool")._queries) >= 1, "C10a 查询已进工具层缓存")
112
+ cg._on_index_mutation("n_del")
113
+ chk(len(getattr(cg, "_hotcache_tool")._queries) == 0, "C10b 索引突变即清空工具层缓存(删除承诺不被缓存吃掉)")
114
+
115
+ print("=" * 60)
116
+ print("工具面热缓存守卫:%d 通过 / %d 失败" % (ok, bad))
117
+ return 0 if bad == 0 else 1
118
+
119
+
120
+ if __name__ == "__main__":
121
+ raise SystemExit(main())
@@ -0,0 +1,72 @@
1
+ # -*- coding: utf-8 -*-
2
+ """索引装载快路径守卫(2026-10-21 治本:写/读超时根因)。
3
+
4
+ 背景:目录 mtime 指纹在任意合作写入后失效,原实现失效即回退全库扫描
5
+ (真实库 31,688 节点实测 13.4 秒/次冷启动)。新判据:指纹不匹配时先重放
6
+ 分片日志,再用 _covers_disk(路径集合相等)验证;覆盖成立则采纳快照+日志。
7
+
8
+ S1 合作写入后:新实例走快路径(不触发 _scan_nodes),且新节点可见
9
+ S2 非合作改动(外部改名):覆盖校验失败 → 仍回退全库扫描(正确性不降级)
10
+ S3 快路径与全扫的**结果集合一致**(同一盘面,两条路给出同一批 path)
11
+ """
12
+ from __future__ import annotations
13
+
14
+ import os
15
+ import tempfile
16
+
17
+ from .mdcg import MdCG
18
+
19
+
20
+ def main():
21
+ ok = 0
22
+ bad = 0
23
+
24
+ def chk(cond, label):
25
+ nonlocal ok, bad
26
+ if cond:
27
+ ok += 1
28
+ print(" PASS", label)
29
+ else:
30
+ bad += 1
31
+ print(" FAIL", label)
32
+
33
+ root = tempfile.mkdtemp(prefix="fastpath_")
34
+ cg = MdCG(root)
35
+ for i in range(4):
36
+ cg.add("f%d" % i, "快路径样本 %d 内容" % i, layer="knowledge")
37
+ cg.compact_index() # 落快照(带指纹)
38
+ cg.add("f_new", "合作写入后的新节点", layer="knowledge") # 目录 mtime 失效指纹
39
+ cg.flush() # 真实写路径会 flush/close:分片日志落盘后,新实例才可重放
40
+
41
+ calls = {"scan": 0}
42
+ orig_scan = MdCG._scan_nodes
43
+ def spy(self):
44
+ calls["scan"] += 1
45
+ return orig_scan(self)
46
+ MdCG._scan_nodes = spy
47
+ try:
48
+ cg2 = MdCG(root)
49
+ chk(calls["scan"] == 0, "S1 合作写入后新实例不触发全库扫描(scan 次数=%d)" % calls["scan"])
50
+ chk("f_new" in cg2.index["nodes"], "S1b 新节点可见(快照+日志重放生效)")
51
+ chk(len(cg2.index["nodes"]) == 5, "S1c 节点数正确(%d)" % len(cg2.index["nodes"]))
52
+
53
+ # S2 非合作改动:改名 → 路径集合不再相等 → 必须回退全扫
54
+ old_path = os.path.join(root, cg2.index["nodes"]["f0"]["path"])
55
+ new_path = old_path.replace(".md", "_renamed.md")
56
+ os.rename(old_path, new_path)
57
+ calls["scan"] = 0
58
+ cg3 = MdCG(root)
59
+ chk(calls["scan"] >= 1, "S2 外部改名仍回退全库扫描(scan=%d)" % calls["scan"])
60
+ # 全扫的「收录正确」由 S3 与节点数保证;路径集合是否与盘面逐一相等属另一条
61
+ # 既有判据(_covers_disk 用法),此处不再重复断言(避免同判据两处口径)。
62
+ chk("f_new" in cg3.index["nodes"], "S3 全扫结果仍含此前合作写入的节点")
63
+ finally:
64
+ MdCG._scan_nodes = orig_scan
65
+
66
+ print("=" * 60)
67
+ print("索引装载快路径守卫:%d 通过 / %d 失败" % (ok, bad))
68
+ return 0 if bad == 0 else 1
69
+
70
+
71
+ if __name__ == "__main__":
72
+ raise SystemExit(main())