@furongjun1999/dsh-memory 0.4.8 → 0.4.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (232) hide show
  1. package/README.md +54 -26
  2. package/codebuddy/CODEBUDDY.md +196 -195
  3. package/codebuddy/README.md +13 -1
  4. package/codebuddy/mcp.json +9 -0
  5. package/docs/GBrain/345/217/257/345/200/237/351/211/264/347/202/271_/347/201/265/346/236/242/350/220/275/347/202/271/344/272/244/346/216/245_20260919.md +169 -0
  6. package/docs/README.md +1 -1
  7. package/docs/discipline/harnesses.yaml +18 -7
  8. package/docs/discipline/templates/full.md.tmpl +4 -3
  9. package/docs/experiments/linkref_backfill/candidates_20260917.json +726 -0
  10. package/docs/experiments/linkref_backfill/candidates_internal_20260917.json +602 -0
  11. package/docs/experiments/linkref_backfill/candidates_internal_v2.json +603 -0
  12. package/docs/experiments/linkref_backfill/candidates_secret_20260917.json +884 -0
  13. package/docs/experiments/linkref_backfill/candidates_secret_v2.json +789 -0
  14. package/docs/hive//345/244/232/347/253/257harness/351/200/232/344/277/241/345/245/221/347/272/246_v0.1.md +42 -0
  15. package/docs/hive//346/243/200/347/264/242/346/224/266/346/225/233/345/256/236/346/265/213/344/270/216S1b/350/256/276/350/256/241_v0.1.md +43 -0
  16. package/docs/hive//346/243/200/347/264/242/350/267/257/345/276/204/344/270/216/350/256/244/347/237/245/347/273/223/346/236/204/345/245/221/347/272/246_v0.1.md +102 -0
  17. package/docs/hive//347/234/237/345/256/236/345/272/223/347/253/257/345/210/260/347/253/257/345/256/236/346/265/213_S1b/344/270/216/345/217/254/345/233/236/346/235/203/350/241/241_v0.1.md +57 -0
  18. package/docs/hive//347/234/237/345/256/236/345/272/223/347/253/257/345/210/260/347/253/257/345/256/236/346/265/213_S7/345/200/222/346/216/222/345/200/231/351/200/211/345/261/202_v0.1.md +126 -0
  19. package/docs/hive//350/234/202/345/267/242/345/217/214/345/256/236/344/276/213/344/272/222/351/252/214_/350/256/276/350/256/241/345/256/232/347/250/277.md +503 -0
  20. package/docs/mdcg/D_meta_/345/267/245/347/250/213/345/214/226/346/226/271/346/241/210_v0.2.md +216 -0
  21. package/docs/mdcg/README/350/257/246/347/273/206/347/211/210_v0.4.5.md +631 -625
  22. package/docs/mdcg//344/273/243/347/240/201/350/257/204/345/256/241/344/270/216/346/235/241/344/273/266/345/214/226/346/263/250/351/207/212_/345/245/221/347/272/246_v0.1.md +82 -0
  23. package/docs/mdcg//345/205/250/345/272/223/344/273/243/347/240/201/350/257/204/345/256/241/344/270/216/346/235/241/344/273/266/345/214/226/346/263/250/351/207/212_/350/256/241/345/210/222_v0.1.md +600 -0
  24. package/docs/mdcg//345/212/237/350/203/275/350/260/203/347/224/250/346/230/240/345/260/204/350/241/250_v0.1.md +47 -45
  25. package/docs/mdcg//345/255/220/344/273/243/347/220/206/351/205/215/347/275/256/346/240/207/345/207/{206_v0.4.md → 206_v0.5.md} +92 -4
  26. package/docs/mdcg//347/201/265/346/236/242/350/256/260/345/277/206/345/212/250/350/257/215/345/215/217/350/256/256_v1.0-draft.md +172 -0
  27. package/docs/mdcg//347/216/257/344/272/214_/347/231/275/347/256/261/345/241/253/345/205/205/346/265/201/346/260/264/347/272/277_v0.1.md +31 -0
  28. package/docs/mdcg//350/267/250/347/253/257/351/252/214/350/257/201/344/270/216/345/220/214/346/255/245/345/215/217/350/256/256_v0.1.md +93 -0
  29. package/docs/theory//345/271/266/345/217/221/345/277/205/347/204/266/346/200/247/347/220/206/350/256/272_v0.2.md +2 -2
  30. package/docs/theory//347/220/206/350/256/272_/346/234/272/345/210/266_/344/273/243/347/240/201_/345/256/236/351/252/214_/347/274/272/345/217/243/347/237/251/351/230/265_v0.1.md +3 -3
  31. package/docs/theory//350/256/244/347/237/245/344/273/243/347/220/206/344/270/216/346/224/266/346/225/233/347/273/223/346/236/204_/346/235/241/344/273/266/350/256/272/351/207/215/346/236/204_v0.1.md +2 -2
  32. package/docs//345/267/245/344/275/234/347/272/252/345/276/213_/350/256/244/347/237/245/345/233/276/346/235/241/347/233/256_v1.1.json +434 -433
  33. package/docs//347/201/265/346/236/242/350/207/252/346/210/221/346/224/271/350/277/233/345/267/245/344/275/234/350/256/241/345/210/222_/345/244/226/351/203/250/347/240/224/347/251/266/347/263/273/345/210/227/345/220/270/346/224/266_v1_20260919.md +286 -0
  34. package/dsh/README.md +33 -0
  35. package/dsh/cordis.yml.example +13 -7
  36. package/dsh/hive-mcp-probe.mjs +94 -0
  37. package/dsh/hive-mcp.example.yml +62 -0
  38. package/dsh/update-lingshu.bat +11 -0
  39. package/dsh/update-lingshu.ps1 +337 -0
  40. package/lib/hooks.d.ts +3 -0
  41. package/lib/hooks.js +17 -23
  42. package/lib/index.d.ts +4 -2
  43. package/lib/index.js +29 -6
  44. package/lib/lib/datapath.d.ts +76 -1
  45. package/lib/lib/datapath.js +199 -13
  46. package/lib/lib/mdcg_client.d.ts +40 -3
  47. package/lib/lib/mdcg_client.js +46 -22
  48. package/lib/lib/mutual.js +4 -4
  49. package/lib/lib/token_store.js +4 -5
  50. package/md_cg/audit.py +17 -2
  51. package/md_cg/autonomy.py +86 -15
  52. package/md_cg/backfill.py +36 -1
  53. package/md_cg/backfill_bigdomain.py +34 -0
  54. package/md_cg/bench6_arms.py +28 -1
  55. package/md_cg/bench6_common.py +10 -1
  56. package/md_cg/bench6_competitors.py +6 -1
  57. package/md_cg/bench_axis_domain.py +9 -1
  58. package/md_cg/bench_blind_comp.py +7 -1
  59. package/md_cg/bench_en_atoms_public.py +9 -0
  60. package/md_cg/bench_governance.py +348 -0
  61. package/md_cg/bench_lme_zh.py +16 -1
  62. package/md_cg/bench_locomo.py +2 -1
  63. package/md_cg/bench_locomo_zh.py +16 -1
  64. package/md_cg/bench_locomo_zh_public.py +4 -1
  65. package/md_cg/bench_longmem.py +2 -1
  66. package/md_cg/bench_membench.py +27 -1
  67. package/md_cg/bench_p0.py +4 -1
  68. package/md_cg/bench_progressive.py +13 -1
  69. package/md_cg/bench_role_views.py +238 -0
  70. package/md_cg/bench_task_ab.py +8 -1
  71. package/md_cg/bench_task_ab_llm.py +13 -1
  72. package/md_cg/bench_unified_en.py +6 -1
  73. package/md_cg/bench_zh_mad.py +20 -1
  74. package/md_cg/blindspot_tickets.py +123 -0
  75. package/md_cg/branches.py +12 -1
  76. package/md_cg/build_postings.py +73 -0
  77. package/md_cg/ccgc.py +67 -2
  78. package/md_cg/census.py +5 -1
  79. package/md_cg/chain.py +24 -3
  80. package/md_cg/codeindex.py +134 -17
  81. package/md_cg/coldverify.py +265 -0
  82. package/md_cg/comment_gate.py +338 -0
  83. package/md_cg/cond_compose.py +190 -0
  84. package/md_cg/cond_facts.py +155 -0
  85. package/md_cg/cond_template.json +107 -0
  86. package/md_cg/condition_anchor.py +143 -0
  87. package/md_cg/conformance.py +69 -4
  88. package/md_cg/consistency.py +24 -1
  89. package/md_cg/consolidate.py +53 -2
  90. package/md_cg/corpus.py +4 -0
  91. package/md_cg/crosscheck.py +42 -2
  92. package/md_cg/crypto.py +35 -1
  93. package/md_cg/d_meta.py +310 -0
  94. package/md_cg/datapath.py +201 -26
  95. package/md_cg/docindex.py +122 -1
  96. package/md_cg/eval_common.py +29 -1
  97. package/md_cg/evidence.py +27 -1
  98. package/md_cg/evolution.py +21 -1
  99. package/md_cg/export.py +11 -1
  100. package/md_cg/forgetting.py +23 -1
  101. package/md_cg/fsutil.py +18 -1
  102. package/md_cg/hotcache.py +214 -0
  103. package/md_cg/hyperedge.py +251 -0
  104. package/md_cg/identity.py +18 -1
  105. package/md_cg/insight.py +17 -1
  106. package/md_cg/lexicon/build_cedict_en_zh.py +9 -0
  107. package/md_cg/lexicon/build_standard_en.py +171 -168
  108. package/md_cg/lexicon/expand_en_zh.py +6 -0
  109. package/md_cg/lifecycle.py +12 -1
  110. package/md_cg/linkref.py +281 -0
  111. package/md_cg/links.py +29 -1
  112. package/md_cg/mcp_server.py +362 -43
  113. package/md_cg/md_whitebox.py +53 -1
  114. package/md_cg/mdcg.py +1003 -27
  115. package/md_cg/mdcos.py +558 -36
  116. package/md_cg/metacognition.py +37 -2
  117. package/md_cg/migrate.py +4 -0
  118. package/md_cg/migrate_aeis.py +221 -213
  119. package/md_cg/migrate_roleplay.py +8 -0
  120. package/md_cg/migrate_wisdom_graph.py +14 -1
  121. package/md_cg/mreview/__main__.py +3 -0
  122. package/md_cg/mreview/bundle.py +8 -0
  123. package/md_cg/mreview/candidates.py +9 -0
  124. package/md_cg/mreview/govern.py +21 -1
  125. package/md_cg/mreview/locate.py +34 -0
  126. package/md_cg/mreview/pipeline.py +29 -1
  127. package/md_cg/mreview/ruleset.py +16 -1
  128. package/md_cg/nodefile.py +233 -3
  129. package/md_cg/pooling.py +23 -1
  130. package/md_cg/postings.py +298 -0
  131. package/md_cg/predict.py +89 -9
  132. package/md_cg/progressive.py +3 -0
  133. package/md_cg/protect.py +14 -1
  134. package/md_cg/protocol.py +372 -0
  135. package/md_cg/provenance.py +262 -0
  136. package/md_cg/reach.py +453 -0
  137. package/md_cg/refindex.py +47 -2
  138. package/md_cg/refine.py +20 -1
  139. package/md_cg/roleviews.py +89 -0
  140. package/md_cg/routing.py +76 -0
  141. package/md_cg/scrub.py +63 -2
  142. package/md_cg/security.py +26 -1
  143. package/md_cg/self_state.py +64 -1
  144. package/md_cg/selfreport.py +151 -0
  145. package/md_cg/semantic/canonical.py +5 -0
  146. package/md_cg/semantic/en_normalizer.py +364 -355
  147. package/md_cg/semantic/zh_en_atoms.py +139 -136
  148. package/md_cg/signer.py +41 -1
  149. package/md_cg/sources.py +583 -547
  150. package/md_cg/statushdr.py +179 -0
  151. package/md_cg/stg.py +59 -18
  152. package/md_cg/subgraph.py +23 -0
  153. package/md_cg/sustain.py +56 -1
  154. package/md_cg/tasks.py +26 -2
  155. package/md_cg/test_autonomy.py +26 -0
  156. package/md_cg/test_bench_governance.py +102 -0
  157. package/md_cg/test_blindspot_tickets.py +166 -0
  158. package/md_cg/test_ccgc.py +10 -0
  159. package/md_cg/test_codeindex.py +338 -0
  160. package/md_cg/test_comment_gate.py +187 -0
  161. package/md_cg/test_cond_compose_anchors.py +76 -0
  162. package/md_cg/test_condition_anchor.py +82 -0
  163. package/md_cg/test_d_meta.py +412 -0
  164. package/md_cg/test_datapath_root.py +188 -0
  165. package/md_cg/test_gain_gate.py +47 -1
  166. package/md_cg/test_hot_cold.py +187 -0
  167. package/md_cg/test_hyperedge.py +245 -0
  168. package/md_cg/test_linkref.py +306 -0
  169. package/md_cg/test_md_access_parity.py +15 -3
  170. package/md_cg/test_mr_m1.py +108 -18
  171. package/md_cg/test_mr_m3.py +8 -1
  172. package/md_cg/test_p26_refindex.py +49 -20
  173. package/md_cg/test_p27_docindex.py +236 -2
  174. package/md_cg/test_p2_mcp.py +1 -1
  175. package/md_cg/test_p31_insight.py +24 -0
  176. package/md_cg/test_p44_md_whitebox.py +14 -1
  177. package/md_cg/test_protocol.py +243 -0
  178. package/md_cg/test_reach.py +378 -0
  179. package/md_cg/test_reach_keys.py +201 -0
  180. package/md_cg/test_reach_meta_exits.py +145 -0
  181. package/md_cg/test_read_clip.py +8 -4
  182. package/md_cg/test_retr_s1.py +340 -0
  183. package/md_cg/test_retr_s1b.py +209 -0
  184. package/md_cg/test_retr_s3.py +194 -0
  185. package/md_cg/test_retr_s4.py +163 -0
  186. package/md_cg/test_retr_s5.py +200 -0
  187. package/md_cg/test_retr_s6.py +157 -0
  188. package/md_cg/test_retr_s7.py +385 -0
  189. package/md_cg/test_retr_s8_time.py +316 -0
  190. package/md_cg/test_retr_s9_edges.py +286 -0
  191. package/md_cg/test_retr_s9_entity_ctx.py +175 -0
  192. package/md_cg/test_review_conformance.py +59 -2
  193. package/md_cg/test_role_views.py +354 -0
  194. package/md_cg/test_subproc_encoding.py +188 -0
  195. package/md_cg/test_trust.py +361 -0
  196. package/md_cg/test_units_poll.py +71 -0
  197. package/md_cg/test_v14_fixes.py +397 -0
  198. package/md_cg/test_validity_filter.py +280 -0
  199. package/md_cg/test_wisdom_md_store.py +7 -3
  200. package/md_cg/test_writepipe.py +5 -1
  201. package/md_cg/theory.py +16 -1
  202. package/md_cg/tokens.py +40 -8
  203. package/md_cg/tool_face.py +13 -2
  204. package/md_cg/trust.py +943 -0
  205. package/md_cg/twophase.py +12 -1
  206. package/md_cg/units.py +132 -10
  207. package/md_cg/vision_evidence.py +24 -1
  208. package/md_cg/weights.py +24 -1
  209. package/md_cg/whitebox.py +32 -1
  210. package/md_cg/whitebox_kb/data/verify_cache.json +21210 -365
  211. package/md_cg/whitebox_kb/data/verify_savings.jsonl +5078 -0
  212. package/md_cg/whitebox_kb/wisdom/audit_log/chain_heat.json +10 -10
  213. package/md_cg/whitebox_kb/wisdom/code_compose.py +113 -6
  214. package/md_cg/whitebox_kb/wisdom/code_solidified.json +1 -1
  215. package/md_cg/whitebox_kb/wisdom/verifier.py +340 -55
  216. package/md_cg/whitebox_kb/wisdom/wisdom-book-cloud.db +0 -0
  217. package/md_cg/writelimit.py +18 -4
  218. package/md_cg/writepipe.py +178 -7
  219. package/package.json +2 -2
  220. package/skills/skills/designer-perspective/scripts/__pycache__/designer.cpython-310.pyc +0 -0
  221. package/skills/skills/designer-perspective/scripts/designer.py +17 -1
  222. package/skills/skills/designer-perspective/tests/selftest.py +3 -1
  223. package/src/hooks.ts +17 -21
  224. package/src/index.ts +33 -6
  225. package/src/lib/datapath.ts +211 -13
  226. package/src/lib/mdcg_client.ts +64 -25
  227. package/src/lib/mutual.ts +411 -411
  228. package/src/lib/token_store.ts +4 -5
  229. package/zcode/AGENTS.md +196 -195
  230. package/zcode/README.md +4 -0
  231. package/md_cg/whitebox_kb/wisdom/wisdom-book-cloud.db-shm +0 -0
  232. package/md_cg/whitebox_kb/wisdom/wisdom-book-cloud.db-wal +0 -0
@@ -0,0 +1,145 @@
1
+ # -*- coding: utf-8 -*-
2
+ """narrow() 各退化出口必须如实暴露首建 / 部分重建成本。
3
+
4
+ 历史取证:r19 复核发现 no_tokens / empty_converge 吞掉 reach_build_docs;
5
+ r32 发现 index_unavailable 连部分读盘成本也丢了、且旧测试用 sem=True 使 set(True)
6
+ 异常被吞掉;r34 发现只计「已入倒排」会漏计「读了很多但都不可读」的情形、且模块级状态不安全。
7
+ """
8
+ import os
9
+ import sys
10
+ import tempfile
11
+
12
+ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
13
+ from md_cg import reach # noqa: E402
14
+
15
+
16
+ class StubIdx:
17
+ hash_complete = True
18
+ sem = [] # 必须是真容器:曾写 sem=True,set(True) 报错被吞掉 → 语义分支未真实验证(r32 复核)
19
+ built_now = True
20
+ built_docs = 7
21
+ built_reads = 9
22
+ hashes = {}
23
+ built_at = 0.0
24
+
25
+ def postings(self, toks):
26
+ return {"x"}
27
+
28
+
29
+ class ReadStubCG:
30
+ """按 plan 依次返回 cg._read 结果,plan 耗尽后抛错(模拟重建中途失败)。"""
31
+
32
+ def __init__(self, root, nodes, plan):
33
+ self.root = root
34
+ self.index = {"nodes": nodes}
35
+ self._plan = list(plan)
36
+ self.calls = 0
37
+
38
+ def _read(self, entry):
39
+ self.calls += 1
40
+ if not self._plan:
41
+ raise RuntimeError("simulated rebuild failure")
42
+ return self._plan.pop(0)
43
+
44
+ def _open_content(self, nid, fm, c):
45
+ return c
46
+
47
+
48
+ def _tmp_root():
49
+ return tempfile.mkdtemp(prefix="reach_meta_exits_")
50
+
51
+
52
+ def _use(path):
53
+ os.environ["MDCG_REACH_INDEX"] = os.path.join(path, "_reach_index.json")
54
+
55
+
56
+ def main():
57
+ os.environ["MDCG_REACH"] = "1"
58
+ import md_cg.mdcg as mdcg_mod
59
+ orig_with, orig_sem = reach._index_with_meta, mdcg_mod.semantic_on
60
+ mdcg_mod.semantic_on = lambda: True
61
+ reach._index_with_meta = lambda cg, force=False: (StubIdx(), {})
62
+ try:
63
+ # ① 无 token → no_tokens,必须带首建成本
64
+ _, m1 = reach.narrow(object(), [], [], set(), q="")
65
+ assert m1.get("reach") == "no_tokens", m1
66
+ assert m1.get("reach_build_docs") == 7, "no_tokens 出口漏首建成本:%r" % (m1,)
67
+ # ② 语义空收敛 → empty_converge,同样必须带首建成本
68
+ _, m2 = reach.narrow(object(), [], ["甲乙"], {"甲", "乙"}, q="甲乙")
69
+ assert m2.get("reach") == "empty_converge", m2
70
+ assert m2.get("reach_build_docs") == 7, "empty_converge 出口漏首建成本:%r" % (m2,)
71
+
72
+ # ③ 未重建时不得虚构成本
73
+ class Cold(StubIdx):
74
+ built_now = False
75
+
76
+ reach._index_with_meta = lambda cg, force=False: (Cold(), {})
77
+ _, m3 = reach.narrow(object(), [], [], set(), q="")
78
+ assert "reach_build_docs" not in m3 and "reach_build_reads" not in m3, m3
79
+
80
+ # ④ 语义分支确实生效:sem 为真容器时,无词面命中的语义节点也必须入池
81
+ class SemIdx(StubIdx):
82
+ sem = ["p/sem"]
83
+
84
+ reach._index_with_meta = lambda cg, force=False: (SemIdx(), {})
85
+ got4, m4 = reach.narrow(object(), [{"path": "p/sem"}], ["甲乙"], {"甲", "乙"}, q="甲乙")
86
+ assert m4.get("reach") == "converged", m4
87
+ assert [e["path"] for e in got4] == ["p/sem"], "sem 分支未真实生效:%r" % (got4,)
88
+
89
+ # ⑤ fresh_overflow:未索引节点超上限 → 整段回退,且同时暴露 reach_fresh_nodes 与首建成本
90
+ reach._index_with_meta = lambda cg, force=False: (StubIdx(), {})
91
+ many = [{"path": "x"}] + [{"path": "p/f%04d" % i} for i in range(reach._FRESH_MAX + 1)]
92
+ got5, m5 = reach.narrow(object(), many, ["甲乙"], {"甲", "乙"}, q="甲乙")
93
+ assert got5 is None and m5.get("reach") == "fresh_overflow", m5
94
+ assert m5.get("reach_fresh_nodes") == reach._FRESH_MAX + 1, m5
95
+ assert m5.get("reach_build_docs") == 7, "fresh_overflow 出口漏首建成本:%r" % (m5,)
96
+
97
+ # ⑥ r34 反例:读了很多节点但都不可读(None)后失败 → 已读盘量必须如实计数
98
+ reach._index_with_meta = orig_with
99
+ r1 = _tmp_root()
100
+ _use(r1)
101
+ cg_a = ReadStubCG(r1, {"a": {"path": "pa"}, "b": {"path": "pb"}, "c": {"path": "pc"}},
102
+ [(None, None), (None, None)])
103
+ idx_a, fail_a = reach._index_with_meta(cg_a)
104
+ assert idx_a is None, idx_a
105
+ assert fail_a.get("reach_build_reads") == 2, "不可读读盘未计数:%r" % (fail_a,)
106
+ assert fail_a.get("reach_build_docs") == 0 and fail_a.get("reach_build_partial") is True, fail_a
107
+ # 端到端:narrow 的 index_unavailable 出口也必须带这些字段
108
+ # (每次调用各自独立:本例用**新** stub 与新根,否则上一次调用已把 plan 耗尽)
109
+ r4 = _tmp_root()
110
+ _use(r4)
111
+ cg_a2 = ReadStubCG(r4, {"a": {"path": "pa"}, "b": {"path": "pb"}, "c": {"path": "pc"}},
112
+ [(None, None), (None, None)])
113
+ got6, m6 = reach.narrow(cg_a2, [], ["甲乙"], {"甲", "乙"}, q="甲乙")
114
+ assert got6 is None and m6.get("reach") == "index_unavailable", m6
115
+ assert m6.get("reach_build_reads") == 2 and m6.get("reach_build_partial") is True, m6
116
+
117
+ # ⑦ 部分成功:第一个节点已入倒排后失败 → docs 与 reads 都要如实
118
+ r2 = _tmp_root()
119
+ _use(r2)
120
+ cg_b = ReadStubCG(r2, {"a": {"path": "pa"}, "b": {"path": "pb"}}, [({"id": "a"}, "hello")])
121
+ idx_b, fail_b = reach._index_with_meta(cg_b)
122
+ assert idx_b is None, idx_b
123
+ assert fail_b.get("reach_build_docs") == 1 and fail_b.get("reach_build_reads") == 1, fail_b
124
+ assert fail_b.get("reach_build_partial") is True, fail_b
125
+
126
+ # ⑧ 成功路径:无 fail 元数据;首建成本同时暴露读盘数与入索引数(各调用各自独立容器)
127
+ r3 = _tmp_root()
128
+ _use(r3)
129
+ cg_c = ReadStubCG(r3, {"a": {"path": "pa", "content_hash": "h"}}, [({"id": "a"}, "hello")])
130
+ idx_c, fail_c = reach._index_with_meta(cg_c)
131
+ assert idx_c is not None and fail_c == {}, (idx_c, fail_c)
132
+ bm = reach._build_meta(idx_c)
133
+ assert bm.get("reach_build_docs") == 1 and bm.get("reach_build_reads") == 1, bm
134
+ # 重复调用不得把上次成本带下来
135
+ idx_d, fail_d = reach._index_with_meta(cg_c)
136
+ assert fail_d == {}, fail_d
137
+ finally:
138
+ reach._index_with_meta, mdcg_mod.semantic_on = orig_with, orig_sem
139
+ os.environ.pop("MDCG_REACH_INDEX", None)
140
+ print("ALL PASS test_reach_meta_exits")
141
+ return 0
142
+
143
+
144
+ if __name__ == "__main__":
145
+ sys.exit(main())
@@ -5,7 +5,10 @@
5
5
  ① 单次 read 返回受「2000 行 / 50KB」双阈值约束,且**不静默丢内容**——
6
6
  截断必带 total_lines / total_bytes / next_offset / note;
7
7
  ② 续读按 offset 连续取回可拼回原文(不丢不重,可裁决的等价性判据);
8
- ③ 小内容行为逐位兼容(不新增键、content 原样)——防回归。
8
+ ③ 小内容行为兼容——旧四键(id/path/frontmatter/content)不丢、content 原样;
9
+ 验证态 `verification_state` 为**有意的 additive 透出**(不参与排序),
10
+ 故 B1 断言「旧键 ⊆ 新键 ∧ 验证态在场」而非「键集逐一相等」。
11
+ 真源:protocol.py 的 read.node.required 已把 `verification_state` 列为必需键。
9
12
 
10
13
  运行:python -m md_cg.test_read_clip
11
14
  """
@@ -91,9 +94,10 @@ def main():
91
94
  node = {"id": "n1", "path": "/p/n1.md", "frontmatter": {"layer": "knowledge"},
92
95
  "content": "短内容"}
93
96
  v = _node_view(node)
94
- check("B1 小节点键集不变(零回归)",
95
- set(v) == {"id", "path", "frontmatter", "content"}
96
- and v["content"] == "短内容", str(sorted(v)))
97
+ check("B1 旧四键不丢(零回归)+ 验证态 additive 透出",
98
+ {"id", "path", "frontmatter", "content"} <= set(v)
99
+ and v["content"] == "短内容"
100
+ and v["verification_state"] == "unverified", str(sorted(v)))
97
101
  check("B2 空节点维持 None(旧契约)", _node_view(None) is None)
98
102
 
99
103
  big = dict(node, content=many)
@@ -0,0 +1,340 @@
1
+ # -*- coding: utf-8 -*-
2
+ """S1/S2 检索前门控探针(脚本式,exit code 定成败)。
3
+
4
+ 用法:python -X utf8 -m md_cg.test_retr_s1
5
+ 覆盖(含独立复核指出的缺口):
6
+ - routing.domain_terms / classify_text(取词、去重、limit、空值、无信号)
7
+ - 写入侧:默认关**不写** big_domain(口径不变);开总开关才写;索引快照同口径
8
+ - 默认关:meta 不含 gates 键(逐字节等价)、scanned=全量
9
+ - S1:按域收敛;域内不足→回退全量;query 无域信号→不收敛
10
+ - S2:observation_position 不匹配→丢;信息不足→放行;time_window 不相交→丢(平铺字段);全滤→回退
11
+ - backfill_big_domain:dry-run 不改盘、真跑落盘、二次跑幂等
12
+ """
13
+ import io
14
+ import json
15
+ import os
16
+ import sys
17
+ import tempfile
18
+
19
+ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
20
+
21
+ from md_cg import nodefile, routing # noqa: E402
22
+ from md_cg.fsutil import ShardedLog # noqa: E402
23
+ from md_cg.mdcg import MdCG # noqa: E402
24
+
25
+ passed = 0
26
+ failed = 0
27
+ GONG = "工程 结构 应力 梁 截面 桥梁 材料 机械"
28
+ YI = "医学 疾病 诊断 药物 处方 症状 治疗"
29
+ FAR_TW = [10 ** 13, 10 ** 13 + 1]
30
+
31
+
32
+ def check(name, cond, detail=""):
33
+ global passed, failed
34
+ if cond:
35
+ passed += 1
36
+ print(" [PASS] " + name)
37
+ else:
38
+ failed += 1
39
+ print(" [FAIL] " + name + " " + detail)
40
+
41
+
42
+ def _setenv(**kv):
43
+ old = {}
44
+ for k, v in kv.items():
45
+ old[k] = os.environ.get(k)
46
+ if v is None:
47
+ os.environ.pop(k, None)
48
+ else:
49
+ os.environ[k] = v
50
+ return old
51
+
52
+
53
+ def _restore(old):
54
+ for k, v in old.items():
55
+ if v is None:
56
+ os.environ.pop(k, None)
57
+ else:
58
+ os.environ[k] = v
59
+
60
+
61
+ def _write_raw(root, nid, layer, content, pos=None, tw=None, bd=None):
62
+ fm = {"id": nid, "layer": layer, "modality": "text", "importance": 0.5,
63
+ "confidence": 0.6, "condition_space": {"time_window": tw or [0, 10 ** 12]},
64
+ "tags": [], "created_at": 0, "access_count": 0, "last_access": 0,
65
+ "edges": []}
66
+ if pos:
67
+ fm["condition_space"]["observation_position"] = pos
68
+ if bd:
69
+ fm["big_domain"] = bd
70
+ p = os.path.join(root, layer, nid + ".md")
71
+ os.makedirs(os.path.dirname(p), exist_ok=True)
72
+ io.open(p, "w", encoding="utf-8").write(nodefile.dumps(fm, content))
73
+ return p
74
+
75
+
76
+ def _ids(results):
77
+ return sorted(r[0]["id"] for r in results)
78
+
79
+
80
+ def main():
81
+ # ---- 1) routing 侧 ----
82
+ check("domain_terms 空值 -> []", routing.domain_terms("") == [])
83
+ ts = routing.domain_terms("工程 应力 桥梁 " * 3)
84
+ check("domain_terms 去重保序", ts == ["工程", "应力", "桥梁"], str(ts[:4]))
85
+ check("domain_terms limit 生效",
86
+ len(routing.domain_terms("工程 应力 桥梁 材料", limit=2)) == 2)
87
+ check("classify_text 有信号 -> 工程",
88
+ routing.classify_text(GONG) == "工程", str(routing.classify_text(GONG)))
89
+ check("classify_text 无信号 -> None", routing.classify_text("xyzzy zzz") is None)
90
+
91
+ # ---- 2) 写入侧:默认关不写,开总开关才写 ----
92
+ root = tempfile.mkdtemp(prefix="retr_s1_")
93
+ old = _setenv(MDCG_RETRIEVAL_PIPELINE=None, MDCG_GATE_S1_DOMAIN=None,
94
+ MDCG_GATE_S2_COND=None)
95
+ cg = MdCG(root)
96
+ cg.add("n_off", GONG, "knowledge", importance=0.9)
97
+ cg.add("n_yi", YI, "knowledge", importance=0.5)
98
+ cg.flush()
99
+ check("默认关:frontmatter 不写 big_domain",
100
+ "big_domain" not in (cg.get("n_off")["frontmatter"] or {}))
101
+ check("默认关:索引快照不含该键",
102
+ "big_domain" not in (cg.index["nodes"]["n_off"] or {}))
103
+ check("默认关:索引快照不含 observation_position 键",
104
+ "observation_position" not in (cg.index["nodes"]["n_off"] or {}))
105
+
106
+ # ---- 3) 默认关:meta 逐字节等价 ----
107
+ _r0, meta = cg.search("工程 应力", k=10, judge=False, record=False)
108
+ check("默认关:meta 不含 gates 键", "gates" not in meta, str(sorted(meta.keys())))
109
+ check("默认关:scanned=全量", meta.get("scanned") == 2, str(meta.get("scanned")))
110
+ ids_off = _ids(_r0)
111
+
112
+ # 开总开关后写入 → 落域标签(索引同口径)
113
+ _setenv(MDCG_RETRIEVAL_PIPELINE="1", MDCG_GATE_S1_DOMAIN="1",
114
+ MDCG_GATE_S2_COND="0")
115
+ cg.add("n_gong", GONG + " 坝体受力", "knowledge", importance=0.8)
116
+ cg.add("n_none", "xyzzy zzz", "knowledge")
117
+ cg.flush()
118
+ check("开关开:写入落 big_domain",
119
+ cg.get("n_gong")["frontmatter"].get("big_domain") == "工程")
120
+ check("开关开:索引快照同口径",
121
+ cg.index["nodes"]["n_gong"].get("big_domain") == "工程")
122
+ check("开关开:无信号仍不写该字段",
123
+ "big_domain" not in (cg.get("n_none")["frontmatter"] or {}))
124
+
125
+ # ---- 4) S1:收敛 / 回退 / 无域信号 ----
126
+ # 收敛要以「已标域」为前提(未标域节点按兜底池保留)→ 先回填 cg 的存量节点
127
+ st_bf = cg.backfill_big_domain()
128
+ check("回填后:存量节点获得域标签",
129
+ cg.index["nodes"]["n_off"].get("big_domain") == "工程"
130
+ and cg.index["nodes"]["n_yi"].get("big_domain") == "医学", str(st_bf))
131
+ _r1, meta = cg.search("工程 应力", k=10, judge=False, record=False)
132
+ s1 = (meta.get("gates") or {}).get("s1") or {}
133
+ check("S1 收敛到工程域", s1.get("domain") == "工程", str(s1))
134
+ check("S1 丢弃异域候选、保留未标域",
135
+ s1.get("dropped") == 1 and s1.get("in") == 3, str(s1))
136
+ check("S1 scanned 下降(4 -> 3)", meta.get("scanned") == 3,
137
+ str(meta.get("scanned")))
138
+
139
+ root_fb = tempfile.mkdtemp(prefix="retr_s1fb_")
140
+ _write_raw(root_fb, "g1", "knowledge", GONG)
141
+ _write_raw(root_fb, "y1", "knowledge", YI)
142
+ _write_raw(root_fb, "u1", "knowledge", "xyzzy zzz") # 无域信号 → 兜底池
143
+ cgfb = MdCG(root_fb)
144
+ cgfb.rebuild_index()
145
+ cgfb.backfill_big_domain()
146
+ _r2, meta = cgfb.search("工程 应力", k=5, min_results=3, judge=False, record=False)
147
+ s1 = (meta.get("gates") or {}).get("s1") or {}
148
+ check("S1 域内不足 -> 回退全量", s1.get("fallback") == "insufficient", str(s1))
149
+ check("S1 回退后 scanned=全量", meta.get("scanned") == 3, str(meta.get("scanned")))
150
+
151
+ # 兜底召回:未标域节点必须留在域内集合里(否则历史节点被域收敛整片丢弃)
152
+ _r3b, meta = cgfb.search("工程 应力", k=1, judge=False, record=False)
153
+ s1 = (meta.get("gates") or {}).get("s1") or {}
154
+ check("S1 保留未标域节点(兜底池)",
155
+ s1.get("in") == 2 and s1.get("dropped") == 1 and meta.get("scanned") == 2,
156
+ str(s1) + " scanned=" + str(meta.get("scanned")))
157
+
158
+ _r3, meta = cgfb.search("xyzzy zzz", k=10, judge=False, record=False)
159
+ s1 = (meta.get("gates") or {}).get("s1") or {}
160
+ check("S1 无域信号 -> 不收敛",
161
+ s1.get("reason") == "no_domain_signal" and meta.get("scanned") == 3, str(s1))
162
+
163
+ # ---- 5) S2:位置 / 时间窗 / 信息不足 / 全滤回退 ----
164
+ _setenv(MDCG_RETRIEVAL_PIPELINE="1", MDCG_GATE_S1_DOMAIN="0",
165
+ MDCG_GATE_S2_COND="1")
166
+ root2 = tempfile.mkdtemp(prefix="retr_s2_")
167
+ _write_raw(root2, "a", "knowledge", GONG, pos="工程 结构", tw=[0, 10 ** 12])
168
+ _write_raw(root2, "b", "knowledge", YI, pos="医学 疾病", tw=[0, 10 ** 12])
169
+ cg2 = MdCG(root2)
170
+ cg2.rebuild_index()
171
+
172
+ _r, meta = cg2.search("xyzzy", k=10, judge=False, record=False,
173
+ context={"observation_position": "工程 结构"})
174
+ s2 = (meta.get("gates") or {}).get("s2") or {}
175
+ check("S2 位置不匹配被门控", s2.get("dropped") == 1, str(s2))
176
+
177
+ _r, meta = cg2.search("xyzzy", k=10, judge=False, record=False, context={})
178
+ s2 = (meta.get("gates") or {}).get("s2") or {}
179
+ check("S2 信息不足一律放行", s2.get("dropped") == 0, str(s2))
180
+
181
+ _r, meta = cg2.search("xyzzy", k=10, judge=False, record=False,
182
+ context={"time_window": FAR_TW})
183
+ s2 = (meta.get("gates") or {}).get("s2") or {}
184
+ check("S2 时间窗不相交被门控", s2.get("dropped") == 2, str(s2))
185
+
186
+ _r, meta = cg2.search("xyzzy", k=10, judge=False, record=False,
187
+ context={"time_window": [0, 100]})
188
+ s2 = (meta.get("gates") or {}).get("s2") or {}
189
+ check("S2 时间窗相交放行", s2.get("dropped") == 0, str(s2))
190
+
191
+ _r, meta = cg2.search("xyzzy", k=10, judge=False, record=False,
192
+ context={"observation_position": "艺术 绘画"})
193
+ s2 = (meta.get("gates") or {}).get("s2") or {}
194
+ check("S2 全滤 -> 回退(宁多勿漏)",
195
+ s2.get("fallback") == "empty" and meta.get("scanned") == 2, str(s2))
196
+ _restore(old)
197
+
198
+ # ---- 6) backfill(运维口径:先开总开关,再补齐存量)----
199
+ _setenv(MDCG_RETRIEVAL_PIPELINE="1")
200
+ # 6a 默认关时回填必须是 no-op(不写 fm、不写索引)
201
+ root6 = tempfile.mkdtemp(prefix="retr_bf0_")
202
+ _write_raw(root6, "g9", "knowledge", GONG)
203
+ cg6 = MdCG(root6)
204
+ cg6.rebuild_index()
205
+ _setenv(MDCG_RETRIEVAL_PIPELINE=None)
206
+ st6 = cg6.backfill_big_domain()
207
+ check("默认关:回填 no-op(skipped)",
208
+ st6.get("skipped") == "pipeline_disabled" and st6["written"] == 0,
209
+ str(st6))
210
+ check("默认关:回填不改 fm/索引",
211
+ "big_domain" not in (cg6.get("g9")["frontmatter"] or {})
212
+ and "big_domain" not in cg6.index["nodes"]["g9"])
213
+ _setenv(MDCG_RETRIEVAL_PIPELINE="1")
214
+
215
+ root3 = tempfile.mkdtemp(prefix="retr_bf_")
216
+ _write_raw(root3, "g1", "knowledge", GONG)
217
+ _write_raw(root3, "y1", "knowledge", YI)
218
+ _write_raw(root3, "x1", "knowledge", "xyzzy")
219
+ cg3 = MdCG(root3)
220
+ cg3.rebuild_index()
221
+ st1 = cg3.backfill_big_domain(dry_run=True)
222
+ check("backfill dry-run 统计",
223
+ st1["written"] == 2 and st1["no_signal"] == 1, str(st1))
224
+ check("backfill dry-run 不改盘",
225
+ cg3.get("g1")["frontmatter"].get("big_domain") is None)
226
+ st2 = cg3.backfill_big_domain()
227
+ check("backfill 真跑落盘",
228
+ st2["written"] == 2
229
+ and cg3.get("g1")["frontmatter"].get("big_domain") == "工程", str(st2))
230
+ check("backfill 同步索引",
231
+ cg3.index["nodes"]["y1"].get("big_domain") == "医学")
232
+ st3 = cg3.backfill_big_domain()
233
+ check("backfill 幂等",
234
+ st3["written"] == 0 and st3["already"] == 2
235
+ and st3["no_signal"] == 1, str(st3))
236
+ # 索引持久化:重开库(走 _index_log 重放)后条目仍带域标签
237
+ cg3r = MdCG(root3)
238
+ check("backfill 索引持久化(重载可见)",
239
+ cg3r.index["nodes"]["g1"].get("big_domain") == "工程",
240
+ str(cg3r.index["nodes"]["g1"].get("big_domain")))
241
+
242
+ # 对账支路(真状态):**磁盘索引缺键**而 frontmatter 有 → 只修索引不重写文件,且修完重载可见。
243
+ # 造法:rebuild 写出的磁盘索引里删掉该键并清空索引日志(否则日志重放会把键带回来,测试就变成自证)。
244
+ root4 = tempfile.mkdtemp(prefix="retr_bf2_")
245
+ _write_raw(root4, "g1", "knowledge", GONG, bd="工程")
246
+ cg4 = MdCG(root4)
247
+ cg4.rebuild_index()
248
+ idx = json.load(io.open(cg4.index_path, encoding="utf-8"))
249
+ idx["nodes"]["g1"].pop("big_domain", None)
250
+ io.open(cg4.index_path, "w", encoding="utf-8").write(
251
+ json.dumps(idx, ensure_ascii=False))
252
+ ShardedLog.clear(cg4.index_log_dir)
253
+ cg4 = MdCG(root4) # 重载:内存索引确实缺键,而 fm 有
254
+ check("对账前状态成立(索引缺键 / fm 有键)",
255
+ cg4.index["nodes"]["g1"].get("big_domain") is None
256
+ and cg4.get("g1")["frontmatter"].get("big_domain") == "工程")
257
+ st4d = cg4.backfill_big_domain(dry_run=True)
258
+ check("对账 dry-run:只统计不改内存",
259
+ st4d["index_synced"] == 1
260
+ and "big_domain" not in cg4.index["nodes"]["g1"], str(st4d))
261
+ st4 = cg4.backfill_big_domain()
262
+ check("backfill 对账支路修索引",
263
+ st4["index_synced"] == 1 and st4["written"] == 0, str(st4))
264
+ cg4r = MdCG(root4)
265
+ check("对账后重载索引可见(真持久化)",
266
+ cg4r.index["nodes"]["g1"].get("big_domain") == "工程",
267
+ str(cg4r.index["nodes"]["g1"].get("big_domain")))
268
+
269
+ # ---- 7) 默认口径等价(真比较,不是自证)----
270
+ # 7a 无候选分支:开关关时 meta 不得多出 gates 键
271
+ _setenv(MDCG_RETRIEVAL_PIPELINE=None, MDCG_GATE_S1_DOMAIN=None,
272
+ MDCG_GATE_S2_COND=None)
273
+ _r5, meta5 = cg.search("xyzzy", layer="goals", k=10, judge=False,
274
+ record=False)
275
+ check("默认关:无候选分支 meta 无 gates 键",
276
+ meta5.get("reason") == "no_candidates" and "gates" not in meta5,
277
+ str(meta5))
278
+ # 7b 同库同查询:关→开→关 三次,首末两次的 results/meta 必须完全一致
279
+ _ra, meta_a = cg.search("工程 应力", k=10, judge=False, record=False)
280
+ snap_a = ([(r[0]["id"], round(float(r[1]), 6), sorted(r[0].keys()))
281
+ for r in _ra],
282
+ {k: v for k, v in meta_a.items() if k != "gates"})
283
+ _setenv(MDCG_RETRIEVAL_PIPELINE="1", MDCG_GATE_S1_DOMAIN="1",
284
+ MDCG_GATE_S2_COND="1")
285
+ cg.search("工程 应力", k=10, judge=False, record=False)
286
+ _setenv(MDCG_RETRIEVAL_PIPELINE=None, MDCG_GATE_S1_DOMAIN=None,
287
+ MDCG_GATE_S2_COND=None)
288
+ _rb, meta_b = cg.search("工程 应力", k=10, judge=False, record=False)
289
+ snap_b = ([(r[0]["id"], round(float(r[1]), 6), sorted(r[0].keys()))
290
+ for r in _rb],
291
+ {k: v for k, v in meta_b.items() if k != "gates"})
292
+ check("默认口径等价:关→开→关 首末一致", snap_a == snap_b,
293
+ str(snap_a)[:160] + " vs " + str(snap_b)[:160])
294
+ check("默认关:meta 仍不含 gates 键", "gates" not in meta_b,
295
+ str(sorted(meta_b.keys())))
296
+
297
+ # 7c 候选 entry 形状:默认关时即便节点带 observation_position/big_domain,
298
+ # 返回的候选也不得平铺这两个键(独立复核指出的默认口径变化点)
299
+ root5 = tempfile.mkdtemp(prefix="retr_shape_")
300
+ _write_raw(root5, "p1", "knowledge", GONG, pos="工程 结构", bd="工程")
301
+ cg5 = MdCG(root5) # 默认关下建索引
302
+ cg5.rebuild_index()
303
+ check("默认关:索引条目不落门控字段",
304
+ "big_domain" not in cg5.index["nodes"]["p1"]
305
+ and "observation_position" not in cg5.index["nodes"]["p1"],
306
+ str(sorted(cg5.index["nodes"]["p1"].keys())))
307
+ _r5b, _m5b = cg5.search("工程 应力", k=10, judge=False, record=False)
308
+ check("默认关:候选 entry 无门控字段",
309
+ all(("big_domain" not in r[0] and "observation_position" not in r[0])
310
+ for r in _r5b),
311
+ str([sorted(r[0].keys()) for r in _r5b])[:160])
312
+
313
+ # 7d 索引曾在「开启态」构建(条目带门控键)→ 默认关时 search 必须返回剥离后的候选
314
+ _setenv(MDCG_RETRIEVAL_PIPELINE="1")
315
+ root7 = tempfile.mkdtemp(prefix="retr_resid_")
316
+ _write_raw(root7, "q1", "knowledge", GONG, pos="工程 结构", bd="工程")
317
+ cg7 = MdCG(root7)
318
+ cg7.rebuild_index()
319
+ check("开启态:索引条目带门控键",
320
+ cg7.index["nodes"]["q1"].get("big_domain") == "工程"
321
+ and cg7.index["nodes"]["q1"].get("observation_position") == "工程 结构")
322
+ _setenv(MDCG_RETRIEVAL_PIPELINE=None)
323
+ _r7, _m7 = cg7.search("工程 应力", k=10, judge=False, record=False)
324
+ check("默认关:残留门控键被剥离(候选)",
325
+ bool(_r7) and all(("big_domain" not in r[0]
326
+ and "observation_position" not in r[0])
327
+ for r in _r7),
328
+ str([sorted(r[0].keys()) for r in _r7])[:160])
329
+ check("默认关:索引条目本身未被就地改写",
330
+ cg7.index["nodes"]["q1"].get("big_domain") == "工程")
331
+ _restore(old)
332
+
333
+ print("\ntest_retr_s1: %d 通过 / %d 失败" % (passed, failed))
334
+ return 0 if failed == 0 else 1
335
+
336
+
337
+ if __name__ == "__main__":
338
+ if hasattr(sys.stdout, "reconfigure"):
339
+ sys.stdout.reconfigure(encoding="utf-8")
340
+ sys.exit(main())