@furongjun1999/dsh-memory 0.5.0 → 0.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (138) hide show
  1. package/README.md +552 -465
  2. package/docs/README.md +1 -0
  3. package/docs/eval/bench_lingshu_self/bench_self.py +140 -0
  4. package/docs/eval/bench_lingshu_self/self_bench_result.json +404 -0
  5. package/docs/eval/bench_lingshu_self//347/201/265/346/236/242/350/207/252/345/272/223/347/253/257/345/210/260/347/253/257/346/243/200/347/264/242/345/256/236/346/265/213_v1.0.md +38 -0
  6. package/docs/eval//344/270/215/345/217/257/351/235/240/346/200/247/350/220/275/345/234/260_P0_v1.0.md +185 -0
  7. package/docs/eval//346/225/205/351/232/234/346/263/250/345/205/245/345/256/236/346/265/213_v1.0.md +422 -0
  8. package/docs/eval//347/253/257/345/210/260/347/253/257LoCoMoQA/345/220/214/345/217/243/345/276/204/345/257/271/347/205/247_v1.0.md +100 -0
  9. package/docs/eval//347/253/257/345/210/260/347/253/257/345/271/262/346/211/260/346/261/240/350/257/204/346/265/213_/347/241/256/345/256/232/346/200/247/350/243/201/345/206/263vsLLM_judge_v1.1.md +197 -0
  10. package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v1.md +156 -0
  11. package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v10.md +210 -0
  12. package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v11.md +227 -0
  13. package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v12.md +203 -0
  14. package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v13.md +233 -0
  15. package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v14.md +191 -0
  16. package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v15.md +213 -0
  17. package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v16.md +214 -0
  18. package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v17.md +199 -0
  19. package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v2.md +156 -0
  20. package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v3.md +152 -0
  21. package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v4.md +128 -0
  22. package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v5.md +114 -0
  23. package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v6.md +192 -0
  24. package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v7.md +187 -0
  25. package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v8.md +207 -0
  26. package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v9.md +203 -0
  27. package/docs/hive//345/256/211/345/205/250/345/256/241/350/256/241/345/256/236/351/224/232_v0.1.md +202 -0
  28. package/docs/hive//350/234/202/345/267/242/345/217/214/345/256/236/344/276/213/344/272/222/351/252/214_/350/256/276/350/256/241/345/256/232/347/250/277.md +19 -3
  29. package/docs/images/lingshu-moonlight-covenant-poster-preview.jpg +0 -0
  30. package/docs/images/lingshu-moonlight-covenant-poster.png +0 -0
  31. package/docs/mdcg//345/212/237/350/203/275/350/260/203/347/224/250/346/230/240/345/260/204/350/241/250_v0.1.md +40 -40
  32. package/docs/theory//344/270/215/345/217/257/351/235/240/345/256/232/347/220/206/344/270/216/345/244/261/346/225/210/344/274/230/345/205/210/346/241/206/346/236/266_v0.3.md +365 -0
  33. package/docs//344/270/215/345/217/257/351/235/240/346/200/247/347/220/206/350/256/272_v0.1.md +343 -0
  34. package/lib/bridge.d.ts +9 -0
  35. package/lib/bridge.js +35 -0
  36. package/lib/index.js +7 -1
  37. package/lib/lib/roleplay_web.js +530 -443
  38. package/lib/lib/token_store.d.ts +7 -1
  39. package/lib/lib/token_store.js +12 -3
  40. package/md_cg/audit.py +12 -1
  41. package/md_cg/backfill.py +16 -15
  42. package/md_cg/backfill_bucket_zh.py +35 -0
  43. package/md_cg/bench_e2e_judge.py +532 -0
  44. package/md_cg/bench_e2e_locomo_qa.py +368 -0
  45. package/md_cg/bench_e2e_qa.py +256 -0
  46. package/md_cg/branches.py +18 -2
  47. package/md_cg/ccgc.py +3 -2
  48. package/md_cg/chain.py +19 -4
  49. package/md_cg/consolidate.py +7 -6
  50. package/md_cg/crosscheck.py +4 -3
  51. package/md_cg/crypto.py +439 -437
  52. package/md_cg/datapath.py +395 -335
  53. package/md_cg/evidence.py +582 -580
  54. package/md_cg/export.py +3 -1
  55. package/md_cg/forgetting.py +2 -2
  56. package/md_cg/fsutil.py +48 -0
  57. package/md_cg/hotcache.py +255 -238
  58. package/md_cg/interop.py +161 -22
  59. package/md_cg/judgment_manifest.py +177 -0
  60. package/md_cg/links.py +655 -622
  61. package/md_cg/mcp_server.py +280 -29
  62. package/md_cg/mdcg.py +616 -125
  63. package/md_cg/mdcos.py +430 -66
  64. package/md_cg/mreview/govern.py +5 -4
  65. package/md_cg/postings.py +4 -2
  66. package/md_cg/readcache.py +76 -18
  67. package/md_cg/reconcile.py +228 -0
  68. package/md_cg/review_cli.py +170 -0
  69. package/md_cg/routing.py +28 -0
  70. package/md_cg/run_tests.py +211 -0
  71. package/md_cg/scrub.py +862 -852
  72. package/md_cg/security.py +385 -275
  73. package/md_cg/selfreport.py +3 -2
  74. package/md_cg/signer.py +565 -562
  75. package/md_cg/sources.py +3 -2
  76. package/md_cg/stg.py +6 -0
  77. package/md_cg/sustain.py +1168 -1138
  78. package/md_cg/test_access_hints.py +147 -0
  79. package/md_cg/test_branch_discard_tombstone.py +136 -0
  80. package/md_cg/test_branches.py +259 -249
  81. package/md_cg/test_chain_read_isolate.py +168 -0
  82. package/md_cg/test_datapath_device_name.py +203 -0
  83. package/md_cg/test_emit_negtail_cache.py +156 -0
  84. package/md_cg/test_en_pipeline.py +186 -166
  85. package/md_cg/test_govern_directread.py +421 -0
  86. package/md_cg/test_i32_hotcache_env_key.py +218 -0
  87. package/md_cg/test_identity_attribution.py +228 -147
  88. package/md_cg/test_index_durability.py +238 -224
  89. package/md_cg/test_interop.py +4 -2
  90. package/md_cg/test_interop_judgment.py +228 -0
  91. package/md_cg/test_issue39_utf8_stdio.py +273 -0
  92. package/md_cg/test_links_concurrent_write.py +188 -0
  93. package/md_cg/test_merge_upsert.py +168 -0
  94. package/md_cg/test_n123_derive_expiry_chain.py +205 -0
  95. package/md_cg/test_n130_verify_falsified_protect.py +185 -0
  96. package/md_cg/test_n131_merge_gate.py +205 -0
  97. package/md_cg/test_p1x_ref_root.py +160 -0
  98. package/md_cg/test_p27_docindex.py +774 -765
  99. package/md_cg/test_p2_mcp.py +3 -0
  100. package/md_cg/test_p32_backfill.py +304 -298
  101. package/md_cg/test_p39_verify_flow.py +90 -50
  102. package/md_cg/test_p47_session_view.py +60 -25
  103. package/md_cg/test_propose_tail_index.py +157 -0
  104. package/md_cg/test_read_scope_b27.py +277 -0
  105. package/md_cg/test_readcache_default_on.py +168 -0
  106. package/md_cg/test_readcache_precise_inval.py +270 -0
  107. package/md_cg/test_readcache_prodpath.py +55 -7
  108. package/md_cg/test_reconcile_v0.py +294 -0
  109. package/md_cg/test_retr_s1.py +6 -2
  110. package/md_cg/test_retr_s1b.py +67 -0
  111. package/md_cg/test_retr_s7.py +8 -0
  112. package/md_cg/test_retr_s9_entity_ctx.py +181 -175
  113. package/md_cg/test_retr_score_once.py +208 -0
  114. package/md_cg/test_review_onepass.py +170 -0
  115. package/md_cg/test_rrf_graph_seed_cache.py +154 -0
  116. package/md_cg/test_security_audit.py +155 -0
  117. package/md_cg/test_security_audit_b26.py +161 -0
  118. package/md_cg/test_security_audit_v21.py +250 -0
  119. package/md_cg/test_semantic_canonical.py +255 -241
  120. package/md_cg/test_session_isolation.py +168 -0
  121. package/md_cg/test_snapshot_autoclose.py +187 -0
  122. package/md_cg/test_tail_watermark_race.py +208 -0
  123. package/md_cg/test_tenant_env_override_warn.py +139 -0
  124. package/md_cg/test_tenant_registry_corrupt_warn.py +151 -0
  125. package/md_cg/test_v14_fixes.py +415 -397
  126. package/md_cg/test_verify_dirty_reconcile.py +157 -0
  127. package/md_cg/theory.py +276 -273
  128. package/md_cg/tokens.py +734 -677
  129. package/md_cg/units.py +3 -2
  130. package/md_cg/vision_evidence.py +4 -3
  131. package/md_cg/whitebox_kb/wisdom/code_solidified.json +6195 -6195
  132. package/md_cg/whitebox_kb/wisdom/multilang_ir.py +128 -124
  133. package/md_cg/writepipe.py +554 -550
  134. package/package.json +6 -2
  135. package/src/bridge.ts +434 -401
  136. package/src/index.ts +526 -518
  137. package/src/lib/roleplay_web.ts +1019 -932
  138. package/src/lib/token_store.ts +202 -192
@@ -0,0 +1,168 @@
1
+ # -*- coding: utf-8 -*-
2
+ """test_session_isolation —— 会话隔离模型守卫(issue #35 设计定稿)
3
+
4
+ 设计定稿(2026-09-23 使用者拍板):
5
+ · public / internal = 跨会话共享档——通过 clearance 即可见(含他人会话写入);
6
+ · private / secret = 会话绑定档——仅归属会话可见;
7
+ - 归属会话 = frontmatter.session(服务端身份不可伪造);
8
+ - 绑定档无归属(存量)→ fail-closed 仅设计者可见;
9
+ - 查询侧 "*" 不再越过 private 绑定(非 admin 的 "*" 与不传等价);
10
+ · can_admin(设计者)豁免——整体判断需要全景;错误/冲突信息(rejected/
11
+ unresolved)恒不进默认正排,设计者经显式通道获取;
12
+ · tenant 物理根接线(issue #35 本体):已登记租户走登记根,冲突 fail-closed,
13
+ 未登记回落 MDCG_ROOT(零变更)。
14
+
15
+ 运行:python -X utf8 -m md_cg.test_session_isolation
16
+ """
17
+ import os
18
+ import sys
19
+ import tempfile
20
+
21
+ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
22
+
23
+ from md_cg.mcp_server import _resolve_root # noqa: E402
24
+ from md_cg.mdcos import MdCGSecure # noqa: E402
25
+ from md_cg.security import Principal, TenantRegistry # noqa: E402
26
+
27
+ passed = failed = 0
28
+
29
+
30
+ def check(name, cond, detail=""):
31
+ global passed, failed
32
+ if cond:
33
+ passed += 1
34
+ print(" [PASS] " + name)
35
+ else:
36
+ failed += 1
37
+ print(" [FAIL] " + name + " " + str(detail)[:200])
38
+
39
+
40
+ def _principal(sess, **kw):
41
+ kw.setdefault("actor", "tester")
42
+ kw.setdefault("clearance", "private")
43
+ kw.setdefault("can_write", True)
44
+ kw.setdefault("role", "recorder")
45
+ return Principal(session=sess, **kw)
46
+
47
+
48
+ def _search_ids(cg, q="共享 私有 标记", k=20, **kw):
49
+ res, _m = cg.search_rrf(q, k=k, judge=False, **kw)
50
+ return sorted(r[0]["id"] for r in res)
51
+
52
+
53
+ def main():
54
+ root = tempfile.mkdtemp(prefix="sess_iso_")
55
+ pa = _principal("sessA")
56
+ a = MdCGSecure(os.path.join(root, "mem"), principal=pa)
57
+ a.add("pub1", "共享公开级记忆:发布窗口每周三", "knowledge",
58
+ sensitivity="public")
59
+ a.add("int1", "共享默认级记忆:接口约定时间戳毫秒", "knowledge",
60
+ sensitivity="internal")
61
+ a.add("priv1", "会话私有记忆:A 的个人偏好与口令提示", "knowledge",
62
+ sensitivity="private")
63
+ a.flush()
64
+
65
+ print("[1] 跨会话共享档:B 能看到 A 的 public/internal,看不到 private")
66
+ b = MdCGSecure(os.path.join(root, "mem"), principal=_principal("sessB"))
67
+ ids = _search_ids(b)
68
+ check("B(不传 session)见共享档", "pub1" in ids and "int1" in ids, ids)
69
+ check("B(不传 session)不见 A 的 private", "priv1" not in ids, ids)
70
+ check("B get(private) → None(不可见即不存在)",
71
+ b.get("priv1") is None)
72
+ check("B get(public) 可读", b.get("pub1") is not None)
73
+
74
+ print("[2] 查询侧 session 参数与收紧的 \"*\"")
75
+ ids2 = _search_ids(b, session="sessB")
76
+ check("B 传自己 session 仍见共享档(super 硬等值已上收)",
77
+ "pub1" in ids2 and "int1" in ids2 and "priv1" not in ids2, ids2)
78
+ ids3 = _search_ids(b, session="*")
79
+ check("B 传 \"*\" 不越过 private 绑定(收紧)", "priv1" not in ids3, ids3)
80
+
81
+ print("[3] 归属会话自身与设计者豁免")
82
+ a2 = MdCGSecure(os.path.join(root, "mem"), principal=_principal("sessA"))
83
+ ids4 = _search_ids(a2)
84
+ check("A(归属会话)见自己的全部三档",
85
+ {"pub1", "int1", "priv1"} <= set(ids4), ids4)
86
+ adm = MdCGSecure(os.path.join(root, "mem"),
87
+ principal=_principal("sessAdmin", can_admin=True,
88
+ role="designer"))
89
+ ids5 = _search_ids(adm)
90
+ check("设计者(can_admin)跨会话见全部(含 private)",
91
+ {"pub1", "int1", "priv1"} <= set(ids5), ids5)
92
+
93
+ print("[4] 绑定档无归属(存量)fail-closed")
94
+ import json as _json
95
+ e = a.index["nodes"]["priv1"]
96
+ p = os.path.join(root, "mem", e["path"])
97
+ fm, content = a._read(e)
98
+ fm.pop("session", None)
99
+ a._write_node("priv1", p, fm, content)
100
+ a.index["nodes"]["priv1"]["session"] = None
101
+ a.rebuild_index()
102
+ b2 = MdCGSecure(os.path.join(root, "mem"), principal=_principal("sessB"))
103
+ check("无归属 private:普通会话不可见(fail-closed)",
104
+ "priv1" not in _search_ids(b2))
105
+ check("无归属 private:设计者可见",
106
+ "priv1" in _search_ids(adm))
107
+
108
+ print("[5] rejected(错误标记)不进默认正排")
109
+ rj = a.add_rejected("会话私有的错误假设", "守卫用", sensitivity="public")
110
+ a.flush()
111
+ check("rejected 不进任何会话的默认正排",
112
+ rj not in _search_ids(b2) and rj not in _search_ids(adm))
113
+
114
+ print("[6] 租户物理根解析(_resolve_root)")
115
+ reg_path = os.path.join(root, "_tenants.json")
116
+ reg = TenantRegistry(path=reg_path)
117
+ reg.register("tenantA", os.path.join(root, "A"))
118
+ reg.register("tenantB", os.path.join(root, "B"))
119
+
120
+ r, err = _resolve_root({"MDCG_ROOT": os.path.join(root, "A"),
121
+ "MDCG_TENANT": "tenantA",
122
+ "MDCG_TENANT_REGISTRY": reg_path})
123
+ check("已登记租户 + 一致 MDCG_ROOT → 登记根",
124
+ not err and os.path.abspath(r) == os.path.abspath(
125
+ os.path.join(root, "A")), (r, err))
126
+
127
+ r, err = _resolve_root({"MDCG_TENANT": "tenantB",
128
+ "MDCG_TENANT_REGISTRY": reg_path})
129
+ check("已登记租户 + 无 MDCG_ROOT → 登记根(私库可零配置)",
130
+ not err and r == os.path.abspath(os.path.join(root, "B")), (r, err))
131
+
132
+ r, err = _resolve_root({"MDCG_ROOT": os.path.join(root, "A"),
133
+ "MDCG_TENANT": "tenantB",
134
+ "MDCG_TENANT_REGISTRY": reg_path})
135
+ check("登记根与 MDCG_ROOT 冲突 → fail-closed(root=None + 冲突说明)",
136
+ r is None and "fail-closed" in (err or ""), (r, err))
137
+
138
+ r, err = _resolve_root({"MDCG_ROOT": os.path.join(root, "X")})
139
+ check("未设租户 → 维持 MDCG_ROOT(零变更)",
140
+ not err and r == os.path.join(root, "X"), (r, err))
141
+
142
+ r, err = _resolve_root({"MDCG_ROOT": os.path.join(root, "X"),
143
+ "MDCG_TENANT": "未登记租户"})
144
+ check("租户未登记 → 回落 MDCG_ROOT(不误伤)",
145
+ not err and r == os.path.join(root, "X"), (r, err))
146
+
147
+ r, err = _resolve_root({})
148
+ check("全空 → (None, None)(main 侧报缺 MDCG_ROOT)",
149
+ r is None and not err, (r, err))
150
+
151
+ print("[7] 双租户物理隔离端到端(各自根,互不可见)")
152
+ ta = MdCGSecure(os.path.join(root, "A"), principal=_principal("sessA"))
153
+ ta.add("only_a", "租户A的记忆", "knowledge", sensitivity="public")
154
+ ta.flush()
155
+ tb = MdCGSecure(os.path.join(root, "B"), principal=_principal("sessB"))
156
+ tb.add("only_b", "租户B的记忆", "knowledge", sensitivity="public")
157
+ tb.flush()
158
+ check("A 根检索只见 A(物理隔离)",
159
+ "only_a" in _search_ids(ta) and "only_b" not in _search_ids(ta))
160
+ check("B 根检索只见 B(物理隔离)",
161
+ "only_b" in _search_ids(tb) and "only_a" not in _search_ids(tb))
162
+
163
+ print(f"\nsession_isolation: {passed} 通过 / {failed} 失败")
164
+ return 0 if not failed else 1
165
+
166
+
167
+ if __name__ == "__main__":
168
+ sys.exit(main())
@@ -0,0 +1,187 @@
1
+ #!/usr/bin/env python3
2
+ # -*- coding: utf-8 -*-
3
+ """索引快照自动落盘守卫(issue:add→close 永不写快照,重开付全库扫描+重放)。
4
+
5
+ 缺陷形态(缺陷报告实测):flush 只 append 分片日志、compact_index 生产零
6
+ 调用、close/atexit 只 flush——「add→close」的库永远没有快照,每次重开
7
+ _load_index 都 _scan_nodes 全库逐文件 open+frontmatter 解析(1500 池本机
8
+ ~104ms),随后无条件重放全部索引日志(在扫描基底上纯冗余)。
9
+
10
+ 本守卫断言(调用计数/存在性形态,非时间阈值):
11
+ C1 close 落快照:add→close 后 _index.json 存在(旧代码恒不存在 → 红);
12
+ C2 重开零全库扫描:close 后重开期间 _scan_nodes 调用 == 0(旧代码
13
+ 每次重开 1 次全库扫描 → 红);
14
+ C3 重开零日志重放:close 后重开期间 ShardedLog.read_all 产出条数 == 0
15
+ (旧代码重放全部增量日志 → 红);
16
+ C4 节点集一致:close→重开后 index["nodes"] 的 id 集与关闭前一致;
17
+ C5 尾部写入不丢:未达 autoflush 阈值的尾部 add 在 close 后重开可见;
18
+ C6 快照损坏防御:_index.json 写坏后 close/compact 不产出空快照——
19
+ 节点仍全量可见(旧 compact 拿空骨架+增量日志会写出清池快照 → 红);
20
+ C7 空库不落快照:无节点时 close 不产生 _index.json(保持原行为)。
21
+ """
22
+ import os
23
+ import shutil
24
+ import sys
25
+ import tempfile
26
+
27
+ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
28
+
29
+ import md_cg.mdcg as mdcg_mod # noqa: E402
30
+ from md_cg.fsutil import ShardedLog # noqa: E402
31
+ from md_cg.mdcos import MdCGOS # noqa: E402
32
+
33
+ PASS = 0
34
+ FAIL = 0
35
+ FAILS = []
36
+
37
+
38
+ def check(name, cond, detail=""):
39
+ global PASS, FAIL
40
+ if cond:
41
+ PASS += 1
42
+ print(f" OK {name}")
43
+ else:
44
+ FAIL += 1
45
+ FAILS.append(name)
46
+ print(f" FAIL {name} {detail}")
47
+
48
+
49
+ def count_reload_reads(fn):
50
+ """打桩 _scan_nodes 调用数与 ShardedLog.read_all 产出条数。"""
51
+ n_scan, n_replay = 0, 0
52
+ real_scan = mdcg_mod.MdCG._scan_nodes
53
+ real_read = ShardedLog.read_all
54
+
55
+ def _scan_spy(self):
56
+ nonlocal n_scan
57
+ n_scan += 1
58
+ return real_scan(self)
59
+
60
+ def _read_spy(path, *a, **kw):
61
+ nonlocal n_replay
62
+ out = list(real_read(path, *a, **kw))
63
+ n_replay += len(out)
64
+ return iter(out)
65
+
66
+ mdcg_mod.MdCG._scan_nodes = _scan_spy
67
+ ShardedLog.read_all = staticmethod(_read_spy)
68
+ mdcg_mod.ShardedLog.read_all = staticmethod(_read_spy)
69
+ try:
70
+ return fn(), n_scan, n_replay
71
+ finally:
72
+ mdcg_mod.MdCG._scan_nodes = real_scan
73
+ ShardedLog.read_all = real_read
74
+ mdcg_mod.ShardedLog.read_all = real_read
75
+
76
+
77
+ def build(root, n=30):
78
+ cg = MdCGOS(root, autoflush=64) # 高阈值:尾部写入留在 _dirty
79
+ for i in range(n):
80
+ cg.add("mem_%04d" % i,
81
+ "# 功能名:快照守卫 %d\n# 正文:蜂群调度 样本 %d" % (i, i),
82
+ layer="knowledge")
83
+ return cg
84
+
85
+
86
+ def main():
87
+ root = tempfile.mkdtemp(prefix="mdcg_snap_guard_")
88
+ try:
89
+ # C1/C4/C5:close 落快照 + 尾部写入 + 节点集一致
90
+ print("== C1/C4/C5 close 落快照 ==")
91
+ cg = build(root)
92
+ ids_before = set(cg.index["nodes"])
93
+ check("C5pre 尾部写入在 _dirty(未 flush)", len(cg._dirty) > 0,
94
+ f"dirty={len(cg._dirty)}")
95
+ cg.close()
96
+ snap = os.path.join(root, "_index.json")
97
+ check("C1a add→close 后 _index.json 快照存在(旧代码恒不存在)",
98
+ os.path.exists(snap))
99
+
100
+ def _reopen():
101
+ return MdCGOS(root)
102
+
103
+ cg2, n_scan, n_replay = count_reload_reads(_reopen)
104
+ try:
105
+ ids_after = set(cg2.index["nodes"])
106
+ check("C4a 重开后节点集与关闭前一致",
107
+ ids_after == ids_before and len(ids_after) == 30,
108
+ f"before={len(ids_before)} after={len(ids_after)}")
109
+ finally:
110
+ cg2.close()
111
+ check("C2a 重开零全库扫描(_scan_nodes 调用==0,旧每次重开 1 次)",
112
+ n_scan == 0, f"scans={n_scan}")
113
+ check("C3a 重开零日志重放(read_all 条数==0,旧全量重放)",
114
+ n_replay == 0, f"replayed={n_replay}")
115
+
116
+ # C6 快照损坏防御:写坏快照 → 重开(scan 基底)→ close 不产空快照
117
+ print("== C6 快照损坏防御 ==")
118
+ with open(snap, "w", encoding="utf-8") as f:
119
+ f.write("{corrupted json !!!")
120
+ root6 = root # 同库继续
121
+ cg6 = MdCGOS(root6) # 损坏 → _scan_nodes 基底
122
+ n6 = len(cg6.index["nodes"])
123
+ check("C6a 损坏快照下重开走全库扫描(节点全量可见)",
124
+ n6 == 30, f"nodes={n6}")
125
+ cg6.close()
126
+ cg6b = MdCGOS(root6)
127
+ try:
128
+ n6b = len(cg6b.index["nodes"])
129
+ check("C6b 损坏后 close/compact 不产空快照(重开仍 30 节点)",
130
+ n6b == 30, f"nodes={n6b}")
131
+ finally:
132
+ cg6b.close()
133
+
134
+ # C8 未 flush 写入兜底(issue #33 修复中发现的回归形态):实例 A
135
+ # 写节点不收尾(dirty 未 flush、文件已落盘),实例 B close 落快照
136
+ # ——B 的账本不含 A 的节点。此后重开必须仍能看到 A 的节点
137
+ # (compact 计数对账 + 重开指纹校验双保险;任一缺失都会让节点
138
+ # 「在盘上但索引不可见」)。
139
+ print("== C8 未 flush 写入兜底 ==")
140
+ root8 = tempfile.mkdtemp(prefix="mdcg_snap_guard8_")
141
+ try:
142
+ a = MdCGOS(root8, autoflush=64)
143
+ a.add("a1", "# 功能名:甲\n# 正文:甲的节点 蜂群", layer="knowledge")
144
+ a.add("a2", "# 功能名:乙\n# 正文:乙的节点 星云", layer="knowledge")
145
+ b = MdCGOS(root8, autoflush=64)
146
+ b.add("b1", "# 功能名:丙\n# 正文:丙的节点 水位", layer="knowledge")
147
+ b.close() # B 落快照(账本不含 a1/a2)
148
+ c = MdCGOS(root8)
149
+ try:
150
+ ids_c = set(c.index["nodes"])
151
+ check("C8a A 未 flush 的节点在 B close 落快照后重开仍可见",
152
+ {"a1", "a2", "b1"} <= ids_c, f"ids={sorted(ids_c)}")
153
+ finally:
154
+ c.close()
155
+ d = MdCGOS(root8)
156
+ try:
157
+ ids_d = set(d.index["nodes"])
158
+ check("C8b C close 后再次重开节点仍全量可见(快照已含全部)",
159
+ {"a1", "a2", "b1"} <= ids_d, f"ids={sorted(ids_d)}")
160
+ finally:
161
+ d.close()
162
+ finally:
163
+ shutil.rmtree(root8, ignore_errors=True)
164
+
165
+ # C7 空库不落快照
166
+ print("== C7 空库不落快照 ==")
167
+ root7 = tempfile.mkdtemp(prefix="mdcg_snap_guard7_")
168
+ try:
169
+ cg7 = MdCGOS(root7)
170
+ cg7.close()
171
+ check("C7a 空库 close 不产生 _index.json(保持原行为)",
172
+ not os.path.exists(os.path.join(root7, "_index.json")))
173
+ finally:
174
+ shutil.rmtree(root7, ignore_errors=True)
175
+ finally:
176
+ shutil.rmtree(root, ignore_errors=True)
177
+
178
+ print("\n" + "=" * 60)
179
+ print(f"结果:PASS {PASS} / FAIL {FAIL}")
180
+ if FAILS:
181
+ for f in FAILS:
182
+ print(f" - {f}")
183
+ return 0 if FAIL == 0 else 1
184
+
185
+
186
+ if __name__ == "__main__":
187
+ sys.exit(main())
@@ -0,0 +1,208 @@
1
+ # -*- coding: utf-8 -*-
2
+ """四增量对账缓存水位守卫(缺陷:水位存 stat 时刻值,stat→read 窗口并发 append 重复并入)。
3
+
4
+ 缺陷形态(缺陷报告 2026-09-25 复现证实,medium):_pid_status /
5
+ _inbox_phash_index / _inbox_records / _decisions_records 四缓存(mdcos.py
6
+ :1708/:1739/:1768/:1793)扫描水位存「读前 stat 的 size」,而
7
+ fsutil.read_jsonl_tail(:228-249)seek(offset) 后**读到真实 EOF**——跨进程
8
+ append 恰落在 stat 与 read 之间时,新记录本轮已并入但水位偏小,下一轮增量
9
+ 从旧水位重读:列表型缓存重复并入同一条记录(review_list 待审清单重复 pid、
10
+ review_stats 计数虚高、_cascade_dedup 连写两条 reject),且进程内永不自愈
11
+ (稳态 size 只增,不触发全量重建)。
12
+
13
+ 修复:read_jsonl_tail 返回 (记录列表, 真实读到的 EOF 偏移),四处缓存把水位
14
+ 存为**实际消费位置**(含首装/回退全量路径统一走 tail(0) 取精确 EOF);
15
+ 读后再 stat 不可取——re-stat 可能大于已消费位置,下轮会跳过未读记录(丢账)。
16
+
17
+ 红守卫 = 缺陷报告复现口径(monkeypatch mdcos.read_jsonl_tail / read_jsonl,
18
+ 在 stat 后、read 前注入并发 append,等价真实交错):
19
+ A decisions:窗口 append 后 records 必须 == 盘上条数,且第三次调用仍相等
20
+ (旧代码:本轮对、下轮重读翻倍、永不自愈);
21
+ B inbox:review_list 无重复 pid 且长度 == 盘上条数;
22
+ C _pid_status dict 覆盖免疫(proposals == 去重 pid 数,语义锁);
23
+ D 首装全量路径同窗(旧代码全量重建同样把 stat size 当水位,一样翻倍)。
24
+ 运行:python -m md_cg.test_tail_watermark_race
25
+ """
26
+ from __future__ import annotations
27
+
28
+ import os
29
+ import shutil
30
+ import sys
31
+ import tempfile
32
+ import time
33
+
34
+ from . import mdcos
35
+ from .fsutil import append_jsonl, read_jsonl
36
+ from .mdcos import MdCGOS
37
+
38
+ PASS = FAIL = 0
39
+ FAILS = []
40
+
41
+
42
+ def ok(cond, label):
43
+ global PASS, FAIL
44
+ if cond:
45
+ PASS += 1
46
+ print(f" OK {label}")
47
+ else:
48
+ FAIL += 1
49
+ FAILS.append(label)
50
+ print(f" FAIL {label}")
51
+
52
+
53
+ class _WindowInjector:
54
+ """把一条 append 注入 stat→read 窗口:目标函数被调(=stat 已发生)时
55
+ 先落盘注入记录再委托原函数——等价跨进程 append 恰落在窗口内的交错。"""
56
+
57
+ def __init__(self, mod, log_path, rec):
58
+ self.mod, self.path, self.rec = mod, log_path, rec
59
+ self.fired = False
60
+ self._orig_tail = mod.read_jsonl_tail
61
+ self._orig_full = mod.read_jsonl
62
+
63
+ def _maybe_inject(self, path):
64
+ if not self.fired and os.path.abspath(str(path)) == \
65
+ os.path.abspath(self.path):
66
+ self.fired = True
67
+ append_jsonl(self.path, self.rec)
68
+
69
+ def _tail(self, path, offset):
70
+ self._maybe_inject(path)
71
+ return self._orig_tail(path, offset)
72
+
73
+ def _full(self, path):
74
+ self._maybe_inject(path)
75
+ return self._orig_full(path)
76
+
77
+ def __enter__(self):
78
+ self.mod.read_jsonl_tail = self._tail
79
+ self.mod.read_jsonl = self._full
80
+ return self
81
+
82
+ def __exit__(self, *a):
83
+ self.mod.read_jsonl_tail = self._orig_tail
84
+ self.mod.read_jsonl = self._orig_full
85
+
86
+
87
+ def _n_on_disk(path):
88
+ return len(list(read_jsonl(path)))
89
+
90
+
91
+ def _dec(pid, decision="noop"):
92
+ return {"t": time.time(), "pid": pid, "decision": decision,
93
+ "status": decision, "round": 1, "actor": "race_guard"}
94
+
95
+
96
+ def _inbox(pid):
97
+ return {"t": time.time(), "pid": pid, "id": "m_" + pid,
98
+ "content": "窗口竞态守卫正文 " + pid, "layer": "knowledge"}
99
+
100
+
101
+ def _scenario_decisions(tmp):
102
+ """A:decisions 尾部窗口(review_stats 口径)。"""
103
+ print("== A decisions 窗口竞态(review_stats) ==")
104
+ root = os.path.join(tmp, "dec")
105
+ os.makedirs(root)
106
+ cg = MdCGOS(root)
107
+ append_jsonl(cg.decisions_log, _dec("p1"))
108
+ append_jsonl(cg.decisions_log, _dec("p2"))
109
+ st1 = cg.review_stats()
110
+ ok(st1["records"] == 2, f"A1 基线装载 records=2(实测 {st1['records']})")
111
+
112
+ append_jsonl(cg.decisions_log, _dec("p3_ext")) # stat 前并发 append
113
+ with _WindowInjector(mdcos, cg.decisions_log, _dec("p4_ext")):
114
+ st2 = cg.review_stats() # 读窗内并发 append
115
+ disk = _n_on_disk(cg.decisions_log)
116
+ ok(st2["records"] == disk,
117
+ f"A2 窗口轮 records==盘上({st2['records']} vs 盘上 {disk})")
118
+
119
+ st3 = cg.review_stats()
120
+ ok(st3["records"] == disk,
121
+ f"A3 下一轮不重读(records={st3['records']},盘上 {disk}——"
122
+ f"旧代码重读 p4_ext 虚高为 {st3['records']})")
123
+ st4 = cg.review_stats()
124
+ ok(st4["records"] == disk,
125
+ f"A4 永不自愈形态不得出现(第三次调用 records={st4['records']},"
126
+ f"盘上 {disk})")
127
+ disk_noop = sum(1 for r in read_jsonl(cg.decisions_log)
128
+ if r.get("decision") == "noop")
129
+ ok(st4["by_decision"].get("noop", 0) == disk_noop,
130
+ f"A5 noop 计数与盘上一致({st4['by_decision'].get('noop', 0)} vs "
131
+ f"{disk_noop})")
132
+ # C:_pid_status 为 dict 后写覆盖——窗口竞态不得影响 pid 去重口径
133
+ pids_disk = {r.get("pid") for r in read_jsonl(cg.decisions_log)}
134
+ st5 = cg.review_stats()
135
+ ok(st5["proposals"] == len(pids_disk)
136
+ and set(cg._pid_status()) == pids_disk,
137
+ f"C1 proposals==盘上去重 pid 数({st5['proposals']} vs "
138
+ f"{len(pids_disk)})——dict 覆盖免疫不受影响")
139
+
140
+
141
+ def _scenario_inbox(tmp):
142
+ """B:inbox 尾部窗口(review_list 待审清单)。"""
143
+ print("== B inbox 窗口竞态(review_list) ==")
144
+ root = os.path.join(tmp, "inbox")
145
+ os.makedirs(root)
146
+ cg = MdCGOS(root)
147
+ append_jsonl(cg.inbox_log, _inbox("q1"))
148
+ append_jsonl(cg.inbox_log, _inbox("q2"))
149
+ rl1 = cg.review_list()
150
+ ok(len(rl1) == 2, f"B1 基线装载待审 2(实测 {len(rl1)})")
151
+
152
+ append_jsonl(cg.inbox_log, _inbox("q3_ext"))
153
+ with _WindowInjector(mdcos, cg.inbox_log, _inbox("q4_ext")):
154
+ rl2 = cg.review_list()
155
+ disk = _n_on_disk(cg.inbox_log)
156
+ ok(len(rl2) == disk,
157
+ f"B2 窗口轮待审数==盘上({len(rl2)} vs {disk})")
158
+
159
+ rl3 = cg.review_list()
160
+ pids = [r.get("pid") for r in rl3]
161
+ ok(len(rl3) == disk and len(set(pids)) == len(pids),
162
+ f"B3 下一轮无重复 pid(len={len(rl3)},去重 {len(set(pids))},"
163
+ f"盘上 {disk}——旧代码 q4_ext 重复并入)")
164
+ rl4 = cg.review_list()
165
+ pids4 = [r.get("pid") for r in rl4]
166
+ ok(len(rl4) == disk and len(set(pids4)) == len(pids4),
167
+ f"B4 第三次调用仍无重复(len={len(rl4)},盘上 {disk})")
168
+
169
+
170
+ def _scenario_first_load(tmp):
171
+ """D:首装全量路径同窗(旧代码全量重建也把 stat size 当水位)。"""
172
+ print("== D 首装全量路径同窗 ==")
173
+ root = os.path.join(tmp, "first")
174
+ os.makedirs(root)
175
+ cg = MdCGOS(root)
176
+ append_jsonl(cg.decisions_log, _dec("d1"))
177
+ append_jsonl(cg.decisions_log, _dec("d2"))
178
+ with _WindowInjector(mdcos, cg.decisions_log, _dec("d3_ext")):
179
+ st1 = cg.review_stats() # 首装发生在窗口内
180
+ disk = _n_on_disk(cg.decisions_log)
181
+ ok(st1["records"] == disk,
182
+ f"D1 首装窗口轮 records==盘上({st1['records']} vs {disk})")
183
+ st2 = cg.review_stats()
184
+ ok(st2["records"] == disk,
185
+ f"D2 首装后下一轮不重读(records={st2['records']},盘上 {disk})")
186
+
187
+
188
+ def main():
189
+ os.environ.pop("MDCG_READ_CACHE", None)
190
+ tmp = tempfile.mkdtemp(prefix="mdcg_tail_race_")
191
+ try:
192
+ _scenario_decisions(tmp)
193
+ _scenario_inbox(tmp)
194
+ _scenario_first_load(tmp)
195
+ finally:
196
+ os.environ.pop("MDCG_READ_CACHE", None)
197
+ shutil.rmtree(tmp, ignore_errors=True)
198
+
199
+ print("\n" + "=" * 60)
200
+ print(f"结果:PASS {PASS} / FAIL {FAIL}")
201
+ if FAILS:
202
+ for f in FAILS:
203
+ print(f" - {f}")
204
+ return 0 if FAIL == 0 else 1
205
+
206
+
207
+ if __name__ == "__main__":
208
+ sys.exit(main())