@furongjun1999/dsh-memory 0.9.22 → 0.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.en.md +450 -0
- package/CHANGELOG.md +112 -0
- package/README.en.md +244 -0
- package/README.md +272 -260
- package/codebuddy/CODEBUDDY.md +1 -1
- package/docs/README.md +2 -0
- package/docs/discipline/templates/full.md.tmpl +1 -1
- package/docs/discipline/templates/rules.mdc.tmpl +1 -1
- package/docs/en/README.md +190 -0
- package/docs/en/TRANSLATION.md +78 -0
- package/docs/en/TRANSLATION_MANIFEST.json +170 -0
- package/docs/en/TRANSLATION_REPORT.md +11 -0
- package/docs/en/dev/README.md +10 -0
- package/docs/en/dev/coding-testing.md +72 -0
- package/docs/en/dev/switch-matrix.md +10 -0
- package/docs/en/glossary.md +97 -0
- package/docs/en/harnesses/claude.md +100 -0
- package/docs/en/harnesses/codebuddy.md +93 -0
- package/docs/en/harnesses/codex.md +88 -0
- package/docs/en/harnesses/dsh.md +84 -0
- package/docs/en/harnesses/zcode.md +42 -0
- package/docs/en/mdcg/ccg.md +32 -0
- package/docs/en/mdcg/generated-docs.md +20 -0
- package/docs/en/mdcg/guardrail-charter.md +170 -0
- package/docs/en/mdcg/mcp-tool-names.md +43 -0
- package/docs/en/mdcg/multi-harness.md +43 -0
- package/docs/en/mdcg/recall-metadata.md +30 -0
- package/docs/en/mdcg/world-model-endpoint.md +379 -0
- package/docs/en/mdcg/write-aliases.md +130 -0
- package/docs/en/ops/README.md +11 -0
- package/docs/en/ops/environment-known-issues.md +244 -0
- package/docs/en/ops/install.md +72 -0
- package/docs/en/ops/operations-troubleshooting.md +86 -0
- package/docs/en/ops/permissions-review.md +79 -0
- package/docs/en/ops/release-governance.md +49 -0
- package/docs/en/ops/tool-name-migration.md +30 -0
- package/docs/en/ops/ui-install-troubleshooting.md +53 -0
- package/docs/en/standards/ai-coding-standard-checklist.md +71 -0
- package/docs/en/standards/ai-coding-standard-lingshu.md +47 -0
- package/docs/en/standards/ai-coding-standard.md +185 -0
- package/docs/en/ui/minimal.md +62 -0
- package/docs/mdcg/cogmap_ops.md +16 -15
- package/lib/client.js +29 -3
- package/lib/hooks.d.ts +34 -0
- package/lib/hooks.js +167 -33
- package/lib/index.js +19 -3
- package/lib/lib/console_web.mjs +30 -3
- package/lib/lib/hook_audit.d.ts +3 -2
- package/lib/lib/hook_audit.js +14 -4
- package/lib/lib/mdcg_client.d.ts +5 -2
- package/lib/lib/mdcg_client.js +8 -4
- package/lib/lib/prompt_safety.d.ts +24 -0
- package/lib/lib/prompt_safety.js +77 -1
- package/lib/lib/token_store.js +5 -2
- package/lib/tools.js +5 -3
- package/md_cg/bucketindex.py +231 -0
- package/md_cg/bucketscope.py +339 -0
- package/md_cg/config_registry_bulk.py +19 -10
- package/md_cg/config_validate.py +14 -2
- package/md_cg/detrerank.py +137 -0
- package/md_cg/fsutil.py +215 -25
- package/md_cg/govern/__init__.py +23 -0
- package/md_cg/iface/__init__.py +46 -0
- package/md_cg/iface/_verify.py +609 -0
- package/md_cg/iface/contracts.py +337 -0
- package/md_cg/judgment_manifest.py +14 -0
- package/md_cg/kernel/__init__.py +27 -0
- package/md_cg/mcp_server.py +457 -23
- package/md_cg/mdcg.py +1034 -79
- package/md_cg/mdcos.py +263 -21
- package/md_cg/metacognition.py +66 -13
- package/md_cg/mutate_extract_shipped.py +126 -0
- package/md_cg/mutate_ledger_e2e.py +100 -0
- package/md_cg/nodefile.py +147 -23
- package/md_cg/op_table.py +22 -4
- package/md_cg/probe_c_ledger_integrity.py +258 -0
- package/md_cg/probe_d_retirement_discipline.py +263 -0
- package/md_cg/reconcile.py +9 -11
- package/md_cg/review_cli.py +6 -1
- package/md_cg/rollback.py +16 -1
- package/md_cg/run_tests.py +30 -1
- package/md_cg/scopeselect.py +214 -0
- package/md_cg/scrub.py +160 -23
- package/md_cg/security.py +4 -2
- package/md_cg/state_extract.py +149 -0
- package/md_cg/state_slots.py +58 -10
- package/md_cg/structboost.py +179 -0
- package/md_cg/test_alternatives_historical.py +162 -0
- package/md_cg/test_b2_sensitivity_landing.py +226 -223
- package/md_cg/test_b3_merge_keeps_content.py +608 -583
- package/md_cg/test_bucket_cards.py +85 -0
- package/md_cg/test_bucket_day_order.py +74 -0
- package/md_cg/test_bucket_quota.py +79 -0
- package/md_cg/test_bucket_recency_guards.py +75 -0
- package/md_cg/test_bucket_three_causes.py +83 -0
- package/md_cg/test_bucketindex.py +130 -0
- package/md_cg/test_bucketscope_early.py +71 -0
- package/md_cg/test_cond_term_fold.py +183 -0
- package/md_cg/test_cond_term_fold_gate.py +1210 -0
- package/md_cg/test_cond_term_neg_scope.py +147 -0
- package/md_cg/test_cond_terms_colon.py +348 -0
- package/md_cg/test_detrerank.py +157 -0
- package/md_cg/test_gate_four_kpi.py +136 -0
- package/md_cg/test_generation_guard.py +76 -43
- package/md_cg/test_issue192_move_layer_indices_guard.py +48 -0
- package/md_cg/test_issue362_neg_condition.py +44 -7
- package/md_cg/test_issue39_utf8_stdio.py +572 -572
- package/md_cg/test_issue413_preimage_failclosed.py +147 -9
- package/md_cg/test_issue444_scrub_evidence_plaintext.py +14 -8
- package/md_cg/test_issue677_flush_batch_io.py +139 -0
- package/md_cg/test_issue677_info_fast.py +82 -0
- package/md_cg/test_issue677_remember_dedupe.py +106 -0
- package/md_cg/test_issue712_tokens_sessions_read.py +199 -0
- package/md_cg/test_issue722_strict_conditions.py +96 -0
- package/md_cg/test_issue730_narrowing_audit.py +121 -0
- package/md_cg/test_issue733_verify_override.py +193 -0
- package/md_cg/test_issue733_verify_override_and_overwrite_inherit.py +315 -0
- package/md_cg/test_issue740_filelock_fd_leak.py +218 -0
- package/md_cg/test_issue741_content_hash_same_source.py +158 -0
- package/md_cg/test_issue747_slot_contradiction.py +125 -0
- package/md_cg/test_issue754_nested_root_guard.py +197 -0
- package/md_cg/test_issue757_index_hash_compare.py +85 -0
- package/md_cg/test_issue758_disk_hash_single.py +67 -0
- package/md_cg/test_issue759_twophase_hash.py +67 -0
- package/md_cg/test_issue768_overwrite_time_window_anchor.py +135 -0
- package/md_cg/test_issue789_log_plaintext_probe.py +100 -0
- package/md_cg/test_issue790_verb_negation.py +77 -0
- package/md_cg/test_issue810_root_guard.py +71 -0
- package/md_cg/test_issue812_info_index_stale.py +78 -0
- package/md_cg/test_issue827_weaken_dedup.py +361 -0
- package/md_cg/test_issue829_calibration_positive_gate.py +473 -0
- package/md_cg/test_kernel_iface_contracts.py +18 -0
- package/md_cg/test_md_cg_no_hive_dep.py +238 -17
- package/md_cg/test_n204_n205_n226_n227_n228_n229_exit_gates.py +545 -542
- package/md_cg/test_n209_verify_write_face_gates.py +5 -2
- package/md_cg/test_neg_condition_hits.py +336 -333
- package/md_cg/test_probe_cd_shipped.py +385 -0
- package/md_cg/test_recent_tail.py +93 -0
- package/md_cg/test_run_tests_entry_gates.py +110 -0
- package/md_cg/test_scopeselect.py +70 -0
- package/md_cg/test_scopeselect_clock.py +70 -0
- package/md_cg/test_scrub_source_line_false_positive.py +64 -0
- package/md_cg/test_session_attribution_consistency.py +835 -813
- package/md_cg/test_state_extract_shipped.py +162 -0
- package/md_cg/test_state_ledger_chain.py +207 -0
- package/md_cg/test_state_ledger_e2e.py +145 -0
- package/md_cg/test_structboost.py +102 -0
- package/md_cg/test_timeboost.py +83 -0
- package/md_cg/test_timeboost_relativedate.py +71 -0
- package/md_cg/test_topicagg.py +129 -0
- package/md_cg/test_twophase.py +286 -286
- package/md_cg/timeboost.py +223 -0
- package/md_cg/tokens.py +55 -7
- package/md_cg/topicagg.py +142 -0
- package/md_cg/twophase.py +15 -6
- package/md_cg/whitebox_kb/wisdom/code_compose.py +33 -5
- package/md_cg/whitebox_kb/wisdom/compose_engine.py +5 -1
- package/md_cg/whitebox_kb/wisdom/role_compose.py +271 -268
- package/md_cg/writepipe.py +35 -3
- package/package.json +4 -1
- package/scripts/workspace_index.py +2 -0
- package/src/hooks.ts +182 -34
- package/src/index.ts +19 -3
- package/src/lib/console_web.mjs +30 -3
- package/src/lib/hook_audit.ts +15 -5
- package/src/lib/mdcg_client.ts +8 -4
- package/src/lib/prompt_safety.ts +73 -1
- package/src/lib/token_store.ts +5 -2
- package/src/tools.ts +5 -3
- package/ui/minimal/integrity.json +2 -2
- package/ui/minimal/payload/build-ui.mjs +41 -2
- package/ui/minimal/payload/lib/client.js +29 -3
- package/ui/minimal/payload/lib/lib/console_web.mjs +30 -3
- package/ui/minimal/payload/src/lib/console_web.mjs +30 -3
- package/ui/minimal/payload/web/app.js +3 -1
- package/ui/minimal/payload/web/client.js +30 -3
- package/web/app.js +3 -1
- package/web/client.js +30 -3
- package/zcode/AGENTS.md +1 -1
|
@@ -18,12 +18,14 @@ override 的价值来自「可回滚」,拍不出前像就不该放行。
|
|
|
18
18
|
"""
|
|
19
19
|
from __future__ import annotations
|
|
20
20
|
|
|
21
|
+
import atexit
|
|
21
22
|
import io
|
|
22
23
|
import json
|
|
23
24
|
import os
|
|
24
25
|
import re
|
|
25
26
|
import shutil
|
|
26
27
|
import sys
|
|
28
|
+
import tempfile
|
|
27
29
|
import time
|
|
28
30
|
|
|
29
31
|
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
|
|
@@ -32,10 +34,41 @@ sys.path.insert(0, _REPO)
|
|
|
32
34
|
|
|
33
35
|
# 沙箱用 gettempdir + makedirs(**不用 mkdtemp**):受限宿主里 mkdtemp 目录带
|
|
34
36
|
# 「拒绝再建子目录」的 ACL,会让本守卫整批 PermissionError(与代码无关)。
|
|
35
|
-
|
|
37
|
+
#
|
|
38
|
+
# 两条硬约束(2026-11-08 修,实测泄漏形态):
|
|
39
|
+
#
|
|
40
|
+
# ① **锚点必须在系统临时目录,不得在仓根**。原先锚在 `dirname(dirname(__file__))`,
|
|
41
|
+
# 即仓根,于是每跑一次就在仓根留一个 `_i413b_sandbox_<pid>_<ts>/`,`git status`
|
|
42
|
+
# 每次多一条噪声且条数随跑测次数线性增长(实测 4 小时从 1 个涨到 16 个)。
|
|
43
|
+
#
|
|
44
|
+
# ② **模块顶层不得做文件系统写**,只能算路径。残留的主因不是「忘了删」——
|
|
45
|
+
# `main()` 里确有收尾 rmtree;真因是 `-m` 跑法下**本模块被建两次**:runpy 先
|
|
46
|
+
# 以真名 import(触发模块级 makedirs),再以 `__main__` 重新执行一遍(见 runpy
|
|
47
|
+
# 的 `RuntimeWarning: ... found in sys.modules after import of package`)。
|
|
48
|
+
# 两份 `_SB` 的 pid/时间戳不同,第一份从没有 `main()` 收尾 ⇒ 每跑必留一个。
|
|
49
|
+
# 把建目录推迟到 `main()` 内的 `_ensure_sb()` 后,死掉的那份 import 只算了个
|
|
50
|
+
# 字符串,不落盘,泄漏归零(Z1 断言钉住「导入本模块后仓根与临时目录都不得新增」)。
|
|
51
|
+
#
|
|
52
|
+
# ③ **删目录前必须先把实例摘出 `_LIVE_CGS`**。`mdcg` 有进程退出兜底:
|
|
53
|
+
# `atexit` → `_atexit_flush_all()` → 对每个仍存活实例 `close()` → `rebuild_index()`
|
|
54
|
+
# → `atomic_write()` → `os.makedirs()`。所以「main() 删目录 → 解释器 atexit →
|
|
55
|
+
# close() 把 lib_a_1/… 原样建回来」,沙箱在盘上**复活**(实测复活 6 个文件)。
|
|
56
|
+
# 删树不撤登记 = 白删。见 `_purge_and_rm()`,Z3 用**独立子进程**钉这条
|
|
57
|
+
# (在进程内断言看不见 atexit 之后的事,必须跨进程看)。
|
|
58
|
+
_SB = os.path.join(tempfile.gettempdir(),
|
|
36
59
|
"_i413b_sandbox_%d_%d" % (os.getpid(), int(time.time() * 1000)))
|
|
37
|
-
|
|
38
|
-
|
|
60
|
+
_SB_READY = [False]
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _ensure_sb():
|
|
64
|
+
"""首次使用时才建沙箱目录(幂等)。模块顶层刻意不调它——见上面约束②。"""
|
|
65
|
+
if not _SB_READY[0]:
|
|
66
|
+
shutil.rmtree(_SB, ignore_errors=True)
|
|
67
|
+
os.makedirs(_SB, exist_ok=True)
|
|
68
|
+
_SB_READY[0] = True
|
|
69
|
+
return _SB
|
|
70
|
+
|
|
71
|
+
|
|
39
72
|
os.environ.update({"MDCG_ROOT": os.path.join(_SB, "brain"),
|
|
40
73
|
"MDCG_STATE_ROOT": os.path.join(_SB, "state"),
|
|
41
74
|
"MDCG_AUX_ROOT": os.path.join(_SB, "auxstore"),
|
|
@@ -69,6 +102,7 @@ def _princ(role, clr, sess):
|
|
|
69
102
|
|
|
70
103
|
def _fresh(tag):
|
|
71
104
|
"""新库:A 写 self 层 private 节点 V0;返回 root。(tag 自动加序号:R 腿会二次调用)"""
|
|
105
|
+
_ensure_sb()
|
|
72
106
|
_N[0] += 1
|
|
73
107
|
root = os.path.join(_SB, "lib_%s_%d" % (tag, _N[0]))
|
|
74
108
|
os.environ["MDCG_ROOT"] = root
|
|
@@ -218,20 +252,124 @@ def leg_R():
|
|
|
218
252
|
exec(compile(orig, p, "exec"), protect.__dict__) # 恢复原文
|
|
219
253
|
|
|
220
254
|
|
|
255
|
+
def _purge_and_rm():
|
|
256
|
+
"""先摘登记,再删沙箱——**顺序是关键**(2026-11-08 修,实测泄漏链)。
|
|
257
|
+
|
|
258
|
+
`mdcg` 模块级有 `_LIVE_CGS = weakref.WeakSet()`:每个 `MdCGSecure` 实例在
|
|
259
|
+
`__init__` 里登记,进程退出时 `_atexit_flush_all()` 对**仍存活的每个**实例调
|
|
260
|
+
`close()`,而 `close()` 会 `rebuild_index()` → `atomic_write()` → `os.makedirs()`。
|
|
261
|
+
于是「main() 删目录 → 解释器跑 atexit → close() 把 lib_a_1/… 原样建回来」,
|
|
262
|
+
沙箱在磁盘上**复活**(实测复活 6 个文件)。这正是「明明有 rmtree 却每跑必留一个」
|
|
263
|
+
的最后一环:删目录只是把树砍了,登记还在,atexit 又把它栽回去。
|
|
264
|
+
|
|
265
|
+
故删之前必须把本沙箱的实例从 `_LIVE_CGS` 里摘掉,让 atexit 无从复活它。
|
|
266
|
+
"""
|
|
267
|
+
from md_cg import mdcg as _M
|
|
268
|
+
sb_norm = os.path.normcase(os.path.abspath(_SB))
|
|
269
|
+
for cg in list(getattr(_M, "_LIVE_CGS", ())):
|
|
270
|
+
try:
|
|
271
|
+
r = os.path.normcase(os.path.abspath(getattr(cg, "root", "") or ""))
|
|
272
|
+
except Exception: # noqa: BLE001
|
|
273
|
+
continue
|
|
274
|
+
if r == sb_norm or r.startswith(sb_norm + os.sep):
|
|
275
|
+
_M._LIVE_CGS.discard(cg)
|
|
276
|
+
shutil.rmtree(_SB, ignore_errors=True)
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def _z3_child():
|
|
280
|
+
"""Z3 的子进程:跑一遍 legs + 收尾,然后**正常退出**(让 atexit 钩子有机会复活沙箱)。
|
|
281
|
+
|
|
282
|
+
父进程看不到 atexit 之后的事,所以这条判据必须跨进程:子进程退出码不能说明
|
|
283
|
+
问题(它照样 rc=0),只有「子进程退干净后,它声明的沙箱还在不在盘上」才作数。
|
|
284
|
+
输出一行 `SANDBOX=<path>` 给父进程核对。
|
|
285
|
+
"""
|
|
286
|
+
import contextlib
|
|
287
|
+
mute = contextlib.redirect_stdout(io.StringIO())
|
|
288
|
+
try:
|
|
289
|
+
with mute:
|
|
290
|
+
leg_A()
|
|
291
|
+
leg_C()
|
|
292
|
+
finally:
|
|
293
|
+
_purge_and_rm()
|
|
294
|
+
print("SANDBOX=" + _SB)
|
|
295
|
+
return 0
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
def _z1_child():
|
|
299
|
+
"""Z1 的子进程:**只 import、不跑 main**,随后正常退出。打印它看到的沙箱路径。
|
|
300
|
+
|
|
301
|
+
这就是 `-m` 跑法里那份「被丢弃的 import」的等价形态:真名 import 触发模块级
|
|
302
|
+
代码,若顶层建目录,此刻就落一个;`runpy` 随后以 `__main__` 再跑一遍并另建
|
|
303
|
+
一个,前一个没人收尾 ⇒ 每跑一次必泄漏一个。父进程据此检查该路径是否落在盘上。
|
|
304
|
+
"""
|
|
305
|
+
print("SANDBOX=" + _SB)
|
|
306
|
+
print("READY=%d" % (1 if _SB_READY[0] else 0))
|
|
307
|
+
print("EXISTS=%d" % (1 if os.path.isdir(_SB) else 0))
|
|
308
|
+
return 0
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
def _leg_z():
|
|
312
|
+
"""Z 组:沙箱卫生。判据全部**跨进程**——进程内看不到 import 期与 atexit 后的事。"""
|
|
313
|
+
import subprocess
|
|
314
|
+
|
|
315
|
+
def _child(flag):
|
|
316
|
+
r = subprocess.run([sys.executable, "-X", "utf8", "-m",
|
|
317
|
+
"md_cg.test_issue413_preimage_failclosed", flag],
|
|
318
|
+
cwd=_REPO, capture_output=True, text=True,
|
|
319
|
+
encoding="utf-8", errors="replace", timeout=300)
|
|
320
|
+
kv = {}
|
|
321
|
+
for ln in (r.stdout or "").splitlines():
|
|
322
|
+
if "=" in ln and ln.split("=", 1)[0] in ("SANDBOX", "READY", "EXISTS"):
|
|
323
|
+
k, v = ln.split("=", 1)
|
|
324
|
+
kv[k] = v.strip()
|
|
325
|
+
kv["rc"] = r.returncode
|
|
326
|
+
return kv
|
|
327
|
+
|
|
328
|
+
# Z1:**import 期零副作用**。子进程只 import 本模块、不跑 main,然后退出;
|
|
329
|
+
# 若模块顶层就 makedirs,那个目录此刻一定在盘上(且没人会删它)。
|
|
330
|
+
z1 = _child("--z1-child")
|
|
331
|
+
z1_sb = z1.get("SANDBOX", "")
|
|
332
|
+
_check("Z1 只 import 不 main ⇒ 沙箱不得落盘(导入期零副作用)",
|
|
333
|
+
bool(z1_sb) and not os.path.exists(z1_sb),
|
|
334
|
+
"child_rc=%s sb=%s exists=%s" % (z1.get("rc"), z1_sb or "?",
|
|
335
|
+
os.path.exists(z1_sb) if z1_sb else "?"))
|
|
336
|
+
# Z3:**收尾之后盘上真没了**。子进程跑完整腿再正常退出,让 `_atexit_flush_all`
|
|
337
|
+
# 有机会把 lib_*/ 建回来;它照样 rc=0,只有目录是否复活能判。
|
|
338
|
+
z3 = _child("--z3-child")
|
|
339
|
+
z3_sb = z3.get("SANDBOX", "")
|
|
340
|
+
_check("Z3 子进程正常退出后其沙箱未复活(atexit 不得重建)",
|
|
341
|
+
bool(z3_sb) and not os.path.exists(z3_sb),
|
|
342
|
+
"child_rc=%s sb=%s exists=%s" % (z3.get("rc"), z3_sb or "?",
|
|
343
|
+
os.path.exists(z3_sb) if z3_sb else "?"))
|
|
344
|
+
|
|
345
|
+
|
|
221
346
|
def main():
|
|
347
|
+
if "--z1-child" in sys.argv:
|
|
348
|
+
return _z1_child()
|
|
349
|
+
if "--z3-child" in sys.argv:
|
|
350
|
+
return _z3_child()
|
|
222
351
|
mutate = "--mutate" in sys.argv
|
|
223
352
|
print("=" * 72)
|
|
224
353
|
print("md 认知图 · issue #413 站点2 守卫 · override 的前像 fail-closed")
|
|
225
354
|
print("沙箱 =", _SB, "| mutate =", mutate)
|
|
226
355
|
print("=" * 72)
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
356
|
+
# Z0:锚点不得落在仓根下(2026-11-08 实测泄漏形态:每跑一次在仓根留一个
|
|
357
|
+
# `_i413b_sandbox_*/`,`git status` 噪声随跑测次数线性增长)。锚点改回仓根即转红。
|
|
358
|
+
_repo_norm = os.path.normcase(os.path.abspath(_REPO)) + os.sep
|
|
359
|
+
_sb_norm = os.path.normcase(os.path.abspath(_SB))
|
|
360
|
+
_check("Z0 沙箱不在仓根下(仓根只许放源码,不放运行态)",
|
|
361
|
+
not _sb_norm.startswith(_repo_norm), _SB)
|
|
362
|
+
try:
|
|
363
|
+
leg_A()
|
|
364
|
+
leg_B()
|
|
365
|
+
leg_C()
|
|
366
|
+
if mutate:
|
|
367
|
+
leg_R()
|
|
368
|
+
finally:
|
|
369
|
+
_purge_and_rm()
|
|
370
|
+
_leg_z()
|
|
232
371
|
print("\n" + ("全部 PASS(%d 项)" % _PASSED[0] if not FAILS
|
|
233
372
|
else "FAIL %d 项:%s" % (len(FAILS), FAILS)))
|
|
234
|
-
shutil.rmtree(_SB, ignore_errors=True)
|
|
235
373
|
return 1 if FAILS else 0
|
|
236
374
|
|
|
237
375
|
|
|
@@ -137,12 +137,19 @@ def _src(rel):
|
|
|
137
137
|
|
|
138
138
|
|
|
139
139
|
def _contra_src():
|
|
140
|
-
|
|
140
|
+
"""证据串生成面的源码段:`_contradiction` + 槽位支 `_slot_contradiction`(#747)。
|
|
141
|
+
|
|
142
|
+
#747 把极性支搬到 `_slot_contradiction` 后,只扫 `_contradiction` 会让 E1b 失去覆盖面;
|
|
143
|
+
故扫描**证据串可能出现的全部函数**(新增分支也要加进来)。
|
|
144
|
+
"""
|
|
145
|
+
src = _src("md_cg/scrub.py")
|
|
146
|
+
tree = ast.parse(src, "scrub.py")
|
|
147
|
+
segs = []
|
|
141
148
|
for node in ast.walk(tree):
|
|
142
|
-
if isinstance(node, ast.FunctionDef) and node.name
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
return ""
|
|
149
|
+
if isinstance(node, ast.FunctionDef) and node.name in (
|
|
150
|
+
"_contradiction", "_slot_contradiction"):
|
|
151
|
+
segs.append(ast.get_source_segment(src, node) or "")
|
|
152
|
+
return "\n".join(segs)
|
|
146
153
|
|
|
147
154
|
|
|
148
155
|
def group_e(tmp):
|
|
@@ -185,9 +192,8 @@ _MUTATIONS = (
|
|
|
185
192
|
("A2", "A3", "A4", "A5", "A6")),
|
|
186
193
|
("M2 极性支改回共享词面",
|
|
187
194
|
"md_cg/scrub.py",
|
|
188
|
-
((
|
|
189
|
-
|
|
190
|
-
" \"why\": \"极性相反且共享内容:\" + \"、\".join(sorted(shared)[:5])}"),),
|
|
195
|
+
(('f"(占比 {ratio:.2f}),词面不入证据串")',
|
|
196
|
+
'f"(占比 {ratio:.2f}),词面不入证据串" + "、".join(sorted(shared)[:5]))'),),
|
|
191
197
|
("E1b",)),
|
|
192
198
|
)
|
|
193
199
|
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""issue #677(C 根治):`_flush_batch` 的锁内 IO 从「每条一次 flush」降为「每批一次」。
|
|
3
|
+
|
|
4
|
+
## 病灶
|
|
5
|
+
`_flush_batch` 在 `FileLock(index_path)` 内逐条 `ShardedLog.append`,而 `append` **每条都 flush**
|
|
6
|
+
⇒ 锁内 IO 次数 = 脏集条数。大库一批上千条 = 上千次 flush,MCP stdio 串行时后续请求全排队,
|
|
7
|
+
客户端 60s 超时(「写成功但没回话」)。
|
|
8
|
+
|
|
9
|
+
## 修法
|
|
10
|
+
`ShardedLog.append_many`:**一次 write + 一次 flush**(批量编码、单次落盘);
|
|
11
|
+
`_flush_batch` 用它,整批仍在同一把 FileLock 内 ⇒ 与 compact 的 read→clear 互斥语义不变。
|
|
12
|
+
|
|
13
|
+
## 断言
|
|
14
|
+
- B1 一批 300 条:`append` 调用 **0** 次、`append_many` 至少 1 次、总条数 300;
|
|
15
|
+
- B2 重开库后 300 条都在(日志重放正确);
|
|
16
|
+
- B3 单条 flush 仍工作(走 append_many,条数 1)。
|
|
17
|
+
|
|
18
|
+
运行:python -X utf8 -m md_cg.test_issue677_flush_batch_io
|
|
19
|
+
"""
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import os
|
|
23
|
+
import shutil
|
|
24
|
+
import sys
|
|
25
|
+
import tempfile
|
|
26
|
+
|
|
27
|
+
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
|
|
28
|
+
|
|
29
|
+
_REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
30
|
+
sys.path.insert(0, _REPO)
|
|
31
|
+
|
|
32
|
+
from md_cg import fsutil # noqa: E402
|
|
33
|
+
from md_cg.mdcg import MdCG # noqa: E402
|
|
34
|
+
|
|
35
|
+
FAILS = []
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def ok(cond, name, detail=""):
|
|
39
|
+
if cond:
|
|
40
|
+
print(" PASS " + name)
|
|
41
|
+
else:
|
|
42
|
+
print(" FAIL " + name + " " + str(detail)[:300])
|
|
43
|
+
FAILS.append(name)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
BODY = "# 功能名:t%d\n# 生效条件:x\n# 子功能:y\n# 执行:z\n# 验证方式:test\n# 不适用条件:无\n\nbody %d"
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def main():
|
|
50
|
+
print("=" * 68)
|
|
51
|
+
print("issue #677 C · flush 锁内批量 IO")
|
|
52
|
+
print("=" * 68)
|
|
53
|
+
counts = {"append": 0, "many": 0, "many_n": 0}
|
|
54
|
+
orig_append = fsutil.ShardedLog.append
|
|
55
|
+
orig_many = fsutil.ShardedLog.append_many
|
|
56
|
+
|
|
57
|
+
def _a1(self, record):
|
|
58
|
+
counts["append"] += 1
|
|
59
|
+
return orig_append(self, record)
|
|
60
|
+
|
|
61
|
+
def _a2(self, records):
|
|
62
|
+
n = orig_many(self, records)
|
|
63
|
+
counts["many"] += 1
|
|
64
|
+
counts["many_n"] += n
|
|
65
|
+
return n
|
|
66
|
+
|
|
67
|
+
fsutil.ShardedLog.append = _a1
|
|
68
|
+
fsutil.ShardedLog.append_many = _a2
|
|
69
|
+
tmp = tempfile.mkdtemp(prefix="iss677c_")
|
|
70
|
+
try:
|
|
71
|
+
cg = MdCG(os.path.join(tmp, "r"), autoflush=10 ** 9)
|
|
72
|
+
for i in range(300):
|
|
73
|
+
cg.add("n%03d" % i, BODY % (i, i), layer="contextual")
|
|
74
|
+
cg.flush()
|
|
75
|
+
ok(counts["append"] == 0, "B1a 锁内不再逐条 append", counts)
|
|
76
|
+
ok(counts["many"] >= 1 and counts["many_n"] == 300, "B1b 批量一次写盘、条数 300", counts)
|
|
77
|
+
cg.close()
|
|
78
|
+
cg2 = MdCG(os.path.join(tmp, "r"))
|
|
79
|
+
ok(len(cg2.index.get("nodes") or {}) == 300, "B2 重开后 300 条都在(重放正确)",
|
|
80
|
+
len(cg2.index.get("nodes") or {}))
|
|
81
|
+
cg2.close()
|
|
82
|
+
counts.update({"append": 0, "many": 0, "many_n": 0})
|
|
83
|
+
cg3 = MdCG(os.path.join(tmp, "r2"), autoflush=10 ** 9)
|
|
84
|
+
cg3.add("one", BODY % (0, 0), layer="contextual")
|
|
85
|
+
cg3.flush()
|
|
86
|
+
ok(counts["many_n"] == 1, "B3 单条 flush 仍工作", counts)
|
|
87
|
+
cg3.close()
|
|
88
|
+
# B4(#677 C 复核 P1):残尾修补——先造「无换行结尾」的半行,再 append_many;
|
|
89
|
+
# 不补换行时新记录会与残尾粘成一行(解析失败、残尾与新记录一起丢)。
|
|
90
|
+
_d = os.path.join(tmp, "shard_torn")
|
|
91
|
+
os.makedirs(_d, exist_ok=True)
|
|
92
|
+
_log = fsutil.ShardedLog(_d)
|
|
93
|
+
with open(_log.path, "a", encoding="utf-8", newline="\n") as _f:
|
|
94
|
+
_f.write('{"id":"torn"')
|
|
95
|
+
_log.append_many([{"id": "good", "e": {"layer": "contextual"}}])
|
|
96
|
+
_log.close()
|
|
97
|
+
_recs = fsutil.read_jsonl(_log.path)
|
|
98
|
+
ok([r.get("id") for r in _recs] == ["good"],
|
|
99
|
+
"B4 残尾修补:新记录不被粘行吞掉", [r.get("id") for r in _recs])
|
|
100
|
+
# B5(#677 C 复核 P2):失败回填必须保留 superseded 标记——
|
|
101
|
+
# 否则下一轮把「已判陈旧」的记录当新记录写进分片(旧盖新 = 内容错误)。
|
|
102
|
+
cg4 = MdCG(os.path.join(tmp, "r3"), autoflush=10 ** 9)
|
|
103
|
+
cg4.add("s0", BODY % (0, 0), layer="contextual")
|
|
104
|
+
cg4.add("s1", BODY % (1, 1), layer="contextual")
|
|
105
|
+
cg4._dirty.superseded_ids.add("s0")
|
|
106
|
+
_boom = {"on": True}
|
|
107
|
+
|
|
108
|
+
def _raise(self, records):
|
|
109
|
+
if _boom["on"]:
|
|
110
|
+
_boom["on"] = False
|
|
111
|
+
raise OSError("disk full (simulated)")
|
|
112
|
+
return orig_many(self, records)
|
|
113
|
+
|
|
114
|
+
fsutil.ShardedLog.append_many = _raise
|
|
115
|
+
try:
|
|
116
|
+
cg4.flush()
|
|
117
|
+
except OSError:
|
|
118
|
+
pass
|
|
119
|
+
fsutil.ShardedLog.append_many = _a2
|
|
120
|
+
cg4.flush()
|
|
121
|
+
_ids = []
|
|
122
|
+
for _root, _dn, _fns in os.walk(cg4.index_log_dir):
|
|
123
|
+
for _fn in _fns:
|
|
124
|
+
if _fn.endswith(".log"):
|
|
125
|
+
_ids += [r.get("id") for r in fsutil.read_jsonl(os.path.join(_root, _fn))]
|
|
126
|
+
ok("s0" not in _ids and "s1" in _ids,
|
|
127
|
+
"B5 失败回填保留 superseded(陈旧记录不落盘)", _ids)
|
|
128
|
+
cg4.close()
|
|
129
|
+
finally:
|
|
130
|
+
fsutil.ShardedLog.append = orig_append
|
|
131
|
+
fsutil.ShardedLog.append_many = orig_many
|
|
132
|
+
shutil.rmtree(tmp, ignore_errors=True)
|
|
133
|
+
print()
|
|
134
|
+
print("issue #677 C:%d 项失败" % len(FAILS) if FAILS else "issue #677 C:全部通过")
|
|
135
|
+
return 1 if FAILS else 0
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
if __name__ == "__main__":
|
|
139
|
+
sys.exit(main())
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""issue #677(B):`cg(op=info)` 拆快/慢——诊断面不得因大库拖到 60s。
|
|
3
|
+
|
|
4
|
+
病灶:info 原先一次性跑 health_os + 理论/链接/策略/代校验;MCP stdio 串行 ⇒ 慢 info 挡住后续
|
|
5
|
+
请求(含写入应答),客户端 60s 超时。修法:缺省只回内存态快照;全量诊断走 full=true。
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import json
|
|
10
|
+
import os
|
|
11
|
+
import shutil
|
|
12
|
+
import sys
|
|
13
|
+
import tempfile
|
|
14
|
+
|
|
15
|
+
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
|
|
16
|
+
|
|
17
|
+
_REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
18
|
+
sys.path.insert(0, _REPO)
|
|
19
|
+
|
|
20
|
+
from md_cg import mcp_server # noqa: E402
|
|
21
|
+
from md_cg.mdcos import MdCGOS # noqa: E402
|
|
22
|
+
|
|
23
|
+
FAILS = []
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def ok(cond, name, detail=""):
|
|
27
|
+
if cond:
|
|
28
|
+
print(" PASS " + name)
|
|
29
|
+
else:
|
|
30
|
+
print(" FAIL " + name + " " + str(detail)[:300])
|
|
31
|
+
FAILS.append(name)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _call(cg, **a):
|
|
35
|
+
r = mcp_server.call_tool(cg, "cg", a)
|
|
36
|
+
if isinstance(r, dict) and "content" in r:
|
|
37
|
+
return json.loads(r["content"][0]["text"])
|
|
38
|
+
return r
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
BODY = "# 功能名:t\n# 生效条件:x\n# 子功能:y\n# 执行:z\n# 验证方式:test\n# 不适用条件:无\n\nbody"
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def main():
|
|
45
|
+
print("=" * 68)
|
|
46
|
+
print("issue #677 B · op=info 快/慢拆分")
|
|
47
|
+
print("=" * 68)
|
|
48
|
+
tmp = tempfile.mkdtemp(prefix="iss677b_")
|
|
49
|
+
cg = MdCGOS(os.path.join(tmp, "r"), autoflush=1)
|
|
50
|
+
try:
|
|
51
|
+
cg.add("n1", BODY, layer="knowledge")
|
|
52
|
+
cg.flush()
|
|
53
|
+
_orig = cg.health_os
|
|
54
|
+
|
|
55
|
+
def _boom(*_a, **_k):
|
|
56
|
+
raise AssertionError("快照模式不得调用 health_os")
|
|
57
|
+
|
|
58
|
+
cg.health_os = _boom
|
|
59
|
+
fast = _call(cg, op="info")
|
|
60
|
+
ok(isinstance(fast, dict) and fast.get("full") is False, "I1 缺省 full=false", fast)
|
|
61
|
+
ok("flush_state" in fast and "node_count" in fast, "I1b 含内存态字段", list(fast)[:8])
|
|
62
|
+
ok("os" not in fast, "I3 缺省不含慢面(health_os 的 os 段)", list(fast)[:12])
|
|
63
|
+
ok("theory" in fast and "write_policy" in fast,
|
|
64
|
+
"I3b 既有诊断键仍在(兼容)", [k for k in fast if k in ("theory", "write_policy")])
|
|
65
|
+
cg.health_os = lambda *a, **k: {"health_marker": 1}
|
|
66
|
+
full = _call(cg, op="info", full=True)
|
|
67
|
+
ok(isinstance(full, dict) and full.get("full") is True, "I2 full=true", full)
|
|
68
|
+
ok(full.get("health_marker") == 1, "I2b 慢面被调用并并入", full.get("health_marker"))
|
|
69
|
+
cg.health_os = _orig
|
|
70
|
+
finally:
|
|
71
|
+
try:
|
|
72
|
+
cg.close()
|
|
73
|
+
except Exception:
|
|
74
|
+
pass
|
|
75
|
+
shutil.rmtree(tmp, ignore_errors=True)
|
|
76
|
+
print()
|
|
77
|
+
print("issue #677 B:%d 项失败" % len(FAILS) if FAILS else "issue #677 B:全部通过")
|
|
78
|
+
return 1 if FAILS else 0
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
if __name__ == "__main__":
|
|
82
|
+
sys.exit(main())
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""issue #677(A):自动记忆面幂等——`mdcg_remember(dedupe=true)`。
|
|
3
|
+
|
|
4
|
+
## 病灶
|
|
5
|
+
写入是两级提交(节点文件 + 内存索引先落,应答在后),客户端 60s 超时衡量的是「收到应答」。
|
|
6
|
+
超时后重试若不自幂等,就会新增一条同内容节点(实测三条,importance 0.86)。
|
|
7
|
+
|
|
8
|
+
## 修法
|
|
9
|
+
`mdcg_remember` 走与 `cg(op=write)` **同一个闸**(`writepipe._gate_dedupe`:同层 + 内容指纹 ⇒ 复用既有 id),
|
|
10
|
+
位置在链首(未落盘、未建边、不过闸审计)。缺省关 —— 既有调用方逐位不变。
|
|
11
|
+
|
|
12
|
+
## 断言
|
|
13
|
+
- R1 首次写入:落盘、deduped 不存在;
|
|
14
|
+
- R2 同内容再写 + dedupe=true:**不新增**、复用同一 id、deduped=True;
|
|
15
|
+
- R3 同内容**不带** dedupe:新增一条(opt-in 语义,既有行为不变);
|
|
16
|
+
- R4 跨层同内容 + dedupe:不短路(跨层是不同语义)。
|
|
17
|
+
|
|
18
|
+
运行:python -X utf8 -m md_cg.test_issue677_remember_dedupe
|
|
19
|
+
"""
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import os
|
|
23
|
+
import shutil
|
|
24
|
+
import sys
|
|
25
|
+
import tempfile
|
|
26
|
+
|
|
27
|
+
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
|
|
28
|
+
|
|
29
|
+
_REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
30
|
+
sys.path.insert(0, _REPO)
|
|
31
|
+
|
|
32
|
+
from md_cg import mcp_server # noqa: E402
|
|
33
|
+
from md_cg.mdcos import MdCGOS # noqa: E402
|
|
34
|
+
|
|
35
|
+
FAILS = []
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def ok(cond, name, detail=""):
|
|
39
|
+
if cond:
|
|
40
|
+
print(" PASS " + name)
|
|
41
|
+
else:
|
|
42
|
+
print(" FAIL " + name + " " + str(detail)[:300])
|
|
43
|
+
FAILS.append(name)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
BODY = """# 功能名:幂等写入口
|
|
47
|
+
# 生效条件:写入超时后重试时
|
|
48
|
+
# 子功能:同层内容指纹短路
|
|
49
|
+
# 执行:mdcg_remember(dedupe=true)
|
|
50
|
+
# 验证方式:节点数不增、id 相同
|
|
51
|
+
# 不适用条件:需要另存一份时"""
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _nodes(cg):
|
|
55
|
+
return len((cg.index.get("nodes") or {}))
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def main():
|
|
59
|
+
print("=" * 68)
|
|
60
|
+
print("issue #677 A · 自动记忆面幂等(mdcg_remember dedupe)")
|
|
61
|
+
print("=" * 68)
|
|
62
|
+
tmp = tempfile.mkdtemp(prefix="iss677a_")
|
|
63
|
+
cg = MdCGOS(os.path.join(tmp, "r"), autoflush=1)
|
|
64
|
+
try:
|
|
65
|
+
r1 = mcp_server.call_tool(cg, "mdcg_remember", {"content": BODY, "layer": "contextual"})
|
|
66
|
+
d1 = r1 if isinstance(r1, dict) else None
|
|
67
|
+
ok(d1 is not None and d1.get("ok") is True, "R1 首次写入成功", r1)
|
|
68
|
+
ok(not (d1 or {}).get("deduped"), "R1b 首次不带 deduped 标记", d1)
|
|
69
|
+
n1 = _nodes(cg)
|
|
70
|
+
r2 = mcp_server.call_tool(cg, "mdcg_remember", {"content": BODY, "layer": "contextual", "dedupe": True})
|
|
71
|
+
d2 = r2 if isinstance(r2, dict) else None
|
|
72
|
+
ok(d2 is not None and d2.get("deduped") is True, "R2 同内容 + dedupe ⇒ 短路复用", r2)
|
|
73
|
+
ok(_nodes(cg) == n1, "R2b 节点数不增", (_nodes(cg), n1))
|
|
74
|
+
ok((d2 or {}).get("id") == (d1 or {}).get("id"), "R2c 复用同一 id",
|
|
75
|
+
((d2 or {}).get("id"), (d1 or {}).get("id")))
|
|
76
|
+
r3 = mcp_server.call_tool(cg, "mdcg_remember", {"content": BODY, "layer": "contextual"})
|
|
77
|
+
ok(_nodes(cg) == n1 + 1, "R3 不带 dedupe ⇒ 新增一条(opt-in)", (_nodes(cg), n1))
|
|
78
|
+
r4 = mcp_server.call_tool(cg, "mdcg_remember", {"content": BODY, "layer": "knowledge", "dedupe": True})
|
|
79
|
+
d4 = r4 if isinstance(r4, dict) else None
|
|
80
|
+
ok(not (d4 or {}).get("deduped"), "R4 跨层同内容不短路", r4)
|
|
81
|
+
# R5(#677 @1):跨会话同内容 + dedupe ⇒ **不**短路(各写一条)。
|
|
82
|
+
# 否则两个会话写同一句会互相顶掉;叠加 sessionPrivate 时会「命中但读不到」。
|
|
83
|
+
_body = BODY + chr(10) + chr(10) + "(跨会话用例)"
|
|
84
|
+
mcp_server.call_tool(cg, "mdcg_remember",
|
|
85
|
+
{"content": _body, "layer": "contextual", "session": "sA"})
|
|
86
|
+
n_mid = _nodes(cg)
|
|
87
|
+
r5 = mcp_server.call_tool(cg, "mdcg_remember",
|
|
88
|
+
{"content": _body, "layer": "contextual",
|
|
89
|
+
"session": "sB", "dedupe": True})
|
|
90
|
+
d5 = r5 if isinstance(r5, dict) else None
|
|
91
|
+
ok(not (d5 or {}).get("deduped") and _nodes(cg) == n_mid + 1,
|
|
92
|
+
"R5 跨会话同内容不幂等(session 不同各写一条)",
|
|
93
|
+
(d5, _nodes(cg), n_mid))
|
|
94
|
+
finally:
|
|
95
|
+
try:
|
|
96
|
+
cg.close()
|
|
97
|
+
except Exception:
|
|
98
|
+
pass
|
|
99
|
+
shutil.rmtree(tmp, ignore_errors=True)
|
|
100
|
+
print()
|
|
101
|
+
print("issue #677 A:%d 项失败" % len(FAILS) if FAILS else "issue #677 A:全部通过")
|
|
102
|
+
return 1 if FAILS else 0
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
if __name__ == "__main__":
|
|
106
|
+
sys.exit(main())
|