@furongjun1999/dsh-memory 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +572 -465
- package/docs/README.md +1 -0
- package/docs/eval/DSH/346/227/245/345/277/227/347/264/242/345/274/225v2_/345/217/202/350/200/203dsh-TUI_v1.0.md +248 -0
- package/docs/eval/DSH/346/227/245/345/277/227/347/264/242/345/274/225/346/225/210/346/236/234/351/252/214/350/257/201_v1.0.md +209 -0
- package/docs/eval/DSH/347/253/257/347/274/272/351/231/267/344/270/223/351/241/271_v1.0.md +254 -0
- package/docs/eval/P1b2_/350/257/273/351/235/242/344/273/243/351/231/205/344/277/256/345/244/215_v1.0.md +124 -0
- package/docs/eval/bench_lingshu_self/bench_self.py +140 -0
- package/docs/eval/bench_lingshu_self/self_bench_result.json +404 -0
- package/docs/eval/bench_lingshu_self//347/201/265/346/236/242/350/207/252/345/272/223/347/253/257/345/210/260/347/253/257/346/243/200/347/264/242/345/256/236/346/265/213_v1.0.md +38 -0
- package/docs/eval//344/270/215/345/217/257/351/235/240/346/200/247/350/220/275/345/234/260_P0_v1.0.md +185 -0
- package/docs/eval//345/217/221/345/270/20306_/344/270/200/351/224/256/351/205/215/347/275/256/344/270/216DSH0172_v1.0.md +171 -0
- package/docs/eval//346/225/205/351/232/234/346/263/250/345/205/245/345/256/236/346/265/213_v1.0.md +422 -0
- package/docs/eval//347/253/257/345/210/260/347/253/257LoCoMoQA/345/220/214/345/217/243/345/276/204/345/257/271/347/205/247_v1.0.md +100 -0
- package/docs/eval//347/253/257/345/210/260/347/253/257/345/271/262/346/211/260/346/261/240/350/257/204/346/265/213_/347/241/256/345/256/232/346/200/247/350/243/201/345/206/263vsLLM_judge_v1.1.md +197 -0
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v1.md +156 -0
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v10.md +210 -0
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v11.md +227 -0
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v12.md +203 -0
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v13.md +233 -0
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v14.md +191 -0
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v15.md +213 -0
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v16.md +214 -0
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v17.md +199 -0
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v18.md +183 -0
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v19.md +207 -0
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v2.md +156 -0
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v3.md +152 -0
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v4.md +128 -0
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v5.md +114 -0
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v6.md +192 -0
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v7.md +187 -0
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v8.md +207 -0
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v9.md +203 -0
- package/docs/hive//345/256/211/345/205/250/345/256/241/350/256/241/345/256/236/351/224/232_v0.1.md +202 -0
- package/docs/hive//350/234/202/345/267/242/345/217/214/345/256/236/344/276/213/344/272/222/351/252/214_/350/256/276/350/256/241/345/256/232/347/250/277.md +19 -3
- package/docs/images/lingshu-moonlight-covenant-poster-preview.jpg +0 -0
- package/docs/images/lingshu-moonlight-covenant-poster.png +0 -0
- package/docs/mdcg//345/212/237/350/203/275/350/260/203/347/224/250/346/230/240/345/260/204/350/241/250_v0.1.md +40 -40
- package/docs/theory//344/270/215/345/217/257/351/235/240/345/256/232/347/220/206/344/270/216/345/244/261/346/225/210/344/274/230/345/205/210/346/241/206/346/236/266_v0.3.md +365 -0
- package/docs//344/270/215/345/217/257/351/235/240/346/200/247/347/220/206/350/256/272_v0.1.md +343 -0
- package/dsh/cordis-patch-profile-web.example.yml +35 -0
- package/lib/bridge.d.ts +9 -0
- package/lib/bridge.js +35 -0
- package/lib/cli.d.ts +3 -0
- package/lib/cli.js +66 -0
- package/lib/index.js +7 -1
- package/lib/init.d.ts +88 -0
- package/lib/init.js +287 -0
- package/lib/lib/roleplay_web.js +530 -443
- package/lib/lib/token_store.d.ts +7 -1
- package/lib/lib/token_store.js +12 -3
- package/md_cg/audit.py +12 -1
- package/md_cg/backfill.py +16 -15
- package/md_cg/backfill_bucket_zh.py +35 -0
- package/md_cg/bench_e2e_judge.py +532 -0
- package/md_cg/bench_e2e_locomo_qa.py +368 -0
- package/md_cg/bench_e2e_qa.py +256 -0
- package/md_cg/branches.py +18 -2
- package/md_cg/ccgc.py +3 -2
- package/md_cg/chain.py +19 -4
- package/md_cg/consolidate.py +7 -6
- package/md_cg/crosscheck.py +4 -3
- package/md_cg/crypto.py +439 -437
- package/md_cg/datapath.py +395 -335
- package/md_cg/evidence.py +8 -3
- package/md_cg/export.py +3 -1
- package/md_cg/forgetting.py +2 -2
- package/md_cg/fsutil.py +48 -0
- package/md_cg/hotcache.py +255 -238
- package/md_cg/interop.py +161 -22
- package/md_cg/judgment_manifest.py +177 -0
- package/md_cg/links.py +655 -622
- package/md_cg/mcp_server.py +282 -31
- package/md_cg/mdcg.py +702 -126
- package/md_cg/mdcos.py +454 -68
- package/md_cg/mreview/govern.py +5 -4
- package/md_cg/postings.py +4 -2
- package/md_cg/readcache.py +76 -18
- package/md_cg/reconcile.py +228 -0
- package/md_cg/review_cli.py +215 -0
- package/md_cg/routing.py +28 -0
- package/md_cg/run_tests.py +211 -0
- package/md_cg/scrub.py +862 -852
- package/md_cg/security.py +385 -275
- package/md_cg/selfreport.py +3 -2
- package/md_cg/signer.py +565 -562
- package/md_cg/sources.py +3 -2
- package/md_cg/stg.py +6 -0
- package/md_cg/sustain.py +1168 -1138
- package/md_cg/test_access_hints.py +147 -0
- package/md_cg/test_branch_discard_tombstone.py +136 -0
- package/md_cg/test_branches.py +259 -249
- package/md_cg/test_chain_read_isolate.py +168 -0
- package/md_cg/test_datapath_device_name.py +203 -0
- package/md_cg/test_emit_negtail_cache.py +156 -0
- package/md_cg/test_en_pipeline.py +186 -166
- package/md_cg/test_govern_directread.py +421 -0
- package/md_cg/test_i32_hotcache_env_key.py +218 -0
- package/md_cg/test_identity_attribution.py +228 -147
- package/md_cg/test_index_crossprocess_reload.py +301 -0
- package/md_cg/test_index_durability.py +238 -224
- package/md_cg/test_interop.py +4 -2
- package/md_cg/test_interop_judgment.py +228 -0
- package/md_cg/test_issue39_utf8_stdio.py +273 -0
- package/md_cg/test_links_concurrent_write.py +188 -0
- package/md_cg/test_merge_upsert.py +168 -0
- package/md_cg/test_n123_derive_expiry_chain.py +205 -0
- package/md_cg/test_n130_verify_falsified_protect.py +185 -0
- package/md_cg/test_n131_merge_gate.py +205 -0
- package/md_cg/test_none_id_write_guard.py +165 -0
- package/md_cg/test_p1x_ref_root.py +160 -0
- package/md_cg/test_p27_docindex.py +774 -765
- package/md_cg/test_p2_mcp.py +3 -0
- package/md_cg/test_p32_backfill.py +304 -298
- package/md_cg/test_p39_verify_flow.py +90 -50
- package/md_cg/test_p47_session_view.py +60 -25
- package/md_cg/test_propose_tail_index.py +157 -0
- package/md_cg/test_read_scope_b27.py +277 -0
- package/md_cg/test_readcache_default_on.py +168 -0
- package/md_cg/test_readcache_precise_inval.py +270 -0
- package/md_cg/test_readcache_prodpath.py +55 -7
- package/md_cg/test_reconcile_v0.py +294 -0
- package/md_cg/test_retr_s1.py +6 -2
- package/md_cg/test_retr_s1b.py +67 -0
- package/md_cg/test_retr_s7.py +8 -0
- package/md_cg/test_retr_s9_entity_ctx.py +181 -175
- package/md_cg/test_retr_score_once.py +208 -0
- package/md_cg/test_review_cli_attribution.py +177 -0
- package/md_cg/test_review_cli_visibility.py +235 -0
- package/md_cg/test_review_onepass.py +170 -0
- package/md_cg/test_rrf_graph_seed_cache.py +154 -0
- package/md_cg/test_security_audit.py +155 -0
- package/md_cg/test_security_audit_b26.py +161 -0
- package/md_cg/test_security_audit_v21.py +250 -0
- package/md_cg/test_semantic_canonical.py +255 -241
- package/md_cg/test_session_isolation.py +168 -0
- package/md_cg/test_snapshot_autoclose.py +187 -0
- package/md_cg/test_tail_watermark_race.py +208 -0
- package/md_cg/test_tenant_env_override_warn.py +139 -0
- package/md_cg/test_tenant_registry_corrupt_warn.py +151 -0
- package/md_cg/test_v14_fixes.py +415 -397
- package/md_cg/test_verify_dirty_reconcile.py +157 -0
- package/md_cg/theory.py +276 -273
- package/md_cg/tokens.py +734 -677
- package/md_cg/units.py +3 -2
- package/md_cg/vision_evidence.py +4 -3
- package/md_cg/whitebox_kb/wisdom/code_solidified.json +6195 -6195
- package/md_cg/whitebox_kb/wisdom/multilang_ir.py +128 -124
- package/md_cg/writepipe.py +554 -550
- package/package.json +11 -2
- package/src/bridge.ts +434 -401
- package/src/cli.ts +65 -0
- package/src/index.ts +526 -518
- package/src/init.ts +347 -0
- package/src/lib/roleplay_web.ts +1019 -932
- package/src/lib/token_store.ts +202 -192
package/md_cg/evidence.py
CHANGED
|
@@ -33,12 +33,14 @@ import time
|
|
|
33
33
|
|
|
34
34
|
from . import identity as _identity
|
|
35
35
|
from . import signer as _signer
|
|
36
|
+
from .datapath import aux_root
|
|
37
|
+
from .fsutil import publish
|
|
36
38
|
|
|
37
39
|
NODE_ID_ENV = "MDCG_NODE_ID"
|
|
38
40
|
SWARM_DIR_ENV = "MDCG_SWARM_DIR"
|
|
39
41
|
ROOT_ENV = "MDCG_ROOT"
|
|
40
42
|
|
|
41
|
-
DEFAULT_DIR =
|
|
43
|
+
DEFAULT_DIR = aux_root()
|
|
42
44
|
DEFAULT_SWARM_DIR = os.path.join(DEFAULT_DIR, "swarm")
|
|
43
45
|
|
|
44
46
|
SUBSYSTEM = "swarm" # 子系统名:智能体可对它声明签名策略(D-4)
|
|
@@ -485,7 +487,7 @@ def _write_json(path: str, obj) -> str:
|
|
|
485
487
|
os.chmod(tmp, 0o600)
|
|
486
488
|
except OSError:
|
|
487
489
|
pass
|
|
488
|
-
|
|
490
|
+
publish(tmp, path)
|
|
489
491
|
return path
|
|
490
492
|
|
|
491
493
|
|
|
@@ -504,7 +506,10 @@ def _open_cg(root: str = None):
|
|
|
504
506
|
clearance="private", can_write=True, can_admin=True,
|
|
505
507
|
role="designer", layers_allow=spec["layers_allow"],
|
|
506
508
|
ops_allow=spec["ops_allow"], auth_mode="legacy_env")
|
|
507
|
-
|
|
509
|
+
# autoflush=1(P1b,2026-09-26,与 mcp_server.py:3695-3702 / review_cli._cg
|
|
510
|
+
# 同款):短命工具进程被 kill 时 close/atexit 兜底不执行,缺省 64 会让
|
|
511
|
+
# 写入停在内存 _dirty、分片日志从未落——逐条落日志保任意退出形态跨进程可见。
|
|
512
|
+
return MdCGSecure(r, principal=p, autoflush=1)
|
|
508
513
|
|
|
509
514
|
|
|
510
515
|
# 生效条件:当传入 obj 时,打印 json.dumps(obj, ensure_ascii=False, indent=1, default=str)。
|
package/md_cg/export.py
CHANGED
|
@@ -18,6 +18,8 @@ import json
|
|
|
18
18
|
import os
|
|
19
19
|
import time
|
|
20
20
|
|
|
21
|
+
from .fsutil import publish
|
|
22
|
+
|
|
21
23
|
SCHEMA = 1
|
|
22
24
|
EXPORT_ACTIONS = ("graph", "nodes", "slice", "stat")
|
|
23
25
|
|
|
@@ -117,7 +119,7 @@ def _write_jsonl(cg, out_path: str, entries, include_content: bool = True):
|
|
|
117
119
|
by_layer[lay] = by_layer.get(lay, 0) + 1
|
|
118
120
|
if written % 500 == 0:
|
|
119
121
|
f.flush() # 定期刷盘,控制缓冲区
|
|
120
|
-
|
|
122
|
+
publish(tmp, out_path) # 流式 + 原子:要么完整、要么无
|
|
121
123
|
return {"ok": True, "out": out_path, "written": written,
|
|
122
124
|
"skipped_unreadable": skipped, "by_layer": by_layer,
|
|
123
125
|
"bytes": os.path.getsize(out_path),
|
package/md_cg/forgetting.py
CHANGED
|
@@ -48,7 +48,7 @@ import os
|
|
|
48
48
|
import time
|
|
49
49
|
|
|
50
50
|
from . import lifecycle, nodefile
|
|
51
|
-
from .fsutil import append_jsonl, atomic_write, read_jsonl
|
|
51
|
+
from .fsutil import append_jsonl, atomic_write, publish, read_jsonl
|
|
52
52
|
from .mdcg import bigrams
|
|
53
53
|
|
|
54
54
|
# ---------------------------------------------------------------- 判据常量
|
|
@@ -435,7 +435,7 @@ def longterm_assess(cg, apply=False, out=None, layer=None, keep=LONGTERM_KEEP,
|
|
|
435
435
|
"content_hash": e.get("content_hash")}
|
|
436
436
|
f.write(json.dumps(row, ensure_ascii=False) + "\n")
|
|
437
437
|
written += 1
|
|
438
|
-
|
|
438
|
+
publish(tmp, path)
|
|
439
439
|
atomic_write(current_path(cg), json.dumps(
|
|
440
440
|
{"snapshot_id": snapshot_id, "ts": time.time(), "path": rel,
|
|
441
441
|
"total": total, "tiers": tiers, "by_layer": by_layer,
|
package/md_cg/fsutil.py
CHANGED
|
@@ -41,6 +41,18 @@ def _publish(tmp: str, path: str):
|
|
|
41
41
|
time.sleep(_RENAME_WAIT)
|
|
42
42
|
|
|
43
43
|
|
|
44
|
+
# 生效条件:行为与 _publish 完全一致(直接委派并返回其结果)——公开名,供包内
|
|
45
|
+
# 各「tmp + os.replace」原子写点位统一改走带 Windows 短重试的实现。
|
|
46
|
+
def publish(tmp: str, path: str):
|
|
47
|
+
"""`os.replace` 的公开安全版:目标被 Defender/索引器短暂持锁时短重试。
|
|
48
|
+
|
|
49
|
+
2026-09-25 全量回归实测:裸 `os.replace` 在 Windows 上随机抛
|
|
50
|
+
`PermissionError: [WinError 5]`(retr_s7 的 _postings_meta.json 改名中招,
|
|
51
|
+
失败者随机分布)——md_cg 内原子写一律经本函数,不再各写各的裸 replace。
|
|
52
|
+
"""
|
|
53
|
+
return _publish(tmp, path)
|
|
54
|
+
|
|
55
|
+
|
|
44
56
|
# 生效条件:path 与 data 给定时取 path 所在目录 d 建目录,用 tempfile.mkstemp 在 d 内建临时文件按 encoding 写入 data,durable 为真才 flush+os.fsync(假值不 fsync),再经 _publish(tmp, path) 替换;任一步失败时 finally 里若 tmp 仍非 None 且 os.path.exists(tmp) 为真则 os.remove(OSError 忽略)。
|
|
45
57
|
def atomic_write(path: str, data: str, encoding: str = "utf-8", durable: bool = False):
|
|
46
58
|
"""整文件替换。临时文件与目标同目录(保证同一文件系统,rename 才原子),
|
|
@@ -212,6 +224,42 @@ def read_jsonl(path: str):
|
|
|
212
224
|
continue
|
|
213
225
|
|
|
214
226
|
|
|
227
|
+
# 生效条件:os.path.exists(path) 为假时返回 ([], None);否则以二进制只读打开并 seek 到 offset 后一次性读到 EOF,按 b"\n" 切行逐段 decode("utf-8","replace") 再 strip,空行跳过,json.loads 成功的收入列表,ValueError 跳过;返回 (对象列表, offset+读到的字节数)——EOF 偏移是真实消费位置(stat 与 read 之间并发 append 的字节已计入),供增量缓存作水位。
|
|
228
|
+
def read_jsonl_tail(path: str, offset: int):
|
|
229
|
+
"""从字节偏移 offset 起**增量**读 append-only jsonl → `(records, end_offset)`。
|
|
230
|
+
|
|
231
|
+
存在理由(issue #32):propose 等高频对账每次全量重读 inbox/decisions
|
|
232
|
+
是 O(M+D)/条、批量 O(M²);append-only 契约(写点全部经 append_jsonl,
|
|
233
|
+
无轮转/截断)下「上次扫描到的 size」必为行边界,从该偏移起只解析新增
|
|
234
|
+
字节即可。上一次写入中断在行中间(ends_mid_line 补 \\n 场景)时,残行
|
|
235
|
+
前半已在上一轮装载中被跳过、增量窗口读到的是补写的换行与新行——与
|
|
236
|
+
全量 read_jsonl 的容错结果一致。
|
|
237
|
+
|
|
238
|
+
返回值(水位口径,竞态修复):end_offset 是**实际读到的 EOF 位置**。
|
|
239
|
+
调用方若把「读前 stat 的 size」当水位,stat→read 窗口内的跨进程
|
|
240
|
+
append 会在本轮被并入、而水位偏小——下一轮增量从旧水位重读,同一条
|
|
241
|
+
记录在列表型缓存里重复并入且永不自愈。存本函数返回的 end_offset
|
|
242
|
+
即可精确续读。**读后再 stat 不可取**:re-stat 可能大于已消费位置,
|
|
243
|
+
下轮会跳过未读记录(丢账比重复更糟)。文件不存在返回 (list, None),
|
|
244
|
+
调用方自行回落。
|
|
245
|
+
"""
|
|
246
|
+
if not os.path.exists(path):
|
|
247
|
+
return [], None
|
|
248
|
+
recs = []
|
|
249
|
+
with open(path, "rb") as f:
|
|
250
|
+
f.seek(offset)
|
|
251
|
+
data = f.read()
|
|
252
|
+
for b in data.split(b"\n"):
|
|
253
|
+
line = b.decode("utf-8", "replace").strip()
|
|
254
|
+
if not line:
|
|
255
|
+
continue
|
|
256
|
+
try:
|
|
257
|
+
recs.append(json.loads(line))
|
|
258
|
+
except ValueError:
|
|
259
|
+
continue
|
|
260
|
+
return recs, offset + len(data)
|
|
261
|
+
|
|
262
|
+
|
|
215
263
|
_COUNT_CACHE = {} # abspath -> (bytes_scanned, mtime_ns, lines)
|
|
216
264
|
|
|
217
265
|
|
package/md_cg/hotcache.py
CHANGED
|
@@ -1,238 +1,255 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
|
-
"""md_cg · 热路径缓存(L0 节点缓存 + query 结果缓存)
|
|
3
|
-
|
|
4
|
-
【为什么】检索链路每次都跑四路 RRF 融合(约 50ms),但同一 query
|
|
5
|
-
短时间内反复调用是常态(agent 对话轮次间、batch 调用等)。热缓存
|
|
6
|
-
让重复 query 毫秒级返回,同时 L0 高频节点(session=main)的
|
|
7
|
-
frontmatter 读取也从磁盘降为内存命中。
|
|
8
|
-
|
|
9
|
-
【设计】
|
|
10
|
-
- 节点缓存:LRU dict,key=node_id → value=(frontmatter, body_excerpt)
|
|
11
|
-
- query 缓存:LRU dict,key=(query, k, layer, session, branch, validity, view,
|
|
12
|
-
extra) → value=(results, meta)
|
|
13
|
-
**键必须覆盖全部影响结果的参数**(红线,v14 缺陷 C 的教训):validity /
|
|
14
|
-
view 之外,`include_work` / `roles` / `paths` / `path_weights` /
|
|
15
|
-
`recall_only` / `fusion` / `judge` / `judge_ranking` / `goal_text` /
|
|
16
|
-
`context` / `early_stop_threshold` 同样改变候选资格与排序,经 `extra`
|
|
17
|
-
整体入键(`_KEYED_EXTRA` 为清单真源)。
|
|
18
|
-
不入键的后果不是「慢」而是**静默错答**:默认查询会返回工作角色节点
|
|
19
|
-
(资格泄漏)、`paths` 口径互相顶替等。
|
|
20
|
-
- 无法稳定规范化的参数(自定义可调用 `query_expand`,见 `_BYPASS_EXTRA`):
|
|
21
|
-
**非默认值即绕行缓存**(读+写双侧),fail-closed——宁可不用缓存,不可串味。
|
|
22
|
-
- 失效策略:写入时 invalidate 受影响节点 + query 全清(保守策略,
|
|
23
|
-
因为 RRF 融合后一个节点的变动可能影响全局排序)
|
|
24
|
-
- TTL:query 缓存 300s(5 分钟),节点缓存无 TTL(写入即失效)
|
|
25
|
-
|
|
26
|
-
【边界】
|
|
27
|
-
- 进程内缓存,不跨进程(每个 cg 实例独立)
|
|
28
|
-
- 不改 RRF 核心算法(bench6 600/600 rank 逐位一致是硬约束)
|
|
29
|
-
- 缓存命中时 meta 带 "cached": True 标记
|
|
30
|
-
|
|
31
|
-
零第三方依赖(D-005)。
|
|
32
|
-
"""
|
|
33
|
-
from __future__ import annotations
|
|
34
|
-
|
|
35
|
-
import os
|
|
36
|
-
import time
|
|
37
|
-
from collections import OrderedDict
|
|
38
|
-
|
|
39
|
-
#: 节点缓存上限(env MDCG_HOTCACHE_MAX_NODES 可覆盖——公共评测池 locomo-zh-500
|
|
40
|
-
#: 为 567 节点,缺省 256 会持续 LRU 淘汰,issue #28:容量应可配或自适应)
|
|
41
|
-
MAX_NODES = 256
|
|
42
|
-
#: query 缓存上限(env MDCG_HOTCACHE_MAX_QUERIES 可覆盖)
|
|
43
|
-
MAX_QUERIES = 64
|
|
44
|
-
#: query 缓存 TTL(秒)
|
|
45
|
-
QUERY_TTL = 300.0
|
|
46
|
-
|
|
47
|
-
#: 进键的「口径参数」清单(**单一真源**):调用方(mdcos.search_rrf)按此
|
|
48
|
-
#: 构造 extra。新增影响结果的检索参数必须同时登记到本清单——漏登即跨口径
|
|
49
|
-
#: 串味(v14 缺陷 C:include_work/roles 致资格泄漏、paths 互相顶替)。
|
|
50
|
-
_KEYED_EXTRA = ("include_work", "roles", "paths", "path_weights", "recall_only",
|
|
51
|
-
"fusion", "judge", "judge_ranking", "goal_text", "context",
|
|
52
|
-
"early_stop_threshold"
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
self.
|
|
86
|
-
self.
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
"""
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
def
|
|
97
|
-
"""
|
|
98
|
-
self._nodes.
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
self._queries.
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
self.
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
def
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""md_cg · 热路径缓存(L0 节点缓存 + query 结果缓存)
|
|
3
|
+
|
|
4
|
+
【为什么】检索链路每次都跑四路 RRF 融合(约 50ms),但同一 query
|
|
5
|
+
短时间内反复调用是常态(agent 对话轮次间、batch 调用等)。热缓存
|
|
6
|
+
让重复 query 毫秒级返回,同时 L0 高频节点(session=main)的
|
|
7
|
+
frontmatter 读取也从磁盘降为内存命中。
|
|
8
|
+
|
|
9
|
+
【设计】
|
|
10
|
+
- 节点缓存:LRU dict,key=node_id → value=(frontmatter, body_excerpt)
|
|
11
|
+
- query 缓存:LRU dict,key=(query, k, layer, session, branch, validity, view,
|
|
12
|
+
extra) → value=(results, meta)
|
|
13
|
+
**键必须覆盖全部影响结果的参数**(红线,v14 缺陷 C 的教训):validity /
|
|
14
|
+
view 之外,`include_work` / `roles` / `paths` / `path_weights` /
|
|
15
|
+
`recall_only` / `fusion` / `judge` / `judge_ranking` / `goal_text` /
|
|
16
|
+
`context` / `early_stop_threshold` 同样改变候选资格与排序,经 `extra`
|
|
17
|
+
整体入键(`_KEYED_EXTRA` 为清单真源)。
|
|
18
|
+
不入键的后果不是「慢」而是**静默错答**:默认查询会返回工作角色节点
|
|
19
|
+
(资格泄漏)、`paths` 口径互相顶替等。
|
|
20
|
+
- 无法稳定规范化的参数(自定义可调用 `query_expand`,见 `_BYPASS_EXTRA`):
|
|
21
|
+
**非默认值即绕行缓存**(读+写双侧),fail-closed——宁可不用缓存,不可串味。
|
|
22
|
+
- 失效策略:写入时 invalidate 受影响节点 + query 全清(保守策略,
|
|
23
|
+
因为 RRF 融合后一个节点的变动可能影响全局排序)
|
|
24
|
+
- TTL:query 缓存 300s(5 分钟),节点缓存无 TTL(写入即失效)
|
|
25
|
+
|
|
26
|
+
【边界】
|
|
27
|
+
- 进程内缓存,不跨进程(每个 cg 实例独立)
|
|
28
|
+
- 不改 RRF 核心算法(bench6 600/600 rank 逐位一致是硬约束)
|
|
29
|
+
- 缓存命中时 meta 带 "cached": True 标记
|
|
30
|
+
|
|
31
|
+
零第三方依赖(D-005)。
|
|
32
|
+
"""
|
|
33
|
+
from __future__ import annotations
|
|
34
|
+
|
|
35
|
+
import os
|
|
36
|
+
import time
|
|
37
|
+
from collections import OrderedDict
|
|
38
|
+
|
|
39
|
+
#: 节点缓存上限(env MDCG_HOTCACHE_MAX_NODES 可覆盖——公共评测池 locomo-zh-500
|
|
40
|
+
#: 为 567 节点,缺省 256 会持续 LRU 淘汰,issue #28:容量应可配或自适应)
|
|
41
|
+
MAX_NODES = 256
|
|
42
|
+
#: query 缓存上限(env MDCG_HOTCACHE_MAX_QUERIES 可覆盖)
|
|
43
|
+
MAX_QUERIES = 64
|
|
44
|
+
#: query 缓存 TTL(秒)
|
|
45
|
+
QUERY_TTL = 300.0
|
|
46
|
+
|
|
47
|
+
#: 进键的「口径参数」清单(**单一真源**):调用方(mdcos.search_rrf)按此
|
|
48
|
+
#: 构造 extra。新增影响结果的检索参数必须同时登记到本清单——漏登即跨口径
|
|
49
|
+
#: 串味(v14 缺陷 C:include_work/roles 致资格泄漏、paths 互相顶替)。
|
|
50
|
+
_KEYED_EXTRA = ("include_work", "roles", "paths", "path_weights", "recall_only",
|
|
51
|
+
"fusion", "judge", "judge_ranking", "goal_text", "context",
|
|
52
|
+
"early_stop_threshold",
|
|
53
|
+
# 进程级开关快照(见 env_switch_key):参数面之外还有一类口径来自
|
|
54
|
+
# 环境变量(MDCG_SEMANTIC/MDCG_EN_ATOMS/MDCG_UNIFY_QUERY…),它们
|
|
55
|
+
# 同样改变候选资格与打分。v14 只补了参数面,故另立一键
|
|
56
|
+
# (2026-09-24 修复:同 query 下切开关会命中另一口径的缓存 → 静默错答)。
|
|
57
|
+
"env_switch")
|
|
58
|
+
#: 不可稳定规范化的参数(自定义可调用):非默认即**绕行缓存**(fail-closed)。
|
|
59
|
+
_BYPASS_EXTRA = ("query_expand",)
|
|
60
|
+
#: 影响检索结果的进程级开关(新增一个就登记一个——同 _KEYED_EXTRA 的纪律)。
|
|
61
|
+
#: MDCG_CN_GRAMS(v9 N78):mdcg.cn_recall_grams 每调用读 env(="0" 回退旧行为),
|
|
62
|
+
#: 改变 expand_query_terms 召回词集——漏登即运行期翻转后同 query 命中另一口径缓存。
|
|
63
|
+
_ENV_SWITCHES = ("MDCG_SEMANTIC", "MDCG_EN_ATOMS", "MDCG_UNIFY_QUERY",
|
|
64
|
+
"MDCG_RETRIEVAL_PIPELINE", "MDCG_S7_FRESHNESS",
|
|
65
|
+
"MDCG_CN_GRAMS")
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
# 生效条件:无必需形参;返回 _ENV_SWITCHES 中每个变量名的 (名, 当前取值或 None) 元组(稳定、可哈希,随 query 缓存键一起参与比对)。
|
|
69
|
+
def env_switch_key():
|
|
70
|
+
"""当前进程级检索开关的快照(进 query 缓存键,防跨口径串味)。"""
|
|
71
|
+
return tuple((name, os.environ.get(name)) for name in _ENV_SWITCHES)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
class HotCache:
|
|
75
|
+
"""进程内热路径缓存。
|
|
76
|
+
|
|
77
|
+
使用 OrderedDict 实现 LRU 语义:
|
|
78
|
+
- get 命中时 move_to_end(最近使用放尾)
|
|
79
|
+
- put 满时 popitem(last=False)(淘汰头=最久未用)
|
|
80
|
+
"""
|
|
81
|
+
|
|
82
|
+
def __init__(self, *, max_nodes=MAX_NODES, max_queries=MAX_QUERIES,
|
|
83
|
+
query_ttl=QUERY_TTL):
|
|
84
|
+
self._nodes: OrderedDict = OrderedDict()
|
|
85
|
+
self._queries: OrderedDict = OrderedDict()
|
|
86
|
+
self._max_nodes = max_nodes
|
|
87
|
+
self._max_queries = max_queries
|
|
88
|
+
self._query_ttl = query_ttl
|
|
89
|
+
# 统计
|
|
90
|
+
self._stats = {"node_hits": 0, "node_misses": 0,
|
|
91
|
+
"query_hits": 0, "query_misses": 0,
|
|
92
|
+
"invalidations": 0}
|
|
93
|
+
|
|
94
|
+
# ---------- 节点缓存 ----------
|
|
95
|
+
|
|
96
|
+
def get_node(self, node_id: str):
|
|
97
|
+
"""返回 (frontmatter, body_excerpt) 或 None(未缓存)。"""
|
|
98
|
+
entry = self._nodes.get(node_id)
|
|
99
|
+
if entry is None:
|
|
100
|
+
self._stats["node_misses"] += 1
|
|
101
|
+
return None
|
|
102
|
+
self._stats["node_hits"] += 1
|
|
103
|
+
self._nodes.move_to_end(node_id)
|
|
104
|
+
return entry
|
|
105
|
+
|
|
106
|
+
def put_node(self, node_id: str, frontmatter: dict, body_excerpt: str = ""):
|
|
107
|
+
"""写入/更新节点缓存。"""
|
|
108
|
+
self._nodes[node_id] = (dict(frontmatter), body_excerpt)
|
|
109
|
+
self._nodes.move_to_end(node_id)
|
|
110
|
+
while len(self._nodes) > self._max_nodes:
|
|
111
|
+
self._nodes.popitem(last=False)
|
|
112
|
+
|
|
113
|
+
def invalidate_node(self, node_id: str):
|
|
114
|
+
"""失效单个节点。"""
|
|
115
|
+
self._nodes.pop(node_id, None)
|
|
116
|
+
|
|
117
|
+
# ---------- query 缓存 ----------
|
|
118
|
+
|
|
119
|
+
@staticmethod
|
|
120
|
+
def _canon(v):
|
|
121
|
+
"""参数值 → 可哈希、稳定、跨调用一致的键片段(递归规范化容器)。"""
|
|
122
|
+
if v is None or isinstance(v, (bool, int, float, str, bytes)):
|
|
123
|
+
return v
|
|
124
|
+
if isinstance(v, (list, tuple)):
|
|
125
|
+
return tuple(HotCache._canon(x) for x in v)
|
|
126
|
+
if isinstance(v, (set, frozenset)):
|
|
127
|
+
return tuple(sorted((HotCache._canon(x) for x in v), key=repr))
|
|
128
|
+
if isinstance(v, dict):
|
|
129
|
+
return tuple(sorted((str(k), HotCache._canon(x))
|
|
130
|
+
for k, x in v.items()))
|
|
131
|
+
# 其余(含自定义对象/可调用):repr 含内存地址 → 每次不同 →
|
|
132
|
+
# 缓存永不命中(等价绕行),是**安全侧**失败(不串味)。
|
|
133
|
+
return ("repr", repr(v))
|
|
134
|
+
|
|
135
|
+
@staticmethod
|
|
136
|
+
def _query_key(query, k, layer, session, branch, validity=None,
|
|
137
|
+
view=None, extra=None):
|
|
138
|
+
# 键必须覆盖全部影响结果的参数(正确性缺陷,非优化项):
|
|
139
|
+
# validity(不过滤/排已过期)、view(角色视图)、extra(口径参数,
|
|
140
|
+
# 见 _KEYED_EXTRA)任一不入键都会串结果。
|
|
141
|
+
return (query, k, layer or None, session or None, branch or None,
|
|
142
|
+
bool(validity) or None, view or None,
|
|
143
|
+
HotCache._canon(extra) if extra else None)
|
|
144
|
+
|
|
145
|
+
def get_query(self, query, k=20, layer=None, session=None, branch=None,
|
|
146
|
+
validity=None, view=None, extra=None):
|
|
147
|
+
"""返回 (results, meta) 或 None(未缓存/过期)。"""
|
|
148
|
+
key = self._query_key(query, k, layer, session, branch, validity,
|
|
149
|
+
view, extra)
|
|
150
|
+
entry = self._queries.get(key)
|
|
151
|
+
if entry is None:
|
|
152
|
+
self._stats["query_misses"] += 1
|
|
153
|
+
return None
|
|
154
|
+
ts, results, meta = entry
|
|
155
|
+
if time.time() - ts > self._query_ttl:
|
|
156
|
+
self._queries.pop(key, None)
|
|
157
|
+
self._stats["query_misses"] += 1
|
|
158
|
+
return None
|
|
159
|
+
self._stats["query_hits"] += 1
|
|
160
|
+
self._queries.move_to_end(key)
|
|
161
|
+
return results, dict(meta)
|
|
162
|
+
|
|
163
|
+
def put_query(self, query, results, meta, k=20, layer=None,
|
|
164
|
+
session=None, branch=None, validity=None, view=None,
|
|
165
|
+
extra=None):
|
|
166
|
+
"""写入/更新 query 缓存。"""
|
|
167
|
+
key = self._query_key(query, k, layer, session, branch, validity,
|
|
168
|
+
view, extra)
|
|
169
|
+
self._queries[key] = (time.time(), list(results), dict(meta))
|
|
170
|
+
self._queries.move_to_end(key)
|
|
171
|
+
while len(self._queries) > self._max_queries:
|
|
172
|
+
self._queries.popitem(last=False)
|
|
173
|
+
|
|
174
|
+
def invalidate_all_queries(self):
|
|
175
|
+
"""清空全部 query 缓存(写入时保守全清)。"""
|
|
176
|
+
n = len(self._queries)
|
|
177
|
+
self._queries.clear()
|
|
178
|
+
if n:
|
|
179
|
+
self._stats["invalidations"] += 1
|
|
180
|
+
|
|
181
|
+
def invalidate_node_and_queries(self, node_id: str):
|
|
182
|
+
"""失效节点 + 全清 query(节点变动可能影响全局排序)。"""
|
|
183
|
+
self.invalidate_node(node_id)
|
|
184
|
+
self.invalidate_all_queries()
|
|
185
|
+
|
|
186
|
+
# ---------- 全局 ----------
|
|
187
|
+
|
|
188
|
+
def clear(self):
|
|
189
|
+
"""清空全部缓存。"""
|
|
190
|
+
self._nodes.clear()
|
|
191
|
+
self._queries.clear()
|
|
192
|
+
|
|
193
|
+
def stats(self):
|
|
194
|
+
"""返回统计信息。"""
|
|
195
|
+
s = dict(self._stats)
|
|
196
|
+
s["node_cache_size"] = len(self._nodes)
|
|
197
|
+
s["query_cache_size"] = len(self._queries)
|
|
198
|
+
return s
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
#: 进程级默认实例(每个 cg 实例可独立持有,也可共享此单例)
|
|
202
|
+
_DEFAULT = None
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def default_cache():
|
|
206
|
+
"""进程级默认热缓存单例。"""
|
|
207
|
+
global _DEFAULT
|
|
208
|
+
if _DEFAULT is None:
|
|
209
|
+
_DEFAULT = HotCache()
|
|
210
|
+
return _DEFAULT
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
# 生效条件:cg 传入;已挂(_hotcache 非 None)时幂等返回现有实例;未挂时按 max_nodes/max_queries 显式值构造 HotCache,缺省回落 env MDCG_HOTCACHE_MAX_NODES / MDCG_HOTCACHE_MAX_QUERIES(非整数回落模块级默认),再回落模块级常量;返回挂载的 HotCache。
|
|
214
|
+
def attach(cg, *, max_nodes: int = None, max_queries: int = None):
|
|
215
|
+
"""给 cg 实例挂热缓存(幂等:已挂则不重复)。
|
|
216
|
+
|
|
217
|
+
容量解析优先级:显式入参 > env(MDCG_HOTCACHE_MAX_NODES /
|
|
218
|
+
MDCG_HOTCACHE_MAX_QUERIES)> 模块级常量(issue #28:容量可配,
|
|
219
|
+
大库须调大,否则 LRU 持续淘汰使热缓存形同虚设)。
|
|
220
|
+
"""
|
|
221
|
+
if getattr(cg, "_hotcache", None) is not None:
|
|
222
|
+
return cg._hotcache
|
|
223
|
+
|
|
224
|
+
def _cap(explicit, env_key, default):
|
|
225
|
+
if explicit is not None:
|
|
226
|
+
return explicit
|
|
227
|
+
raw = os.environ.get(env_key, "").strip()
|
|
228
|
+
if raw:
|
|
229
|
+
try:
|
|
230
|
+
return int(raw)
|
|
231
|
+
except ValueError:
|
|
232
|
+
pass
|
|
233
|
+
return default
|
|
234
|
+
|
|
235
|
+
cg._hotcache = HotCache(
|
|
236
|
+
max_nodes=_cap(max_nodes, "MDCG_HOTCACHE_MAX_NODES", MAX_NODES),
|
|
237
|
+
max_queries=_cap(max_queries, "MDCG_HOTCACHE_MAX_QUERIES",
|
|
238
|
+
MAX_QUERIES))
|
|
239
|
+
return cg._hotcache
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def get(cg):
|
|
243
|
+
"""取 cg 实例的热缓存(未挂返回 None)。"""
|
|
244
|
+
return getattr(cg, "_hotcache", None)
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
def invalidate(cg, node_id: str = None):
|
|
248
|
+
"""写入后失效(节点 + query 全清)。node_id=None 时全清。"""
|
|
249
|
+
hc = get(cg)
|
|
250
|
+
if hc is None:
|
|
251
|
+
return
|
|
252
|
+
if node_id:
|
|
253
|
+
hc.invalidate_node_and_queries(node_id)
|
|
254
|
+
else:
|
|
255
|
+
hc.clear()
|