@furongjun1999/dsh-memory 0.4.11 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +552 -465
- package/codebuddy/CODEBUDDY.md +11 -3
- package/codebuddy/README.md +92 -90
- package/codebuddy/mcp.json +27 -27
- package/docs/GBrain/345/217/257/345/200/237/351/211/264/347/202/271_/347/201/265/346/236/242/350/220/275/347/202/271/344/272/244/346/216/245_20260919.md +169 -169
- package/docs/Pi/345/217/257/345/255/246/344/271/240/344/274/230/347/202/271_/347/201/265/346/236/242/345/244/247/350/204/221/346/224/271/350/277/233/344/272/244/346/216/245_20260915.md +144 -144
- package/docs/README.md +143 -111
- package/docs/discipline/harnesses.yaml +244 -226
- package/docs/discipline/templates/full.md.tmpl +61 -61
- package/docs/discipline/templates/rules.mdc.tmpl +68 -0
- package/docs/discipline/templates/skill.md.tmpl +23 -23
- package/docs/eval/AGI/344/270/203/347/273/264/350/257/204/345/210/206/346/212/245/345/221/212_md_cg_v1.0.md +299 -299
- package/docs/eval/AGI/344/270/203/347/273/264/350/257/204/345/210/206/346/212/245/345/221/212_md_cg_v2.0.md +239 -239
- package/docs/eval/bench_lingshu_self/bench_self.py +140 -0
- package/docs/eval/bench_lingshu_self/self_bench_result.json +404 -0
- package/docs/eval/bench_lingshu_self//347/201/265/346/236/242/350/207/252/345/272/223/347/253/257/345/210/260/347/253/257/346/243/200/347/264/242/345/256/236/346/265/213_v1.0.md +38 -0
- package/docs/eval//344/270/215/345/217/257/351/235/240/346/200/247/350/220/275/345/234/260_P0_v1.0.md +185 -0
- package/docs/eval//345/256/236/351/252/214/346/226/271/346/241/210_/345/255/246/344/271/240/351/227/255/347/216/257AB/344/270/216/346/250/252/350/257/204_v1.0.md +174 -174
- package/docs/eval//346/225/205/351/232/234/346/263/250/345/205/245/345/256/236/346/265/213_v1.0.md +422 -0
- package/docs/eval//346/250/252/350/257/204_/345/205/255/345/256/266100/351/242/230/344/270/255/350/213/261/345/217/214/346/237/245_v1.0.md +223 -223
- package/docs/eval//347/253/257/345/210/260/347/253/257LoCoMoQA/345/220/214/345/217/243/345/276/204/345/257/271/347/205/247_v1.0.md +100 -0
- package/docs/eval//347/253/257/345/210/260/347/253/257/345/271/262/346/211/260/346/261/240/350/257/204/346/265/213_/347/241/256/345/256/232/346/200/247/350/243/201/345/206/263vsLLM_judge_v1.1.md +197 -0
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v1.md +156 -0
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v10.md +210 -0
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v11.md +227 -0
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v12.md +203 -0
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v13.md +233 -0
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v14.md +191 -0
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v15.md +213 -0
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v16.md +214 -0
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v17.md +199 -0
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v2.md +156 -0
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v3.md +152 -0
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v4.md +128 -0
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v5.md +114 -0
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v6.md +192 -0
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v7.md +187 -0
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v8.md +207 -0
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v9.md +203 -0
- package/docs/{mdcg → hive}//344/273/244/347/211/214/344/270/216/350/247/222/350/211/262/346/235/203/350/201/214/345/210/206/347/246/273_v0.1.md +165 -160
- package/docs/hive//344/273/244/347/211/214/350/257/255/344/271/211/344/277/256/346/255/243_/344/273/262/350/243/201/344/275/215_v0.1.md +76 -0
- package/docs/{mdcg → hive}//345/255/220/344/273/243/347/220/206/351/205/215/347/275/256/346/240/207/345/207/206_v0.5.md +236 -236
- package/docs/hive//345/256/211/345/205/250/345/256/241/350/256/241/345/256/236/351/224/232_v0.1.md +202 -0
- package/docs/hive//346/243/200/347/264/242/346/224/266/346/225/233/345/256/236/346/265/213/344/270/216S1b/350/256/276/350/256/241_v0.1.md +43 -43
- package/docs/hive//346/243/200/347/264/242/347/256/227/346/263/225/345/217/243/345/276/204/345/257/271/347/205/247_v0.1.md +85 -0
- package/docs/hive//346/243/200/347/264/242/350/267/257/345/276/204/344/270/216/350/256/244/347/237/245/347/273/223/346/236/204/345/245/221/347/272/246_v0.1.md +102 -102
- package/docs/hive//350/234/202/345/267/242M6_ingest/345/256/236/346/226/275/350/256/241/345/210/222_v0.1.md +47 -0
- package/docs/hive//350/234/202/345/267/242/345/217/214/345/256/236/344/276/213/344/272/222/351/252/214_/350/256/276/350/256/241/345/256/232/347/250/277.md +519 -503
- package/docs/hive//350/234/202/345/267/242/345/267/245/344/275/234/350/256/260/345/277/206_/351/241/271/347/233/256/350/256/241/345/210/222.md +85 -85
- package/docs/hive//350/234/202/345/267/242/350/256/276/350/256/241_/347/220/206/350/256/272/345/257/271/351/275/220_v0.1.md +191 -0
- package/docs/hive//350/234/202/345/267/242/350/277/255/344/273/243_/345/256/217/350/247/202/344/270/216/347/276/244/344/275/223/350/260/203/345/272/246_v0.1.md +90 -0
- package/docs/images/lingshu-moonlight-covenant-poster-preview.jpg +0 -0
- package/docs/images/lingshu-moonlight-covenant-poster.png +0 -0
- package/docs/mdcg/D_meta_/345/267/245/347/250/213/345/214/226/346/226/271/346/241/210_v0.2.md +216 -216
- package/docs/mdcg/README/350/257/246/347/273/206/347/211/210_v0.4.10.md +646 -646
- package/docs/mdcg/lingshu_tutorial.html +14449 -14449
- package/docs/mdcg/release_v0.4.11.md +49 -0
- package/docs/mdcg/release_v0.4.5.md +55 -55
- package/docs/mdcg/tool_table_v0.3.0.md +117 -117
- package/docs/mdcg//345/205/250/345/272/223/344/273/243/347/240/201/350/257/204/345/256/241/344/270/216/346/235/241/344/273/266/345/214/226/346/263/250/351/207/212_/350/256/241/345/210/222_v0.1.md +600 -600
- package/docs/mdcg//345/212/237/350/203/275/350/260/203/347/224/250/346/230/240/345/260/204/350/241/250_v0.1.md +40 -40
- package/docs/mdcg//345/215/225/345/205/203/350/207/252/346/210/221/351/224/232/347/202/271_/347/263/273/347/273/237/346/217/220/347/244/272/350/257/215/346/240/207/345/207/206_v0.3.md +275 -275
- package/docs/mdcg//345/217/221/345/270/203/351/227/250/347/246/201/351/223/276_v0.1.md +54 -0
- package/docs/mdcg//346/272/220/347/240/201/347/272/247/346/236/266/346/236/204/345/256/241/350/256/241_GPT/346/211/271/350/257/204/345/257/271/347/205/247_v1.0.md +130 -130
- package/docs/mdcg//347/201/265/346/236/24282/345/267/245/345/205/267_/345/212/237/350/203/275/346/225/264/347/220/206/344/270/216/350/277/201/347/247/273/346/230/240/345/260/204_v0.1.md +278 -278
- package/docs/mdcg//347/201/265/346/236/242/350/256/260/345/277/206/345/212/250/350/257/215/345/215/217/350/256/256_v1.0-draft.md +172 -172
- package/docs/mdcg//347/274/272/345/217/243/345/215/225_P0/346/224/266/345/217/243_v0.1.md +270 -270
- package/docs/mdcg//350/256/244/347/237/245/345/233/276_G4-G8/347/274/272/345/217/243/350/243/201/345/256/232/345/215/225_v0.1.md +491 -491
- package/docs/mdcg//350/256/244/347/237/245/345/233/276_/346/235/241/344/273/266/347/251/272/351/227/264/345/220/210/346/210/220/344/270/216/347/224/237/346/225/210/346/235/241/344/273/266/345/217/243/345/276/204_v0.1.md +340 -340
- package/docs/mdcg//350/256/244/347/237/245/345/233/276_/347/264/242/345/274/225/344/270/216/345/267/245/347/250/213/350/247/204/350/214/203/345/214/226_/350/256/241/345/210/222_v0.1.md +588 -588
- package/docs/plans/GridWorld/346/234/200/345/260/217/351/227/255/347/216/257/350/247/204/346/240/274_v0.1.md +176 -176
- package/docs/plans//345/221/275/345/220/215/346/262/273/347/220/206_/351/241/271/347/233/256/350/256/241/345/210/222.md +81 -81
- package/docs/swarm//350/234/202/347/276/244/344/272/222/350/201/224_v0.1.md +704 -704
- package/docs/swarm//350/234/202/347/276/244/345/220/214/351/224/231/346/243/200/346/265/213/345/256/236/351/252/214/345/215/217/350/256/256_v0.1.md +242 -242
- package/docs/theory//344/270/215/345/217/257/351/235/240/345/256/232/347/220/206/344/270/216/345/244/261/346/225/210/344/274/230/345/205/210/346/241/206/346/236/266_v0.3.md +365 -0
- package/docs/theory//345/215/225/347/272/277/347/250/213/344/270/216/346/263/250/346/204/217/345/212/233/351/233/206/344/270/255_/346/227/240/344/272/211/350/256/256/347/220/206/350/256/272/346/226/207/346/241/243_v1.0.md +129 -129
- package/docs/theory//345/271/266/345/217/221/345/277/205/347/204/266/346/200/247/347/220/206/350/256/272_v0.2.md +134 -134
- package/docs/theory//346/231/272/350/203/275/350/256/2723.4.md +5260 -5260
- package/docs/theory//346/246/202/345/277/265/345/210/206/345/261/202/345/257/271/351/275/220/350/241/250_v0.1.md +145 -145
- package/docs/theory//347/220/206/350/256/272_/346/234/272/345/210/266_/344/273/243/347/240/201_/345/256/236/351/252/214_/347/274/272/345/217/243/347/237/251/351/230/265_v0.1.md +144 -144
- package/docs/theory//347/220/206/350/256/272/344/273/223/346/213/206/345/210/206/344/270/216/347/231/275/347/256/261/347/237/245/350/257/206/345/272/223/345/206/205/350/277/201_v0.1.md +198 -198
- package/docs/theory//350/256/244/347/237/245/344/273/243/347/220/206/344/270/216/346/224/266/346/225/233/347/273/223/346/236/204_/346/235/241/344/273/266/350/256/272/351/207/215/346/236/204_v0.1.md +221 -221
- package/docs/world_model//344/270/226/347/225/214/346/250/241/345/236/213_/347/245/236/347/273/217/347/275/221/347/273/234/345/272/225/345/261/202/346/236/266/346/236/204_v1.0.md +237 -237
- package/docs//344/270/215/345/217/257/351/235/240/346/200/247/347/220/206/350/256/272_v0.1.md +343 -0
- package/docs//345/217/221/345/270/203/344/273/266/345/233/236/346/272/257/350/257/264/346/230/216_v0.1.md +82 -82
- package/docs//345/267/245/344/275/234/347/272/252/345/276/213_/350/256/244/347/237/245/345/233/276/346/235/241/347/233/256_v1.1.json +28 -2
- package/dsh/README.md +82 -82
- package/dsh/cordis.yml.example +139 -139
- package/dsh/update-lingshu.bat +11 -11
- package/lib/bridge.d.ts +9 -0
- package/lib/bridge.js +35 -0
- package/lib/hooks.js +36 -2
- package/lib/index.js +7 -1
- package/lib/lib/roleplay_web.js +116 -29
- package/lib/lib/token_store.d.ts +7 -1
- package/lib/lib/token_store.js +12 -3
- package/md_cg/__init__.py +7 -7
- package/md_cg/audit.py +379 -368
- package/md_cg/autonomy.py +287 -287
- package/md_cg/backfill.py +1328 -1327
- package/md_cg/backfill_bigdomain.py +34 -34
- package/md_cg/backfill_bucket_zh.py +35 -0
- package/md_cg/bench6_arms.py +410 -410
- package/md_cg/bench6_common.py +230 -230
- package/md_cg/bench6_competitors.py +212 -212
- package/md_cg/bench_axis_domain.py +257 -257
- package/md_cg/bench_blind_comp.py +308 -308
- package/md_cg/bench_e2e_judge.py +532 -0
- package/md_cg/bench_e2e_locomo_qa.py +368 -0
- package/md_cg/bench_e2e_qa.py +256 -0
- package/md_cg/bench_en_atoms_public.py +230 -230
- package/md_cg/bench_governance.py +348 -348
- package/md_cg/bench_lme_zh.py +410 -410
- package/md_cg/bench_locomo.py +121 -121
- package/md_cg/bench_locomo_zh.py +450 -450
- package/md_cg/bench_locomo_zh_public.py +147 -147
- package/md_cg/bench_longmem.py +112 -112
- package/md_cg/bench_membench.py +632 -632
- package/md_cg/bench_p0.py +149 -149
- package/md_cg/bench_progressive.py +287 -287
- package/md_cg/bench_role_views.py +238 -238
- package/md_cg/bench_task_ab.py +243 -243
- package/md_cg/bench_task_ab_llm.py +408 -408
- package/md_cg/bench_unified_en.py +204 -204
- package/md_cg/bench_zh_mad.py +601 -601
- package/md_cg/blindspot_tickets.py +123 -123
- package/md_cg/branches.py +301 -285
- package/md_cg/build_postings.py +73 -73
- package/md_cg/ccgc.py +1006 -948
- package/md_cg/census.py +132 -132
- package/md_cg/chain.py +315 -300
- package/md_cg/codeindex.py +531 -531
- package/md_cg/coldverify.py +292 -292
- package/md_cg/comment_gate.py +337 -337
- package/md_cg/cond_compose.py +190 -190
- package/md_cg/cond_facts.py +154 -154
- package/md_cg/cond_template.json +106 -106
- package/md_cg/condition_anchor.py +142 -142
- package/md_cg/conformance.py +726 -726
- package/md_cg/consistency.py +717 -717
- package/md_cg/consolidate.py +1537 -1439
- package/md_cg/corpus.py +110 -110
- package/md_cg/crosscheck.py +1098 -1097
- package/md_cg/crypto.py +3 -1
- package/md_cg/d_meta.py +310 -310
- package/md_cg/datapath.py +78 -18
- package/md_cg/docindex.py +473 -473
- package/md_cg/eval_common.py +575 -575
- package/md_cg/evidence.py +4 -2
- package/md_cg/evolution.py +477 -477
- package/md_cg/export.py +222 -220
- package/md_cg/forgetting.py +581 -581
- package/md_cg/fsutil.py +377 -329
- package/md_cg/hotcache.py +48 -7
- package/md_cg/hyperedge.py +251 -251
- package/md_cg/identity.py +390 -390
- package/md_cg/insight.py +500 -500
- package/md_cg/interop.py +338 -0
- package/md_cg/judgment_manifest.py +177 -0
- package/md_cg/lexicon/build_cedict_en_zh.py +329 -329
- package/md_cg/lexicon/build_standard_en.py +171 -171
- package/md_cg/lexicon/expand_en_zh.py +211 -211
- package/md_cg/lifecycle.py +272 -272
- package/md_cg/linkref.py +280 -280
- package/md_cg/links.py +140 -107
- package/md_cg/mcp_server.py +403 -55
- package/md_cg/md_whitebox.py +345 -345
- package/md_cg/mdcg.py +783 -233
- package/md_cg/mdcos.py +500 -70
- package/md_cg/metacognition.py +591 -591
- package/md_cg/migrate.py +119 -119
- package/md_cg/migrate_aeis.py +221 -221
- package/md_cg/migrate_roleplay.py +293 -293
- package/md_cg/migrate_wisdom_graph.py +360 -360
- package/md_cg/mreview/__init__.py +25 -25
- package/md_cg/mreview/__main__.py +110 -110
- package/md_cg/mreview/bundle.py +178 -178
- package/md_cg/mreview/candidates.py +262 -262
- package/md_cg/mreview/govern.py +694 -693
- package/md_cg/mreview/locate.py +939 -939
- package/md_cg/mreview/pipeline.py +728 -728
- package/md_cg/mreview/rules/duplication.json +21 -21
- package/md_cg/mreview/rules/field_coverage.json +54 -54
- package/md_cg/mreview/rules/source_license.json +21 -21
- package/md_cg/mreview/rules/template_flow.json +21 -21
- package/md_cg/mreview/ruleset.py +252 -252
- package/md_cg/nodefile.py +575 -575
- package/md_cg/pooling.py +484 -472
- package/md_cg/postings.py +300 -298
- package/md_cg/predict.py +1100 -1100
- package/md_cg/progressive.py +123 -123
- package/md_cg/protect.py +272 -272
- package/md_cg/protocol/md_cg_gate.proto +33 -33
- package/md_cg/protocol.py +372 -372
- package/md_cg/provenance.py +582 -582
- package/md_cg/reach.py +453 -453
- package/md_cg/readcache.py +143 -0
- package/md_cg/reconcile.py +228 -0
- package/md_cg/refindex.py +833 -833
- package/md_cg/refine.py +604 -604
- package/md_cg/review_cli.py +170 -0
- package/md_cg/roleviews.py +89 -89
- package/md_cg/routing.py +393 -365
- package/md_cg/run_tests.py +211 -0
- package/md_cg/scrub.py +13 -3
- package/md_cg/security.py +128 -18
- package/md_cg/self_state.py +1029 -1029
- package/md_cg/selfreport.py +152 -151
- package/md_cg/semantic/__init__.py +10 -10
- package/md_cg/semantic/canonical.py +122 -122
- package/md_cg/semantic/en_normalizer.py +364 -364
- package/md_cg/semantic/en_zh_map.json +28694 -0
- package/md_cg/semantic/export_en_zh_map.py +64 -0
- package/md_cg/semantic/unify.py +45 -0
- package/md_cg/semantic/zh_en_atoms.py +139 -139
- package/md_cg/signer.py +7 -4
- package/md_cg/sources.py +816 -582
- package/md_cg/statushdr.py +179 -179
- package/md_cg/stg.py +54 -37
- package/md_cg/subgraph.py +729 -729
- package/md_cg/sustain.py +35 -5
- package/md_cg/tasks.py +470 -470
- package/md_cg/test_access_hints.py +147 -0
- package/md_cg/test_action_derive.py +203 -203
- package/md_cg/test_audit_rotate.py +270 -270
- package/md_cg/test_autonomy.py +143 -143
- package/md_cg/test_bench_governance.py +102 -102
- package/md_cg/test_blindspot_tickets.py +166 -166
- package/md_cg/test_branch_discard_tombstone.py +136 -0
- package/md_cg/test_branches.py +13 -3
- package/md_cg/test_ccg_perturb.py +184 -184
- package/md_cg/test_ccgc.py +433 -433
- package/md_cg/test_census_prune.py +81 -81
- package/md_cg/test_chain_read_isolate.py +168 -0
- package/md_cg/test_cond_compose_anchors.py +76 -76
- package/md_cg/test_cond_match.py +165 -165
- package/md_cg/test_condition_anchor.py +81 -81
- package/md_cg/test_d_meta.py +412 -412
- package/md_cg/test_datapath_device_name.py +203 -0
- package/md_cg/test_datapath_root.py +199 -199
- package/md_cg/test_emit_negtail_cache.py +156 -0
- package/md_cg/test_en_pipeline.py +22 -2
- package/md_cg/test_gain_gate.py +212 -212
- package/md_cg/test_govern_directread.py +421 -0
- package/md_cg/test_health_scale.py +173 -173
- package/md_cg/test_hive_ingest.py +285 -0
- package/md_cg/test_hot_cold.py +215 -215
- package/md_cg/test_hyperedge.py +245 -245
- package/md_cg/test_i26_empty_first_write.py +116 -0
- package/md_cg/test_i27_e041_identity.py +128 -0
- package/md_cg/test_i28_hotcache_prodpath.py +122 -0
- package/md_cg/test_i32_hotcache_env_key.py +218 -0
- package/md_cg/test_identity_attribution.py +96 -15
- package/md_cg/test_index_durability.py +17 -3
- package/md_cg/test_interop.py +95 -0
- package/md_cg/test_interop_judgment.py +228 -0
- package/md_cg/test_issue39_utf8_stdio.py +273 -0
- package/md_cg/test_lifecycle.py +309 -309
- package/md_cg/test_linkref.py +306 -306
- package/md_cg/test_links_concurrent_write.py +188 -0
- package/md_cg/test_lock.py +43 -43
- package/md_cg/test_md_access_parity.py +255 -255
- package/md_cg/test_md_writepath.py +345 -345
- package/md_cg/test_mdstore_search_parity.py +160 -0
- package/md_cg/test_merge_upsert.py +168 -0
- package/md_cg/test_mr_m2.py +587 -587
- package/md_cg/test_mr_m3.py +710 -710
- package/md_cg/test_mr_m4.py +485 -485
- package/md_cg/test_n123_derive_expiry_chain.py +205 -0
- package/md_cg/test_n130_verify_falsified_protect.py +185 -0
- package/md_cg/test_n131_merge_gate.py +205 -0
- package/md_cg/test_p0.py +250 -250
- package/md_cg/test_p1.py +316 -316
- package/md_cg/test_p10_identity.py +173 -173
- package/md_cg/test_p11_consistency.py +233 -233
- package/md_cg/test_p12_metacognition.py +212 -212
- package/md_cg/test_p13_encryption.py +241 -241
- package/md_cg/test_p14_sustain.py +249 -249
- package/md_cg/test_p15_scrub.py +280 -280
- package/md_cg/test_p16_self_state.py +301 -301
- package/md_cg/test_p17_predict.py +354 -354
- package/md_cg/test_p18_whitebox.py +171 -171
- package/md_cg/test_p19_migrate_roleplay.py +149 -149
- package/md_cg/test_p1x_ref_root.py +160 -0
- package/md_cg/test_p20_evolution.py +315 -315
- package/md_cg/test_p21_tokens.py +293 -270
- package/md_cg/test_p22_theory.py +175 -175
- package/md_cg/test_p23_links.py +311 -311
- package/md_cg/test_p24_evidence.py +227 -227
- package/md_cg/test_p25_weights.py +156 -156
- package/md_cg/test_p26_refindex.py +416 -416
- package/md_cg/test_p27_docindex.py +16 -7
- package/md_cg/test_p28_refcheck.py +305 -305
- package/md_cg/test_p29_session_ingest_export.py +354 -333
- package/md_cg/test_p2_mcp.py +3 -0
- package/md_cg/test_p3.py +11 -2
- package/md_cg/test_p30_maintain.py +330 -330
- package/md_cg/test_p31_insight.py +534 -534
- package/md_cg/test_p32_backfill.py +7 -1
- package/md_cg/test_p33_ccg_wiring.py +293 -293
- package/md_cg/test_p34_crosscheck.py +331 -331
- package/md_cg/test_p35_conditioned_claim.py +252 -252
- package/md_cg/test_p36_kp_align.py +230 -230
- package/md_cg/test_p37_condition_space.py +248 -248
- package/md_cg/test_p38_concurrent_flush.py +102 -0
- package/md_cg/test_p38_contextualize.py +273 -273
- package/md_cg/test_p39_verify_flow.py +153 -0
- package/md_cg/test_p39_vision_evidence.py +369 -369
- package/md_cg/test_p40_refine_worklist.py +241 -241
- package/md_cg/test_p41_evolve_patrol.py +224 -224
- package/md_cg/test_p42_provenance.py +269 -269
- package/md_cg/test_p43_pooling.py +412 -398
- package/md_cg/test_p44_md_whitebox.py +231 -231
- package/md_cg/test_p45_session_identity.py +219 -219
- package/md_cg/test_p46_unit_scope.py +272 -272
- package/md_cg/test_p47_session_view.py +316 -0
- package/md_cg/test_p4_fuzzy.py +223 -223
- package/md_cg/test_p5_semantic.py +226 -226
- package/md_cg/test_p6_consolidate.py +440 -387
- package/md_cg/test_p7_goals_recent.py +202 -202
- package/md_cg/test_p8_subgraph_chain.py +200 -200
- package/md_cg/test_p9_forget_protect.py +231 -231
- package/md_cg/test_predict_beta.py +135 -135
- package/md_cg/test_preflight_failclosed.py +100 -100
- package/md_cg/test_progressive.py +146 -146
- package/md_cg/test_propose_tail_index.py +157 -0
- package/md_cg/test_protocol.py +243 -243
- package/md_cg/test_reach.py +378 -378
- package/md_cg/test_reach_keys.py +201 -201
- package/md_cg/test_read_clip.py +141 -141
- package/md_cg/test_read_scope_b27.py +277 -0
- package/md_cg/test_readcache_default_on.py +168 -0
- package/md_cg/test_readcache_precise_inval.py +270 -0
- package/md_cg/test_readcache_prodpath.py +203 -0
- package/md_cg/test_reconcile_v0.py +294 -0
- package/md_cg/test_retr_gates_prodpath.py +140 -0
- package/md_cg/test_retr_s1.py +344 -340
- package/md_cg/test_retr_s1b.py +276 -209
- package/md_cg/test_retr_s3.py +194 -194
- package/md_cg/test_retr_s4.py +163 -163
- package/md_cg/test_retr_s5.py +200 -200
- package/md_cg/test_retr_s6.py +157 -157
- package/md_cg/test_retr_s7.py +392 -384
- package/md_cg/test_retr_s8_time.py +369 -316
- package/md_cg/test_retr_s9_edges.py +286 -286
- package/md_cg/test_retr_s9_entity_ctx.py +9 -3
- package/md_cg/test_retr_score_once.py +208 -0
- package/md_cg/test_review_conformance.py +367 -367
- package/md_cg/test_review_onepass.py +170 -0
- package/md_cg/test_role_views.py +354 -354
- package/md_cg/test_rrf_graph_seed_cache.py +154 -0
- package/md_cg/test_security_audit.py +155 -0
- package/md_cg/test_security_audit_b26.py +161 -0
- package/md_cg/test_security_audit_v21.py +250 -0
- package/md_cg/test_sem_noise.py +242 -242
- package/md_cg/test_semantic_canonical.py +16 -2
- package/md_cg/test_session_isolation.py +168 -0
- package/md_cg/test_snapshot_autoclose.py +187 -0
- package/md_cg/test_subproc_encoding.py +192 -192
- package/md_cg/test_sustain_mutual.py +153 -153
- package/md_cg/test_tail_watermark_race.py +208 -0
- package/md_cg/test_tasks.py +409 -409
- package/md_cg/test_tenant_env_override_warn.py +139 -0
- package/md_cg/test_tenant_registry_corrupt_warn.py +151 -0
- package/md_cg/test_tool_face.py +189 -189
- package/md_cg/test_transfer.py +180 -180
- package/md_cg/test_trust.py +361 -361
- package/md_cg/test_twophase.py +286 -286
- package/md_cg/test_v14_fixes.py +38 -20
- package/md_cg/test_validity_filter.py +280 -280
- package/md_cg/test_verify_answer.py +138 -138
- package/md_cg/test_verify_dirty_reconcile.py +157 -0
- package/md_cg/test_wisdom_md_store.py +292 -292
- package/md_cg/test_writelimit.py +197 -197
- package/md_cg/test_writepipe.py +214 -214
- package/md_cg/theory.py +6 -3
- package/md_cg/tokens.py +85 -14
- package/md_cg/tool_face.py +260 -260
- package/md_cg/trust.py +986 -950
- package/md_cg/twophase.py +231 -231
- package/md_cg/units.py +668 -667
- package/md_cg/vision_evidence.py +667 -666
- package/md_cg/weights.py +624 -624
- package/md_cg/whitebox.py +527 -527
- package/md_cg/whitebox_kb/__init__.py +37 -37
- package/md_cg/whitebox_kb/aeis_core/__init__.py +42 -42
- package/md_cg/whitebox_kb/aeis_core/semantic.py +280 -280
- package/md_cg/whitebox_kb/aeis_core/textutil.py +13 -13
- package/md_cg/whitebox_kb/engine.py +310 -310
- package/md_cg/whitebox_kb/seed_knowledge//346/231/272/350/203/275/350/256/2723.4.md +5260 -5260
- package/md_cg/whitebox_kb/wisdom/browser_units.py +2631 -2631
- package/md_cg/whitebox_kb/wisdom/causal_discover.py +432 -432
- package/md_cg/whitebox_kb/wisdom/chat_engine.py +1506 -1506
- package/md_cg/whitebox_kb/wisdom/compiler_code_units.py +3033 -3033
- package/md_cg/whitebox_kb/wisdom/condition_algebra.py +112 -112
- package/md_cg/whitebox_kb/wisdom/condition_frame.py +315 -315
- package/md_cg/whitebox_kb/wisdom/condition_kb.py +108 -108
- package/md_cg/whitebox_kb/wisdom/conflict_map.json +4445 -4445
- package/md_cg/whitebox_kb/wisdom/core/lexer.py +512 -512
- package/md_cg/whitebox_kb/wisdom/core/name_checker.py +1023 -1023
- package/md_cg/whitebox_kb/wisdom/cspmn.py +258 -258
- package/md_cg/whitebox_kb/wisdom/csre.py +264 -264
- package/md_cg/whitebox_kb/wisdom/danmaku_audit.py +252 -252
- package/md_cg/whitebox_kb/wisdom/distilled_condition_units.json +6417 -6417
- package/md_cg/whitebox_kb/wisdom/docs/WB-EVAL-20260902b.json +1950 -1950
- package/md_cg/whitebox_kb/wisdom/docs/WB-EVAL-20260902c.json +1296 -1296
- package/md_cg/whitebox_kb/wisdom/docs/whitebox_capability_graph_demo.json +59 -59
- package/md_cg/whitebox_kb/wisdom/graph_db_units.py +3089 -3089
- package/md_cg/whitebox_kb/wisdom/knowledge_points.py +318 -318
- package/md_cg/whitebox_kb/wisdom/md_access.py +470 -470
- package/md_cg/whitebox_kb/wisdom/md_store.py +276 -251
- package/md_cg/whitebox_kb/wisdom/migrate_wisdom.py +476 -476
- package/md_cg/whitebox_kb/wisdom/multilang_ir.py +128 -124
- package/md_cg/whitebox_kb/wisdom/navigate.py +248 -248
- package/md_cg/whitebox_kb/wisdom/neural_retrieve.py +224 -224
- package/md_cg/whitebox_kb/wisdom/os_units.py +2735 -2735
- package/md_cg/whitebox_kb/wisdom/pattern_separation.py +376 -376
- package/md_cg/whitebox_kb/wisdom/prereq_map.json +364 -364
- package/md_cg/whitebox_kb/wisdom/python_code_units.py +2766 -2766
- package/md_cg/whitebox_kb/wisdom/role_solidified.json +7 -7
- package/md_cg/whitebox_kb/wisdom/route_memory.py +244 -244
- package/md_cg/whitebox_kb/wisdom/scene_reconstruction.py +148 -148
- package/md_cg/whitebox_kb/wisdom/snr_report.json +40 -40
- package/md_cg/whitebox_kb/wisdom/test_code_compose_domains.py +7541 -7541
- package/md_cg/whitebox_kb/wisdom/test_compiler_self_bootstrap.py +354 -354
- package/md_cg/whitebox_kb/wisdom/test_ecosystem_assembly.py +158 -158
- package/md_cg/whitebox_kb/wisdom/test_ecosystem_demos.py +193 -193
- package/md_cg/whitebox_kb/wisdom/test_graph_db.py +292 -292
- package/md_cg/whitebox_kb/wisdom/test_python_self_bootstrap.py +160 -160
- package/md_cg/whitebox_kb/wisdom/trigger_words_index.json +4366 -4366
- package/md_cg/whitebox_kb/wisdom/verifier.py +1297 -1297
- package/md_cg/writelimit.py +356 -356
- package/md_cg/writepipe.py +20 -8
- package/package.json +101 -96
- package/skills/plugin.json +54 -54
- package/skills/skills/designer-perspective/SKILL.md +158 -158
- package/skills/skills/designer-perspective/references/01-observation-position.md +66 -66
- package/skills/skills/designer-perspective/references/02-structure-recognition.md +62 -62
- package/skills/skills/designer-perspective/references/03-direction-judgment.md +55 -55
- package/skills/skills/designer-perspective/references/04-qualification-verdict.md +72 -72
- package/skills/skills/designer-perspective/references/05-condition-attribution.md +74 -74
- package/skills/skills/designer-perspective/scripts/designer.py +545 -545
- package/skills/skills/designer-perspective/tests/cases.jsonl +17 -17
- package/skills/skills/designer-perspective/tests/selftest.py +61 -61
- package/skills/skills/lingshu-browser/SKILL.md +60 -60
- package/skills/skills/lingshu-compiler/SKILL.md +56 -56
- package/skills/skills/lingshu-compiler/units/analyze-type-infer/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/check-name-real/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compile-assign/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compile-expr-tree/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compile-full-pipeline/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compile-func-def/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compile-if-then/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compile-logic-expr/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compile-recursive/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compile-scope/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compile-type-check/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compile-while/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-0010c4bf/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-0355bffb/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-0361708a/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-054a0414/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-05a1691a/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-05eeed1e/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-0622a1f6/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-08b54217/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-0a62b70c/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-0ab24d00/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-0e093688/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-0e82b966/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-0ee9b9b9/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-0f3787e6/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-1028685f/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-11897630/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-16661b9b/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-1f722303/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-25be1262/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-2a76ba07/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-2dbea54a/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-2df52f16/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-2ec2c9d2/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-2f8c8f39/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-38378ac7/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-39457d2e/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-39fb5926/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-47f4fbfa/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-4a8cd1f1/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-4b230b6b/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-4cbbda95/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-4d5a68ff/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-56dc9bdd/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-57e76ebe/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-61ff016b/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-63eae588/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-64c4224b/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-66377955/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-674ab4f6/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-6af76fd6/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-6d979db2/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-6de326c0/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-6f1c0eea/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-724d6c9b/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-7690177f/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-7b229a7d/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-7f471b3a/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-815cad08/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-83c61634/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-8c798c81/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-8dd747ac/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-90324d0a/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-94b8d72d/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-94f12231/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-95937c16/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-98a5625b/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-98b4c42f/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-98e3894a/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-9bdfe4b8/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-9d9b4e83/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-9f7be5ad/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-a6daf076/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-a7995a0c/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-a8399248/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-aabbd099/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-afe169d8/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-b0678bda/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-b09bd196/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-b1396e23/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-b9b31ce0/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-c10264a7/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-cb1e8e4b/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-ccafd438/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-cda9c262/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-ce648068/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-cf5776a4/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-d974e5d3/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-e3979fd3/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-eb1cf2b5/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-ecb30d5b/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-f8c8b24b/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-f99fedbe/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-fa8ff5f7/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/compiler-fe1b058d/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/lex-chinese-program/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/lex-dao-de-jing/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/lex-nine-chapters/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/vm-arithmetic/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/vm-array-ops/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/vm-closure-call/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/vm-closure-create/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/vm-compare/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/vm-cond-jump/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/vm-cond-space/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/vm-exception/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/vm-func-call/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/vm-loop-run/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/vm-profiling/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/vm-refcount/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/vm-run-loop/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/vm-short-circuit/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/vm-stack-guard/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/vm-stack-ops/SKILL.md +45 -45
- package/skills/skills/lingshu-compiler/units/vm-trust-accum/SKILL.md +45 -45
- package/skills/skills/lingshu-graph/SKILL.md +63 -63
- package/skills/skills/lingshu-net/SKILL.md +48 -48
- package/skills/skills/lingshu-os/SKILL.md +64 -64
- package/skills/skills/lingshu-pylang/SKILL.md +71 -71
- package/src/bridge.ts +33 -0
- package/src/hooks.ts +38 -2
- package/src/index.ts +526 -518
- package/src/lib/datapath.ts +326 -326
- package/src/lib/mdcg_client.ts +413 -413
- package/src/lib/mutual.ts +428 -428
- package/src/lib/prompt_safety.ts +62 -62
- package/src/lib/python_path.ts +71 -71
- package/src/lib/roleplay_web.ts +116 -29
- package/src/lib/token_store.ts +13 -3
- package/src/tools.ts +212 -212
- package/zcode/AGENTS.md +11 -3
- package/zcode/README.md +41 -41
- /package/docs/{mdcg → hive}//344/270/273/344/273/243/347/220/206/345/255/220/344/273/243/347/220/206/350/256/260/345/277/206/346/236/266/346/236/204/350/256/276/350/256/241.md" +0 -0
package/md_cg/mdcg.py
CHANGED
|
@@ -248,6 +248,12 @@ def en_zh_terms(text: str) -> list:
|
|
|
248
248
|
独立归一化原子+Jaccard 路(REPRODUCE.md 双语双路裁定,md_cg
|
|
249
249
|
char-bigram 管线跑英文实测比独立方案差 25% vs 52%),本集成为
|
|
250
250
|
opt-in 实验能力与 soul hub 序列化出口,不在默认链路生效;
|
|
251
|
+
⚠ 注意与**统一归一层**的区别(2026-09-24 澄清):`semantic/unify.py`
|
|
252
|
+
的 `unify_query` 是另一个开关且**默认开启**(MDCG_UNIFY_QUERY,
|
|
253
|
+
2026-09-23 口径转正),它在检索入口把英文 query 归一成中文原子——
|
|
254
|
+
于是默认态下英文 query 经**词法路**即可命中中文节点,而本函数
|
|
255
|
+
(召回词扩展侧)仍是默认关。两者不能互相推断,改动其一须核对
|
|
256
|
+
test_en_pipeline / test_semantic_canonical 的双态断言;
|
|
251
257
|
- 仅当 query 含英文字母时触发,纯中文 query 零开销零变化;
|
|
252
258
|
- 代词语素剔除(我/你/他…超泛词防污染),动词/名词单字语素保留
|
|
253
259
|
(「吃/雨」在中文正文检索价值高,_score 终排兜底精度);
|
|
@@ -613,7 +619,7 @@ def _strip_empty_gate_fields(entry, enabled=None):
|
|
|
613
619
|
|
|
614
620
|
# 生效条件:entry 与其 frontmatter 的 condition_space 取 cs(缺键/假值按 {});ctx 为假值或非 dict 时无条件返回 True;
|
|
615
621
|
# ctx 与 cs 同时给出非空 observation_position 且 routing.normalize_domain 归一化后不同时返回 False;
|
|
616
|
-
# ctx 与 cs 同时给出长度 2 的 time_window
|
|
622
|
+
# ctx 与 cs 同时给出长度 2 的 time_window 时两端经 trust.epoch_seconds 归一(毫秒自动 /1000)后比较,不相交返回 False(区间相交或任一侧缺值/不可转数值一律返回 True)。
|
|
617
623
|
def _cond_prefilter_pass(entry, ctx) -> bool:
|
|
618
624
|
"""S2 条件空间前置门控:**不读正文**即可判定的硬槽(契约 §3 S2)。
|
|
619
625
|
|
|
@@ -640,14 +646,267 @@ def _cond_prefilter_pass(entry, ctx) -> bool:
|
|
|
640
646
|
cw, nw = ctx.get("time_window"), (entry.get("time_window") or cs.get("time_window"))
|
|
641
647
|
if (isinstance(cw, (list, tuple)) and len(cw) == 2
|
|
642
648
|
and isinstance(nw, (list, tuple)) and len(nw) == 2):
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
649
|
+
# 单位归一(唯一口径 trust.epoch_seconds):历史毫秒节点的区间若不归一,
|
|
650
|
+
# 与秒级查询窗「必然不相交」→ 整片历史观测被误剔除(issue #23 同根因)。
|
|
651
|
+
c0, c1 = trust.epoch_seconds(cw[0]), trust.epoch_seconds(cw[1])
|
|
652
|
+
n0, n1 = trust.epoch_seconds(nw[0]), trust.epoch_seconds(nw[1])
|
|
653
|
+
if None in (c0, c1, n0, n1):
|
|
654
|
+
return True # 任一端不可转 → 信息不足,放行(宁多勿漏)
|
|
655
|
+
if c1 < n0 or n1 < c0:
|
|
656
|
+
return False
|
|
648
657
|
return True
|
|
649
658
|
|
|
650
659
|
|
|
660
|
+
# 生效条件:entries/terms/big_domain/context/min_results 给定;MDCG_RETRIEVAL_PIPELINE 未设="1" 时 entries 原样返回且 gates 为空 dict(默认路径零变更);总开关开启时按既有开关语义执行收敛——S1 域收敛(MDCG_GATE_S1_DOMAIN 未设=开;域内∪未标域兜底池,域内不足 min_results 回退不收敛)、S1b 桶收敛(MDCG_GATE_S1B_BUCKET 显式=1;topk/min_sim 参数化,命中不足回退并记 would_keep)、S2 条件空间硬槽(MDCG_GATE_S2_COND 未设=开;清空回退)、S4 层级激活审计(MDCG_GATE_S4_LAYER 显式=1;只构造 gates["s4"] 审计,加成本体在 _score);返回 (收敛后 entries, gates 审计字典);
|
|
661
|
+
# apply_retrieval_gates(entries, terms, big_domain, context, min_results) -> tuple:
|
|
662
|
+
def apply_retrieval_gates(entries, terms, big_domain, context, min_results):
|
|
663
|
+
"""S1/S1b/S2 候选收敛 + S4 审计——**两份 search 的唯一实现**。
|
|
664
|
+
|
|
665
|
+
issue #25(2026-09-23):此段逻辑原先只存在于 MdCG.search 内,而生产
|
|
666
|
+
调用链(mcp_server → MdCGSecure → MdCGOS)走的是 MdCGOS.search 覆写——
|
|
667
|
+
门控在 生产路径上从未生效,专项测试(test_retr_s*.py)全测非生产路径,
|
|
668
|
+
绿灯是假信号。修复 = 抽出本共享函数,两份 search 原地调用。
|
|
669
|
+
|
|
670
|
+
开关语义(契约 §3,保持不变):
|
|
671
|
+
· 总开关 MDCG_RETRIEVAL_PIPELINE=1;未设 → 本函数是恒等变换;
|
|
672
|
+
· S1 / S2:未设=开,"=0" 显式关(先于 S1b 存在的默认项);
|
|
673
|
+
· S1b / S4:必须显式 "=1"(会新增收敛/新层加成,非对称开关);
|
|
674
|
+
· S3 spread / S7 postings:仍属 MdCG.search 的候选生成段(新 tier),
|
|
675
|
+
MdCGOS 路径的对应物是 reach——本函数不涉及。
|
|
676
|
+
召回安全不变量:任一收敛「命中不足 min_results / 清空」即回退不收敛;
|
|
677
|
+
orphan/未标域恒留兜底池。
|
|
678
|
+
"""
|
|
679
|
+
gates = {}
|
|
680
|
+
if os.environ.get("MDCG_RETRIEVAL_PIPELINE") != "1":
|
|
681
|
+
return entries, gates
|
|
682
|
+
# S4 层级激活优先级审计(加成本体在 _score:layer_boosts() 由打分面消费,
|
|
683
|
+
# 该面两份 search 共享 → S4 加成在生产路径其实一直生效,缺的只是这份审计)
|
|
684
|
+
_s4 = (os.environ.get("MDCG_GATE_S4_LAYER") == "1")
|
|
685
|
+
if _s4 and entries:
|
|
686
|
+
_bo = layer_boosts()
|
|
687
|
+
_lc = {}
|
|
688
|
+
for _e in entries:
|
|
689
|
+
_l = str(_e.get("layer") or "")
|
|
690
|
+
_lc[_l] = _lc.get(_l, 0) + 1
|
|
691
|
+
gates["s4"] = {"boosts": _bo, "layers": _lc}
|
|
692
|
+
if os.environ.get("MDCG_GATE_S1_DOMAIN", "1") != "0" and big_domain:
|
|
693
|
+
# 域内 ∪ 未标域(兜底池):无域标签的历史节点绝不能因「域内够多」被丢
|
|
694
|
+
#(契约 §3 S1 不变量:ORPHAN/未标域必须可被召回)
|
|
695
|
+
same = [e for e in entries
|
|
696
|
+
if not e.get("big_domain") or e.get("big_domain") == big_domain]
|
|
697
|
+
# 召回安全:域内候选不足以支撑 min_results 时不收敛,留全量兜底
|
|
698
|
+
if len(same) >= max(1, min_results):
|
|
699
|
+
gates["s1"] = {"domain": big_domain, "in": len(same),
|
|
700
|
+
"dropped": len(entries) - len(same)}
|
|
701
|
+
entries = same
|
|
702
|
+
else:
|
|
703
|
+
gates["s1"] = {"domain": big_domain, "in": len(same),
|
|
704
|
+
"dropped": 0, "fallback": "insufficient"}
|
|
705
|
+
elif os.environ.get("MDCG_GATE_S1_DOMAIN", "1") != "0":
|
|
706
|
+
gates["s1"] = {"domain": None, "reason": "no_domain_signal"}
|
|
707
|
+
# ---- S1b 细口径桶收敛(设计见 docs/hive/检索收敛实测与S1b设计_v0.1.md)----
|
|
708
|
+
# 为何:真实库 91.1% 节点有非 orphan 桶、期望扫描仅 2.1%(细口径),但 T0/T1 桶路
|
|
709
|
+
# 只在调用方传 context 时可用;普通 search(q) 无 context 就只能全扫。S1b 让 query 侧
|
|
710
|
+
# 自己推断候选桶(只用索引,不读文件),把这份收敛拿回来。
|
|
711
|
+
# 召回安全:orphan/无桶节点恒留兜底;命中不足 min_results 即回退(不改 entries)。
|
|
712
|
+
# 开关与参数就地定义:S1b 只依赖总开关 + MDCG_GATE_S1B_BUCKET。
|
|
713
|
+
_s1b = (os.environ.get("MDCG_GATE_S1B_BUCKET") == "1")
|
|
714
|
+
try:
|
|
715
|
+
_s1b_topk = int(os.environ.get("MDCG_BUCKET_TOPK", "3"))
|
|
716
|
+
except ValueError:
|
|
717
|
+
_s1b_topk = 3
|
|
718
|
+
try:
|
|
719
|
+
_s1b_minsim = min(1.0, max(0.0, float(os.environ.get("MDCG_BUCKET_MIN_SIM", "0.34"))))
|
|
720
|
+
except ValueError:
|
|
721
|
+
_s1b_minsim = 0.34
|
|
722
|
+
if _s1b and entries:
|
|
723
|
+
if _s1b_topk <= 0:
|
|
724
|
+
gates["s1b"] = {"keys": [], "reason": "disabled_by_topk"}
|
|
725
|
+
else:
|
|
726
|
+
_sizes = {}
|
|
727
|
+
_zh_alias = {} # issue #33:桶 → 中文别名集(英文键的跨语言面)
|
|
728
|
+
for _e in entries:
|
|
729
|
+
_b = _e.get("bucket")
|
|
730
|
+
if _b and _b != routing.ORPHAN:
|
|
731
|
+
_sizes[_b] = _sizes.get(_b, 0) + 1
|
|
732
|
+
_al = _zh_alias.setdefault(_b, set())
|
|
733
|
+
for _w in (_e.get("bucket_zh") or []):
|
|
734
|
+
_al.add(str(_w))
|
|
735
|
+
_best = {}
|
|
736
|
+
for _b in _sizes:
|
|
737
|
+
_kk = routing.bucket_key_readable(_b)
|
|
738
|
+
if not _kk:
|
|
739
|
+
continue
|
|
740
|
+
# 比较面 = 可读键 ∪ 中文别名(别名只来自节点自身内容,零翻译依赖;
|
|
741
|
+
# 无别名的存量库行为与旧版逐位一致——别名集为空即旧口径)
|
|
742
|
+
_faces = [_kk] + sorted(_zh_alias.get(_b, ()))
|
|
743
|
+
_sim = 0.0
|
|
744
|
+
for _t in terms:
|
|
745
|
+
for _f in _faces:
|
|
746
|
+
_sv = routing.domain_similarity(_t, _f)
|
|
747
|
+
if _sv > _sim:
|
|
748
|
+
_sim = _sv
|
|
749
|
+
if _sim >= _s1b_minsim:
|
|
750
|
+
_best[_b] = _sim
|
|
751
|
+
if not _best:
|
|
752
|
+
# 审计止血(issue #33 建议 3):区分「真的无匹配」与「跨语言盲区」——
|
|
753
|
+
# query 带中文信号、而全部桶的比较面(键+别名)都无中文 → 前者是
|
|
754
|
+
# 语义不相关(正常回退),后者是修复盲区的运维可见形态。
|
|
755
|
+
_q_zh = any(routing._ZH_RE.search(str(_t)) for _t in terms)
|
|
756
|
+
_faces_all_zh = any(
|
|
757
|
+
routing._ZH_RE.search(_f)
|
|
758
|
+
for _b in _sizes
|
|
759
|
+
for _f in ([routing.bucket_key_readable(_b)]
|
|
760
|
+
+ sorted(_zh_alias.get(_b, ()))))
|
|
761
|
+
gates["s1b"] = {
|
|
762
|
+
"keys": [],
|
|
763
|
+
"reason": ("cross_lang_no_match"
|
|
764
|
+
if (_q_zh and _sizes and not _faces_all_zh)
|
|
765
|
+
else "no_key_match"),
|
|
766
|
+
"in": len(entries), "buckets": len(_sizes)}
|
|
767
|
+
else:
|
|
768
|
+
_picked = sorted(_best.items(),
|
|
769
|
+
key=lambda kv: (-kv[1], -_sizes[kv[0]],
|
|
770
|
+
str(kv[0])))[:_s1b_topk]
|
|
771
|
+
_keep = {_b for _b, _ in _picked}
|
|
772
|
+
_kept = [e for e in entries
|
|
773
|
+
if (e.get("bucket") in _keep)
|
|
774
|
+
or not e.get("bucket")
|
|
775
|
+
or e.get("bucket") == routing.ORPHAN]
|
|
776
|
+
gates["s1b"] = {"keys": [_b for _b, _ in _picked],
|
|
777
|
+
"sims": [round(_v, 4) for _, _v in _picked],
|
|
778
|
+
"in": len(entries), "out": len(_kept),
|
|
779
|
+
"sizes": {_b: _sizes[_b] for _b, _ in _picked}}
|
|
780
|
+
if len(_kept) >= max(1, min_results):
|
|
781
|
+
entries = _kept
|
|
782
|
+
else:
|
|
783
|
+
# 回退:entries 保持不变 → 审计的 out 必须记「真实输出规模」,
|
|
784
|
+
# 命中桶本可保留的数量另存 would_keep(独立复核 2026-09-19 指出口径误导)
|
|
785
|
+
gates["s1b"]["would_keep"] = len(_kept)
|
|
786
|
+
gates["s1b"]["out"] = len(entries)
|
|
787
|
+
gates["s1b"]["fallback"] = "insufficient"
|
|
788
|
+
if os.environ.get("MDCG_GATE_S2_COND", "1") != "0" and isinstance(context, dict):
|
|
789
|
+
kept = [e for e in entries if _cond_prefilter_pass(e, context)]
|
|
790
|
+
gates["s2"] = {"in": len(entries), "out": len(kept),
|
|
791
|
+
"dropped": len(entries) - len(kept)}
|
|
792
|
+
# 门控清空则回退(宁多勿漏)
|
|
793
|
+
if kept:
|
|
794
|
+
entries = kept
|
|
795
|
+
else:
|
|
796
|
+
gates["s2"]["fallback"] = "empty"
|
|
797
|
+
return entries, gates
|
|
798
|
+
|
|
799
|
+
|
|
800
|
+
# 生效条件:无独立生效条件(模块级哨兵字典类);任何变更操作(setitem/delitem/clear/pop/popitem/setdefault/update)都会使 write_gen 自增 1,读取 write_gen 不变更;setitem 的值为含非空 path 字符串的 dict 时把该 path 记入 path_gen(值为当时的 write_gen)且不动 broad_gen,值为其它形态(含 None tombstone)或经 delitem/pop/popitem/setdefault/update 变更时把 broad_gen 置为当时的 write_gen,clear() 缺省(flush 收尾)只使 write_gen 自增、不改写 path_gen/broad_gen,clear(broad=True)(rebuild_index 收尾)同时把 broad_gen 置为当时的 write_gen。
|
|
801
|
+
class _DirtyDict(dict):
|
|
802
|
+
"""写代际哨兵字典(批次 23,issue #31 D-4 / v20 报告):任何变更使
|
|
803
|
+
`write_gen` **单调自增、永不回退**。
|
|
804
|
+
|
|
805
|
+
为什么不用 `len(self._dirty)` 作写代际:flush 后 `_dirty` 清零,新写入
|
|
806
|
+
会使长度**回到旧值**——代际巧合回退会让读缓存误命中陈旧内容(v20
|
|
807
|
+
`v20_d4_repro.py` stale=True 实测)。单调计数器下「任何写都使代际前进」,
|
|
808
|
+
读缓存的失效判定不再依赖长度巧合。
|
|
809
|
+
|
|
810
|
+
消费方纪律:flush/rebuild 清空必须走 `.clear()`(保住子类钩子),
|
|
811
|
+
**不得** `self._dirty = {}` 直接换新 dict——那会退化为普通 dict,
|
|
812
|
+
write_gen 恒 0、读缓存失效面随之失效。
|
|
813
|
+
|
|
814
|
+
脏集精确失效簿记(缺陷迭代第 14 轮,high:写读交替整池重装):
|
|
815
|
+
全局单一 write_gen 作读缓存哨兵会让**任意一次单节点写使全部缓存条目
|
|
816
|
+
同时 miss**(10k 池写读交替实测 3rep 中位 1677.6ms vs 稳态 43.7ms,
|
|
817
|
+
38.4× 退化、线性于 N)。所有节点文件写路径在落盘后必以
|
|
818
|
+
`_dirty[nid] = entry`(entry 带 path)标脏——本类据此把失效粒度从
|
|
819
|
+
整代际细化到 path:`path_gen[path]` 记该 path 最近标脏时的 write_gen,
|
|
820
|
+
读缓存按「path 最近标脏代际 ≤ 缓存代际」判新鲜(见 readcache.install)。
|
|
821
|
+
tombstone(值 None,_unstage 删除记录)与其余无 path 可辨的变更形态
|
|
822
|
+
记 `broad_gen`(保守整池失效,与旧整代际口径等价的安全兜底)。
|
|
823
|
+
`clear()`(flush 收尾)**只推进 write_gen**:flush 只落索引
|
|
824
|
+
派生物(_index_log 分片/快照),不改节点文件——文件内容变更已在各
|
|
825
|
+
写路径 `_dirty[nid]=entry` 时按 path 精确失效;且生产写路径每次写后
|
|
826
|
+
`writepipe._commit_visibility→flush`(每次写必 clear),clear 若再整池
|
|
827
|
+
失效会让精确失效在生产写读交替负载下恒不生效。`clear(broad=True)`
|
|
828
|
+
(rebuild_index 收尾)额外推进 broad_gen 整池失效:rebuild 以盘面
|
|
829
|
+
扫描为准,ccgc/crosscheck/backfill 等「直写文件 + rebuild 收尾」的
|
|
830
|
+
写方不经 `_dirty` 标脏,其文件改写对读缓存的可见性靠这一兜底。
|
|
831
|
+
"""
|
|
832
|
+
|
|
833
|
+
def __init__(self, *a, **k):
|
|
834
|
+
super().__init__(*a, **k)
|
|
835
|
+
self.write_gen = 0
|
|
836
|
+
self.path_gen = {} # path → 该 path 最近一次标脏时的 write_gen
|
|
837
|
+
self.broad_gen = 0 # 最近一次无 path 可辨变更的 write_gen(整池兜底)
|
|
838
|
+
|
|
839
|
+
def _bump(self):
|
|
840
|
+
self.write_gen += 1
|
|
841
|
+
|
|
842
|
+
def _note(self, v):
|
|
843
|
+
"""标脏登记:值带 path → 精确记路径代际;否则保守记整池代际。"""
|
|
844
|
+
p = v.get("path") if isinstance(v, dict) else None
|
|
845
|
+
if isinstance(p, str) and p:
|
|
846
|
+
self.path_gen[p] = self.write_gen
|
|
847
|
+
else:
|
|
848
|
+
self.broad_gen = self.write_gen
|
|
849
|
+
|
|
850
|
+
def __setitem__(self, k, v):
|
|
851
|
+
self._bump()
|
|
852
|
+
self._note(v)
|
|
853
|
+
super().__setitem__(k, v)
|
|
854
|
+
|
|
855
|
+
def __delitem__(self, k):
|
|
856
|
+
self._bump()
|
|
857
|
+
self.broad_gen = self.write_gen
|
|
858
|
+
super().__delitem__(k)
|
|
859
|
+
|
|
860
|
+
def clear(self, broad: bool = False):
|
|
861
|
+
"""清空脏集。
|
|
862
|
+
|
|
863
|
+
broad=False(flush 收尾):只推进 write_gen,不整池失效——flush 只
|
|
864
|
+
落索引派生物(_index_log 分片),节点文件的内容变更已在各写路径
|
|
865
|
+
`_dirty[nid]=entry`(带 path)时按 path 精确失效;且生产写路径每次
|
|
866
|
+
写后 `writepipe._commit_visibility→flush`,整池失效会让精确失效在
|
|
867
|
+
生产写读交替负载下恒不生效。
|
|
868
|
+
broad=True(rebuild_index 收尾):同时推进 broad_gen 整池失效——
|
|
869
|
+
rebuild 以**盘面扫描**为准重建索引,ccgc/crosscheck/backfill 等
|
|
870
|
+
「直写文件 + rebuild 收尾」的写方不经 `_dirty` 标脏,其文件改写对
|
|
871
|
+
读缓存的可见性全靠这一兜底(与 MdStore「写方落盘后显式 reload」
|
|
872
|
+
同构)。
|
|
873
|
+
"""
|
|
874
|
+
self._bump()
|
|
875
|
+
if broad:
|
|
876
|
+
self.broad_gen = self.write_gen
|
|
877
|
+
super().clear()
|
|
878
|
+
|
|
879
|
+
def pop(self, k, *d):
|
|
880
|
+
self._bump()
|
|
881
|
+
self.broad_gen = self.write_gen
|
|
882
|
+
return super().pop(k, *d)
|
|
883
|
+
|
|
884
|
+
def popitem(self):
|
|
885
|
+
self._bump()
|
|
886
|
+
self.broad_gen = self.write_gen
|
|
887
|
+
return super().popitem()
|
|
888
|
+
|
|
889
|
+
def setdefault(self, k, d=None):
|
|
890
|
+
self._bump()
|
|
891
|
+
self.broad_gen = self.write_gen
|
|
892
|
+
return super().setdefault(k, d)
|
|
893
|
+
|
|
894
|
+
def update(self, *a, **k):
|
|
895
|
+
self._bump()
|
|
896
|
+
self.broad_gen = self.write_gen
|
|
897
|
+
super().update(*a, **k)
|
|
898
|
+
|
|
899
|
+
|
|
900
|
+
# P0-1(批次 24)/ V21-3(批次 34,外部报告):node_id 白名单——id 拼进落盘
|
|
901
|
+
# 路径,穿越(..)与绝对路径/路径分隔符一律拒绝。**不含冒号**:NTFS 上文件名
|
|
902
|
+
# 中的 `:` 是备用数据流(ADS)分隔符(`a:b.md` 实际写 a 的 b.md 流,主文件名
|
|
903
|
+
# 错位)——Windows 目标平台硬约束。V21-3 放行**中文**(\u4e00-\u9fff)与**逗号**:
|
|
904
|
+
# tasks.slugify 设计「不翻译、中文原样保留」(task_任务-名-一 形态)、
|
|
905
|
+
# test_reach 用 comma,node——库自身命名链路必须自洽(V21 报告定案 5 官方红
|
|
906
|
+
# 同源)。中文/逗号无路径语义,P0-1 防线(..禁令+realpath 纵深闸)不变。
|
|
907
|
+
_NODE_ID_RE = re.compile(r"^[A-Za-z0-9_.@\u4e00-\u9fff,-]{1,128}$")
|
|
908
|
+
|
|
909
|
+
|
|
651
910
|
# 生效条件:构造须传入 root,经 os.path.abspath 后以 exist_ok=True 创建该目录及 LAYERS 各层子目录;autoflush 无论取值(默认 64)都原样赋给实例。
|
|
652
911
|
class MdCG:
|
|
653
912
|
# 生效条件:root 传参即被 os.path.abspath 绝对化并 makedirs(exist_ok=True) 建立 root 与模块级 LAYERS 各层目录,autoflush(默认 64,含 0 等假值)原样存入 self.autoflush,随后 _load_index() 载入索引、sweep_stale_temps(self.root) 清扫,并把 self 登记进模块级 _LIVE_CGS;
|
|
@@ -666,7 +925,7 @@ class MdCG:
|
|
|
666
925
|
# 近期事件滚动窗口(白箱第 5 篇第 3 章「近期事件」)
|
|
667
926
|
self.recent_log = os.path.join(self.root, "_recent.jsonl")
|
|
668
927
|
self.autoflush = autoflush
|
|
669
|
-
self._dirty =
|
|
928
|
+
self._dirty = _DirtyDict()
|
|
670
929
|
self._log = None
|
|
671
930
|
self.index = self._load_index()
|
|
672
931
|
sweep_stale_temps(self.root)
|
|
@@ -676,7 +935,7 @@ class MdCG:
|
|
|
676
935
|
|
|
677
936
|
# ---------- 索引(派生物,可重建) ----------
|
|
678
937
|
|
|
679
|
-
# 生效条件:os.path.exists(self.index_path) 为真、json.load 成功且其 "schema" 等于模块级 SCHEMA
|
|
938
|
+
# 生效条件:os.path.exists(self.index_path) 为真、json.load 成功且其 "schema" 等于模块级 SCHEMA、且其 "_fingerprint"(目录树 mtime 指纹)等于当前 self._dir_fingerprint() 时以该快照为基底;快照缺失/损坏/无指纹/指纹不符(盘面在快照写入后有增删——其它存活实例未 flush 的写入或外部改盘)均回退 self._scan_nodes() 全库扫描为基底;随后重放 ShardedLog.read_all(self.index_log_dir):记录无 id 跳过、e 为 None 则 pop 该 nid(tombstone)、e 非 None 则覆盖,最终 buckets 由 _count_buckets 重算;
|
|
680
939
|
def _load_index(self):
|
|
681
940
|
idx = None
|
|
682
941
|
if os.path.exists(self.index_path):
|
|
@@ -687,6 +946,17 @@ class MdCG:
|
|
|
687
946
|
idx = d
|
|
688
947
|
except (ValueError, OSError):
|
|
689
948
|
idx = None
|
|
949
|
+
if idx is not None:
|
|
950
|
+
# 指纹校验(issue #33):快照只在「盘面与写入时一致」时可信。
|
|
951
|
+
# close 自动落盘常态化后,快照路径不再扫盘面——若不校验,
|
|
952
|
+
# 其它存活实例未 flush 的写入(文件已落盘、索引增量还在其
|
|
953
|
+
# _dirty)会从索引不可见。指纹不符 → 回退全库扫描(即原
|
|
954
|
+
# 「无快照」路径的兜底行为,成本不劣于改动前)。旧快照无
|
|
955
|
+
# _fingerprint 键 → 同样回退一次(迁移窗口),下次 close
|
|
956
|
+
# 落新快照后恢复快照路径。
|
|
957
|
+
fp = idx.get("_fingerprint")
|
|
958
|
+
if fp is None or fp != self._dir_fingerprint():
|
|
959
|
+
idx = None
|
|
690
960
|
if idx is None:
|
|
691
961
|
idx = {"schema": SCHEMA, "nodes": self._scan_nodes(), "buckets": {}}
|
|
692
962
|
for rec in ShardedLog.read_all(self.index_log_dir):
|
|
@@ -702,6 +972,38 @@ class MdCG:
|
|
|
702
972
|
idx["buckets"] = self._count_buckets(idx["nodes"])
|
|
703
973
|
return idx
|
|
704
974
|
|
|
975
|
+
# 生效条件:遍历 LAYERS 各层目录树(os.walk,不读文件内容),对每个可达目录记录其相对 root 的正斜杠路径到 os.stat().st_mtime_ns 的映射;stat 抛 OSError 的目录跳过;返回该映射。
|
|
976
|
+
def _dir_fingerprint(self):
|
|
977
|
+
"""盘面目录树 mtime 指纹——检测「快照写入后盘面有增删」的廉价哨兵。
|
|
978
|
+
|
|
979
|
+
只 walk 目录 + 每目录一次 stat(不 open/不解析任何 .md),成本
|
|
980
|
+
与目录数成正比、与节点数无关(平铺库 ~LAYERS 次 stat)。目录
|
|
981
|
+
mtime 在直接子项增删时变化(NTFS 100ns / ext4 ns 粒度)。
|
|
982
|
+
边界:同目录**内容级**改写(不动文件名)不触发——合作写路径
|
|
983
|
+
(add/update)都同步走索引增量日志,不依赖本指纹;外部直改
|
|
984
|
+
文件内容属非合作写者协议(见 readcache 同款声明)。
|
|
985
|
+
"""
|
|
986
|
+
fp = {}
|
|
987
|
+
for layer in LAYERS:
|
|
988
|
+
base = os.path.join(self.root, layer)
|
|
989
|
+
for dirpath, _dirs, _files in os.walk(base):
|
|
990
|
+
try:
|
|
991
|
+
fp[os.path.relpath(dirpath, self.root).replace("\\", "/")] = \
|
|
992
|
+
os.stat(dirpath).st_mtime_ns
|
|
993
|
+
except OSError:
|
|
994
|
+
continue
|
|
995
|
+
return fp
|
|
996
|
+
|
|
997
|
+
# 生效条件:遍历 LAYERS 各层目录树(os.walk,不读文件内容),统计文件名以 ".md" 结尾的文件总数并返回。
|
|
998
|
+
def _count_md_files(self):
|
|
999
|
+
"""盘面 .md 文件计数(不 open/不解析)——compact 写快照前的账本对账。"""
|
|
1000
|
+
n = 0
|
|
1001
|
+
for layer in LAYERS:
|
|
1002
|
+
base = os.path.join(self.root, layer)
|
|
1003
|
+
for _dp, _dirs, files in os.walk(base):
|
|
1004
|
+
n += sum(1 for fn in files if fn.endswith(".md"))
|
|
1005
|
+
return n
|
|
1006
|
+
|
|
705
1007
|
@staticmethod
|
|
706
1008
|
# 生效条件:nodes 须为带 values() 的映射且各元素支持 .get("bucket");仅当 bucket 取值为真值时才计入返回计数,缺键或假值均跳过。
|
|
707
1009
|
def _count_buckets(nodes):
|
|
@@ -712,57 +1014,184 @@ class MdCG:
|
|
|
712
1014
|
buckets[b] = buckets.get(b, 0) + 1
|
|
713
1015
|
return buckets
|
|
714
1016
|
|
|
715
|
-
# 生效条件:os.path.exists(self.index_path) 为真、JSON 可解析且 "schema" 等于 SCHEMA
|
|
1017
|
+
# 生效条件:os.path.exists(self.index_path) 为真、JSON 可解析且 "schema" 等于 SCHEMA 时以该快照为基底;快照不存在用空骨架 {"schema":SCHEMA,"nodes":{},"buckets":{}}(不重扫目录);快照损坏(ValueError/OSError)或 schema 不符时降级以 self._scan_nodes() 全库扫描结果为基底(宁重扫勿清池);基底确定后对 index_log_dir 逐条重放(无 id 跳过、e 为 None 则 pop、否则覆盖),落盘并清空日志后替换 self.index 并返回 idx;
|
|
716
1018
|
def compact_index(self):
|
|
717
1019
|
with FileLock(self.index_path):
|
|
718
1020
|
idx = {"schema": SCHEMA, "nodes": {}, "buckets": {}}
|
|
1021
|
+
scan_fallback = False
|
|
719
1022
|
if os.path.exists(self.index_path):
|
|
720
1023
|
try:
|
|
721
1024
|
with open(self.index_path, encoding="utf-8") as f:
|
|
722
1025
|
d = json.load(f)
|
|
723
1026
|
if d.get("schema") == SCHEMA:
|
|
724
1027
|
idx = d
|
|
1028
|
+
else:
|
|
1029
|
+
scan_fallback = True # schema 不符:视同损坏
|
|
725
1030
|
except (ValueError, OSError):
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
1031
|
+
scan_fallback = True # 损坏:不得拿空骨架覆盖
|
|
1032
|
+
if scan_fallback:
|
|
1033
|
+
# 快照损坏/schema 不符 → 降级全库扫描重建基底(宁重扫勿清池)。
|
|
1034
|
+
# close 自动 compact(issue #33)上生产路径后此防御必须:
|
|
1035
|
+
# 空骨架 + 增量日志会写出丢失全部历史节点的快照,而
|
|
1036
|
+
# _load_index 见快照即不扫目录——池子从此不可见。
|
|
1037
|
+
idx = {"schema": SCHEMA, "nodes": self._scan_nodes(),
|
|
1038
|
+
"buckets": {}}
|
|
1039
|
+
|
|
1040
|
+
def _apply_log(nodes):
|
|
1041
|
+
for rec in ShardedLog.read_all(self.index_log_dir):
|
|
1042
|
+
nid, e = rec.get("id"), rec.get("e")
|
|
1043
|
+
if not nid:
|
|
1044
|
+
continue
|
|
1045
|
+
if e is None: # 删除记录(tombstone),见 _load_index
|
|
1046
|
+
nodes.pop(nid, None)
|
|
1047
|
+
else:
|
|
1048
|
+
nodes[nid] = e
|
|
1049
|
+
return nodes
|
|
1050
|
+
|
|
1051
|
+
_apply_log(idx["nodes"])
|
|
1052
|
+
if not scan_fallback and self._count_md_files() != len(idx["nodes"]):
|
|
1053
|
+
# 计数对账(issue #33):快照+日志账本与盘面不符——其它存活
|
|
1054
|
+
# 实例未 flush 的写入(文件已落盘、索引增量还在其 _dirty)
|
|
1055
|
+
# 或外部增删文件。宁重扫勿写缺账快照:缺账快照会被重开
|
|
1056
|
+
# 路径的指纹校验放行,节点从此「在盘上但不可见」。
|
|
1057
|
+
# 与 _load_index 的指纹兜底双保险:此处保快照**完整**,
|
|
1058
|
+
# 指纹保快照**新鲜**。
|
|
1059
|
+
idx = {"schema": SCHEMA,
|
|
1060
|
+
"nodes": _apply_log(self._scan_nodes()),
|
|
1061
|
+
"buckets": {}}
|
|
735
1062
|
idx["buckets"] = self._count_buckets(idx["nodes"])
|
|
1063
|
+
idx["_fingerprint"] = self._dir_fingerprint()
|
|
736
1064
|
atomic_write(self.index_path, json.dumps(idx, ensure_ascii=False))
|
|
737
1065
|
ShardedLog.clear(self.index_log_dir)
|
|
738
1066
|
self.index = idx
|
|
739
1067
|
return idx
|
|
740
1068
|
|
|
741
|
-
# 生效条件:self._dirty
|
|
1069
|
+
# 生效条件:self._dirty 非空时才在 FileLock(index_path) 下(必要时新建 ShardedLog)逐条 append、清空 _dirty 并关闭分片句柄;self._dirty 为空时立即返回、不写任何记录;
|
|
742
1070
|
def flush(self):
|
|
743
1071
|
if not self._dirty:
|
|
744
1072
|
return
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
#
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
#
|
|
1073
|
+
# 世代/互斥防线(批次9,M3.3 落地):append 必须与 compact/rebuild 的
|
|
1074
|
+
# 「read 日志 → clear 分片」临界区互斥(同一把 index_path 锁)——
|
|
1075
|
+
# 否则 B 实例的 append 落在 A 实例 read_all 之后、clear 之前时,
|
|
1076
|
+
# 记录随分片被删,节点「在盘但索引不可见」(9·12 病灶:3 实例并发,
|
|
1077
|
+
# apply 留痕 7590 vs 盘面 ~1036 的成因链之一)。锁内 append 后,
|
|
1078
|
+
# 后续 clear 必然已包含本批记录。
|
|
1079
|
+
with FileLock(self.index_path):
|
|
1080
|
+
if self._log is None:
|
|
1081
|
+
self._log = ShardedLog(self.index_log_dir)
|
|
1082
|
+
for nid, e in self._dirty.items():
|
|
1083
|
+
self._log.append({"id": nid, "e": e})
|
|
1084
|
+
self._dirty.clear() # 保住 _DirtyDict 钩子(批次 23 D-4:不得换新 dict)
|
|
1085
|
+
# 写完立即关分片句柄:Windows 上「被本进程打开的文件」无法删除,
|
|
1086
|
+
# 若持有句柄,rebuild_index 的 ShardedLog.clear 会静默失败,已并进
|
|
1087
|
+
# 快照的旧记录被永久重放(旧条目反而覆盖新快照)。append 内部已
|
|
1088
|
+
# 每次 flush,句柄无需常驻;下一次 append 会按需重开。
|
|
1089
|
+
self._log.close()
|
|
1090
|
+
|
|
1091
|
+
# 生效条件:随认知图对象生命周期结束调用、可重复;先 flush() 落未达 autoflush 阈值的脏索引(否则尾部写入虽在盘上但不可见),随后自动落快照(issue #33:快照存在 → compact_index 增量并入并清日志;快照不存在且 index["nodes"] 非空 → rebuild_index 全量建快照;空库不写快照;OSError/ValueError 静默吞掉——失败时增量日志仍在、重开可重放,close 是兜底路径不该再抛),最后关闭并置空日志句柄(句柄为假值时跳过关闭);
|
|
757
1092
|
def close(self):
|
|
758
1093
|
# 先落脏索引再关句柄:否则未达 autoflush 阈值的尾部写入会永久丢失,
|
|
759
1094
|
# 已有 _index.json 的根重开时不会重扫目录,节点将「在盘上但不可见」。
|
|
760
1095
|
self.flush()
|
|
1096
|
+
# 自动落快照(issue #33):原 close 只 flush——「add→close」的库
|
|
1097
|
+
# 永远没有快照(compact_index 曾是唯一增量写快照点且生产零调用),
|
|
1098
|
+
# 每次重开都 _scan_nodes 全库逐文件解析(1500 池实测 ~104ms/次)
|
|
1099
|
+
# + 无条件重放全部增量日志(在扫描基底上纯冗余)。close 是一次性
|
|
1100
|
+
# 脚本(review_cli 等)与 MCP server 退出的统一优雅收尾点:
|
|
1101
|
+
# · 有快照 → compact(快照+日志=全量账本,增量并入+清日志);
|
|
1102
|
+
# · 无快照但有节点 → rebuild 全量建快照(历史节点可能无日志记录,
|
|
1103
|
+
# 重放拼不出全量,必须扫一次盘面——本次一次,此后重开零扫描);
|
|
1104
|
+
# · 空库不写(不产生空快照文件,保持原行为)。
|
|
1105
|
+
# 失败静默:与 atexit 兜底吞异常同风格;compact 失败时增量日志
|
|
1106
|
+
# 仍在,重开照常重放,语义退回改动前而不会丢数据。
|
|
1107
|
+
try:
|
|
1108
|
+
if os.path.exists(self.index_path):
|
|
1109
|
+
self.compact_index()
|
|
1110
|
+
elif self.index["nodes"]:
|
|
1111
|
+
self.rebuild_index()
|
|
1112
|
+
except (OSError, ValueError):
|
|
1113
|
+
pass
|
|
761
1114
|
if self._log:
|
|
762
1115
|
self._log.close()
|
|
763
1116
|
self._log = None
|
|
764
1117
|
|
|
765
|
-
#
|
|
1118
|
+
# 生效条件:以节点文件绝对路径 p、其所在层目录名 layer、解析出的 fm 与 content 为入参,构造含 path(相对 root 正斜杠)/layer(fm 回落 layer)/role/content_kind/session/tags/bucket(父目录名,等于层名时 None)/bucket_zh/importance/created_at/verification_basis/has_neg_conditions/content_hash/temporal/spatial/time_window/lifecycle 与 trust 状态字段/branch_id/branched_from/evidence_count/big_domain/observation_position/subgraph/edges/protected/protection_reason/immutable/self_state/derived_from/derived_relation 各键的条目,经 _strip_empty_gate_fields 清洗后返回;
|
|
1119
|
+
def _node_entry(self, p, layer, fm, content):
|
|
1120
|
+
"""单节点索引条目——_scan_nodes 与定向 upsert 共用的**唯一真源**。
|
|
1121
|
+
|
|
1122
|
+
issue #34:merge 裁决从 rebuild_index(O(N) 全库扫描)收尾改为
|
|
1123
|
+
定向 upsert(O(1))后,单文件条目构造必须与全量扫描**同源**,
|
|
1124
|
+
否则 upsert 与 rebuild 两条路径的字段集漂移(重建前后索引形态
|
|
1125
|
+
不一致)。提取本方法即为此——_scan_nodes 循环体与 merge 的
|
|
1126
|
+
upsert 都调它,字段口径零漂移由构造保证。
|
|
1127
|
+
"""
|
|
1128
|
+
rel = os.path.relpath(p, self.root).replace("\\", "/")
|
|
1129
|
+
parent = os.path.basename(os.path.dirname(p))
|
|
1130
|
+
return _strip_empty_gate_fields({
|
|
1131
|
+
"path": rel, "layer": fm.get("layer", layer),
|
|
1132
|
+
# role 必须回填:它写在节点 frontmatter 里(写入时 role or "user"),
|
|
1133
|
+
# 但索引重建时若不复制,os.roles 会全部退化为 (none),
|
|
1134
|
+
# 来源归因打分随之失效(实测 48 条全丢)。
|
|
1135
|
+
"role": fm.get("role"),
|
|
1136
|
+
# content_kind 入快照:角色化读取视图(第四阶段 6.1)的
|
|
1137
|
+
# 候选资格维度须免读文件可判(与 role 同款理由)。
|
|
1138
|
+
# 纯增量键:批次 C 之前零消费方,view=None 零行为变更。
|
|
1139
|
+
"content_kind": fm.get("content_kind"),
|
|
1140
|
+
# 会话归属入快照(P45 归因维度):写入路径 _stage 早已带出
|
|
1141
|
+
# 该键,重建路径若漏掉,rebuild_index() 之后「按会话过滤」
|
|
1142
|
+
# 即静默全空——快照与 _stage 必须同口径(与 role 同款理由)。
|
|
1143
|
+
"session": fm.get("session"),
|
|
1144
|
+
"tags": fm.get("tags", []),
|
|
1145
|
+
"bucket": parent if parent != layer else None,
|
|
1146
|
+
# issue #33:中文别名入快照(fm 派生,与写入路径同口径)——
|
|
1147
|
+
# S1b 跨语言收敛免读文件可判。
|
|
1148
|
+
"bucket_zh": fm.get("bucket_zh") or None,
|
|
1149
|
+
"importance": fm.get("importance", 0.5),
|
|
1150
|
+
"created_at": fm.get("created_at", 0),
|
|
1151
|
+
"verification_basis": fm.get("verification_basis"),
|
|
1152
|
+
"has_neg_conditions": nodefile.has_non_applicable(content),
|
|
1153
|
+
# 内容指纹走 nodefile 的唯一实现(两段式对账依赖同一算法)
|
|
1154
|
+
"content_hash": nodefile.content_hash(content),
|
|
1155
|
+
# 时空字段入索引快照:STG 查询免读文件(大域/目录索引的延伸)
|
|
1156
|
+
"temporal": fm.get("temporal"),
|
|
1157
|
+
"spatial": fm.get("spatial"),
|
|
1158
|
+
"time_window": (fm.get("condition_space") or {}).get("time_window"),
|
|
1159
|
+
# 生命周期状态(② 显式状态机):索引入快照 → 免读文件可查,
|
|
1160
|
+
# 写入路径也因此无需读盘就能校验迁移合法性。重建口径与
|
|
1161
|
+
# _stage 一致(旧库无该字段 → None → state_of 视为 active)。
|
|
1162
|
+
lifecycle.STATE_FIELD: fm.get(lifecycle.STATE_FIELD),
|
|
1163
|
+
# 可验证记忆单元(trust):重建口径与 _stage 三件同源
|
|
1164
|
+
# (验证态 / 依赖 / 双时间轴)——索引缺键即免读文件不可判。
|
|
1165
|
+
trust.STATE_FIELD: fm.get(trust.STATE_FIELD),
|
|
1166
|
+
trust.DEPS_FIELD: fm.get(trust.DEPS_FIELD),
|
|
1167
|
+
trust.FROM_FIELD: fm.get(trust.FROM_FIELD),
|
|
1168
|
+
trust.UNTIL_FIELD: fm.get(trust.UNTIL_FIELD),
|
|
1169
|
+
# 规范时间轴键(2026-09-19 阶段一):与 _stage 同口径,
|
|
1170
|
+
# 重建索引后新旧口径一致(检索面 validity 须免读盘可判)。
|
|
1171
|
+
trust.EFFECTIVE_FROM_FIELD: fm.get(trust.EFFECTIVE_FROM_FIELD),
|
|
1172
|
+
trust.EFFECTIVE_UNTIL_FIELD: fm.get(trust.EFFECTIVE_UNTIL_FIELD),
|
|
1173
|
+
# 记忆演化分支(④):重建口径与 _stage 一致
|
|
1174
|
+
"branch_id": fm.get("branch_id"),
|
|
1175
|
+
"branched_from": fm.get("branched_from"),
|
|
1176
|
+
"evidence_count": fm.get("evidence_count", 0),
|
|
1177
|
+
# S1 大域先验:域标签入索引快照 → 候选收敛零读文件(与 add() 同口径)
|
|
1178
|
+
"big_domain": fm.get("big_domain"),
|
|
1179
|
+
# S2 条件门控所需的可判定硬槽(免读文件即可门控)
|
|
1180
|
+
"observation_position": (fm.get("condition_space") or {}).get("observation_position"),
|
|
1181
|
+
# 嵌套子图 / 关系边入索引快照:递归展开与链式遍历免读文件
|
|
1182
|
+
"subgraph": fm.get("subgraph"),
|
|
1183
|
+
"edges": fm.get("edges") or [],
|
|
1184
|
+
"protected": fm.get("protected"),
|
|
1185
|
+
"protection_reason": fm.get("protection_reason"),
|
|
1186
|
+
"immutable": fm.get("immutable"),
|
|
1187
|
+
# 自我状态卡标记:protect 据此豁免「不可覆盖」(仍不可遗忘)
|
|
1188
|
+
"self_state": fm.get("self_state"),
|
|
1189
|
+
# G8 派生溯源:frontmatter 声明入索引 → 悬空巡检零读文件
|
|
1190
|
+
"derived_from": fm.get("derived_from") or [],
|
|
1191
|
+
"derived_relation": fm.get("derived_relation"),
|
|
1192
|
+
})
|
|
1193
|
+
|
|
1194
|
+
# 生效条件:遍历模块级 LAYERS 各层目录下所有 .md 文件,读取抛 OSError 的跳过;nid 取 fm.get("id"),为假值时回落去掉 .md 的文件名;条目经 self._node_entry(p, layer, fm, content) 构造;返回按 nid 排序的 nodes;
|
|
766
1195
|
def _scan_nodes(self):
|
|
767
1196
|
nodes = {}
|
|
768
1197
|
for layer in LAYERS:
|
|
@@ -775,88 +1204,54 @@ class MdCG:
|
|
|
775
1204
|
try:
|
|
776
1205
|
with open(p, encoding="utf-8") as f:
|
|
777
1206
|
fm, content = nodefile.loads(f.read())
|
|
778
|
-
except OSError:
|
|
1207
|
+
except (OSError, UnicodeDecodeError):
|
|
1208
|
+
# UnicodeDecodeError(批次 53 补):非 UTF-8 字节的损坏
|
|
1209
|
+
# 真源此前会炸穿整库装载/重建(init → _load_index →
|
|
1210
|
+
# 指纹失配回落 _scan_nodes → 一个坏文件全库打不开)。
|
|
1211
|
+
# 与 reconcile v0 的 T9 边界同口径:真源损坏只跳过,
|
|
1212
|
+
# 不猜测不放大——启动对账(reconcile_state)会对其
|
|
1213
|
+
# 告警留痕。
|
|
779
1214
|
continue
|
|
780
1215
|
nid = fm.get("id") or fn[:-3]
|
|
781
|
-
|
|
782
|
-
parent = os.path.basename(dirpath)
|
|
783
|
-
nodes[nid] = _strip_empty_gate_fields({
|
|
784
|
-
"path": rel, "layer": fm.get("layer", layer),
|
|
785
|
-
# role 必须回填:它写在节点 frontmatter 里(写入时 role or "user"),
|
|
786
|
-
# 但索引重建时若不复制,os.roles 会全部退化为 (none),
|
|
787
|
-
# 来源归因打分随之失效(实测 48 条全丢)。
|
|
788
|
-
"role": fm.get("role"),
|
|
789
|
-
# content_kind 入快照:角色化读取视图(第四阶段 6.1)的
|
|
790
|
-
# 候选资格维度须免读文件可判(与 role 同款理由)。
|
|
791
|
-
# 纯增量键:批次 C 之前零消费方,view=None 零行为变更。
|
|
792
|
-
"content_kind": fm.get("content_kind"),
|
|
793
|
-
"tags": fm.get("tags", []),
|
|
794
|
-
"bucket": parent if parent != layer else None,
|
|
795
|
-
"importance": fm.get("importance", 0.5),
|
|
796
|
-
"created_at": fm.get("created_at", 0),
|
|
797
|
-
"verification_basis": fm.get("verification_basis"),
|
|
798
|
-
"has_neg_conditions": nodefile.has_non_applicable(content),
|
|
799
|
-
# 内容指纹走 nodefile 的唯一实现(两段式对账依赖同一算法)
|
|
800
|
-
"content_hash": nodefile.content_hash(content),
|
|
801
|
-
# 时空字段入索引快照:STG 查询免读文件(大域/目录索引的延伸)
|
|
802
|
-
"temporal": fm.get("temporal"),
|
|
803
|
-
"spatial": fm.get("spatial"),
|
|
804
|
-
"time_window": (fm.get("condition_space") or {}).get("time_window"),
|
|
805
|
-
# 生命周期状态(② 显式状态机):索引入快照 → 免读文件可查,
|
|
806
|
-
# 写入路径也因此无需读盘就能校验迁移合法性。重建口径与
|
|
807
|
-
# _stage 一致(旧库无该字段 → None → state_of 视为 active)。
|
|
808
|
-
lifecycle.STATE_FIELD: fm.get(lifecycle.STATE_FIELD),
|
|
809
|
-
# 可验证记忆单元(trust):重建口径与 _stage 三件同源
|
|
810
|
-
# (验证态 / 依赖 / 双时间轴)——索引缺键即免读文件不可判。
|
|
811
|
-
trust.STATE_FIELD: fm.get(trust.STATE_FIELD),
|
|
812
|
-
trust.DEPS_FIELD: fm.get(trust.DEPS_FIELD),
|
|
813
|
-
trust.FROM_FIELD: fm.get(trust.FROM_FIELD),
|
|
814
|
-
trust.UNTIL_FIELD: fm.get(trust.UNTIL_FIELD),
|
|
815
|
-
# 规范时间轴键(2026-09-19 阶段一):与 _stage 同口径,
|
|
816
|
-
# 重建索引后新旧口径一致(检索面 validity 须免读盘可判)。
|
|
817
|
-
trust.EFFECTIVE_FROM_FIELD: fm.get(trust.EFFECTIVE_FROM_FIELD),
|
|
818
|
-
trust.EFFECTIVE_UNTIL_FIELD: fm.get(trust.EFFECTIVE_UNTIL_FIELD),
|
|
819
|
-
# 记忆演化分支(④):重建口径与 _stage 一致
|
|
820
|
-
"branch_id": fm.get("branch_id"),
|
|
821
|
-
"branched_from": fm.get("branched_from"),
|
|
822
|
-
"evidence_count": fm.get("evidence_count", 0),
|
|
823
|
-
# S1 大域先验:域标签入索引快照 → 候选收敛零读文件(与 add() 同口径)
|
|
824
|
-
"big_domain": fm.get("big_domain"),
|
|
825
|
-
# S2 条件门控所需的可判定硬槽(免读文件即可门控)
|
|
826
|
-
"observation_position": (fm.get("condition_space") or {}).get("observation_position"),
|
|
827
|
-
# 嵌套子图 / 关系边入索引快照:递归展开与链式遍历免读文件
|
|
828
|
-
"subgraph": fm.get("subgraph"),
|
|
829
|
-
"edges": fm.get("edges") or [],
|
|
830
|
-
"protected": fm.get("protected"),
|
|
831
|
-
"protection_reason": fm.get("protection_reason"),
|
|
832
|
-
"immutable": fm.get("immutable"),
|
|
833
|
-
# 自我状态卡标记:protect 据此豁免「不可覆盖」(仍不可遗忘)
|
|
834
|
-
"self_state": fm.get("self_state"),
|
|
835
|
-
# G8 派生溯源:frontmatter 声明入索引 → 悬空巡检零读文件
|
|
836
|
-
"derived_from": fm.get("derived_from") or [],
|
|
837
|
-
"derived_relation": fm.get("derived_relation"),
|
|
838
|
-
})
|
|
1216
|
+
nodes[nid] = self._node_entry(p, layer, fm, content)
|
|
839
1217
|
# 索引序确定性:按 nid 排序返回。os.walk 的遍历序是**文件系统事实**
|
|
840
1218
|
# (NTFS 上常为字母序,但换 FS / 目录碎片化后不保证),若直接作为
|
|
841
1219
|
# index["nodes"] 的物理序,就会让「完全并列」的结果顺序依赖重建路径。
|
|
842
1220
|
# 排序后重建序恒定(增量路径另由 cut_by_relevance 的 nid 终键兜住)。
|
|
843
1221
|
return {k: nodes[k] for k in sorted(nodes)}
|
|
844
1222
|
|
|
845
|
-
# 生效条件:每次调用都以 self._scan_nodes() 的结果重建 nodes 与 buckets,在 FileLock 下 atomic_write 覆盖 index_path 并 ShardedLog.clear(index_log_dir),随后替换 self.index、清空 _dirty 并返回 idx(无 .md 时也照样覆盖为空索引);
|
|
1223
|
+
# 生效条件:每次调用都以 self._scan_nodes() 的结果重建 nodes 与 buckets 并附 _dir_fingerprint(),在 FileLock 下 atomic_write 覆盖 index_path 并 ShardedLog.clear(index_log_dir),随后替换 self.index、清空 _dirty 并返回 idx(无 .md 时也照样覆盖为空索引);
|
|
846
1224
|
def rebuild_index(self):
|
|
847
1225
|
nodes = self._scan_nodes()
|
|
848
1226
|
idx = {"schema": SCHEMA, "nodes": nodes,
|
|
849
1227
|
"buckets": self._count_buckets(nodes)}
|
|
850
1228
|
with FileLock(self.index_path):
|
|
1229
|
+
idx["_fingerprint"] = self._dir_fingerprint()
|
|
851
1230
|
atomic_write(self.index_path, json.dumps(idx, ensure_ascii=False))
|
|
852
1231
|
ShardedLog.clear(self.index_log_dir)
|
|
853
1232
|
self.index = idx
|
|
854
|
-
|
|
1233
|
+
# broad=True:rebuild 以盘面扫描为准——ccgc/crosscheck/backfill 等
|
|
1234
|
+
# 「直写文件 + rebuild 收尾」的写方不经 _dirty 标脏,其文件改写对
|
|
1235
|
+
# 读缓存的可见性靠这一整池失效兜底(flush 的 clear 则不失效,见
|
|
1236
|
+
# _DirtyDict.clear 文档串)。
|
|
1237
|
+
self._dirty.clear(broad=True) # 保住 _DirtyDict 钩子(批次 23 D-4)
|
|
855
1238
|
return idx
|
|
856
1239
|
|
|
1240
|
+
# P2-20(批次 30,外部审查报告):索引中的 path 参与所有读/写落盘定位
|
|
1241
|
+
# ——写穿越(P0-1 历史节点/索引污染)可经「读穿越」放大。单点校验:
|
|
1242
|
+
# realpath 必须落在 root 内,越界抛 ValueError(宁可少读,不可越权)。
|
|
1243
|
+
def _node_disk_path(self, e):
|
|
1244
|
+
p = os.path.realpath(os.path.join(self.root, e.get("path") or ""))
|
|
1245
|
+
rr = os.path.realpath(self.root)
|
|
1246
|
+
if p != rr and not p.startswith(rr + os.sep):
|
|
1247
|
+
raise ValueError(
|
|
1248
|
+
f"节点路径越界(P2-20):{e.get('path')!r} -> {p}"
|
|
1249
|
+
"——拒绝读写")
|
|
1250
|
+
return p
|
|
1251
|
+
|
|
857
1252
|
# ---------- 写 ----------
|
|
858
1253
|
|
|
859
|
-
# 生效条件:node_id/content
|
|
1254
|
+
# 生效条件:node_id/content 必填;node_id 不匹配 _NODE_ID_RE(^[A-Za-z0-9_.@-]{1,128}$)或含 ".."、或落盘 realpath 越出 self.root 时抛 ValueError(P0-1 白名单+纵深闸);layer 不在 LAYERS 内、或 verification_basis 非 None 且不在 VERIFICATION_BASIS 内时抛 ValueError;consistency 为真且 _cons.check 判 REJECT 时,on_conflict="reject" 抛 ConsistencyError、on_conflict="defer" 返回 None,verdict 为 BLINDSPOT 且 on_conflict="defer" 同样返回 None,其余情形完成写盘/入索引后返回 node_id。
|
|
860
1255
|
def add(self, node_id: str, content: str, layer: str = "knowledge",
|
|
861
1256
|
tags=None, condition_space=None, importance: float = 0.5,
|
|
862
1257
|
confidence: float = 0.6, edges=None, verification_basis: str = None,
|
|
@@ -905,6 +1300,16 @@ class MdCG:
|
|
|
905
1300
|
"""
|
|
906
1301
|
if layer not in LAYERS:
|
|
907
1302
|
raise ValueError(f"未知层:{layer}(允许:{LAYERS})")
|
|
1303
|
+
# P0-1(批次 24,外部审查报告):node_id 是模型可控输入,直接拼
|
|
1304
|
+
# 文件路径——`..` 穿越出 root、Windows 绝对路径(C:/x)在
|
|
1305
|
+
# os.path.join 下直接丢弃前缀 = 任意 .md 覆盖。白名单先行
|
|
1306
|
+
# (报告建议 1),realpath 断言在落盘前兜底(报告建议 2)。
|
|
1307
|
+
nid_s = str(node_id or "")
|
|
1308
|
+
if not nid_s or len(nid_s) > 128 or ".." in nid_s \
|
|
1309
|
+
or not _NODE_ID_RE.match(nid_s):
|
|
1310
|
+
raise ValueError(
|
|
1311
|
+
f"非法 node_id:{node_id!r}(须匹配 {_NODE_ID_RE.pattern} "
|
|
1312
|
+
f"且不含 '..'——node_id 会拼进落盘路径,穿越/绝对路径一律拒绝)")
|
|
908
1313
|
if verification_basis is not None and verification_basis not in VERIFICATION_BASIS:
|
|
909
1314
|
raise ValueError(f"未知验证基底:{verification_basis}(允许:{VERIFICATION_BASIS})")
|
|
910
1315
|
# 写保护:self/anchor 层、protected 标记、importance≥0.7 的**既有**节点
|
|
@@ -940,13 +1345,48 @@ class MdCG:
|
|
|
940
1345
|
extra["semantic_oov"] = _oov
|
|
941
1346
|
except Exception:
|
|
942
1347
|
pass
|
|
1348
|
+
# 桶路由键必须在**选目录之前**补齐:`routing.route_key()` 只认 tags 里的
|
|
1349
|
+
# `domain:` 前缀(其次才是 condition_space.observation_position),**不读
|
|
1350
|
+
# big_domain**;而 bucket 与落盘目录在这一段就定死了(索引的 bucket 同样由
|
|
1351
|
+
# 父目录名派生)。实测:把补标签放在 fm 构造之后(原 W1 的位置)会导致
|
|
1352
|
+
# big_domain 与 domain: 标签都在、但文件仍写进 knowledge/orphan/。
|
|
1353
|
+
# 只对分桶层补标签(BUCKETED_LAYERS=("knowledge",)),避免污染其它层的 tags。
|
|
1354
|
+
if (layer in BUCKETED_LAYERS
|
|
1355
|
+
and os.environ.get("MDCG_RETRIEVAL_PIPELINE") == "1"
|
|
1356
|
+
and not any(str(t).startswith("domain:")
|
|
1357
|
+
for t in (tags or []))):
|
|
1358
|
+
_bd_pre = extra.get("big_domain") or None
|
|
1359
|
+
if not _bd_pre:
|
|
1360
|
+
try:
|
|
1361
|
+
_bd_pre = routing.classify_text(content)
|
|
1362
|
+
except Exception:
|
|
1363
|
+
_bd_pre = None
|
|
1364
|
+
if _bd_pre:
|
|
1365
|
+
tags = list(tags or [])
|
|
1366
|
+
tags.append("domain:" + str(_bd_pre))
|
|
943
1367
|
bucket = None
|
|
944
1368
|
d = os.path.join(self.root, layer)
|
|
1369
|
+
bucket_zh = []
|
|
945
1370
|
if layer in BUCKETED_LAYERS:
|
|
946
1371
|
bucket = routing.bucket_dir(routing.route_key(condition_space, tags))
|
|
947
1372
|
d = os.path.join(d, bucket)
|
|
1373
|
+
# issue #33:英文桶键配中文别名(节点自身 tags/正文中文词,零翻译依赖),
|
|
1374
|
+
# 供 S1b 跨语言收敛;键含中文或无别名时空。写进 fm 保证 _scan_nodes
|
|
1375
|
+
# 重建后口径一致(索引派生原则,见 _stage 注释)。
|
|
1376
|
+
bucket_zh = routing.bucket_zh_aliases(
|
|
1377
|
+
routing.bucket_key_readable(bucket), tags, content)
|
|
948
1378
|
os.makedirs(d, exist_ok=True)
|
|
949
1379
|
path = os.path.join(d, f"{node_id}.md")
|
|
1380
|
+
# P0-1 纵深(批次 24):realpath 断言兜底——白名单已挡穿越/绝对
|
|
1381
|
+
# 路径,此闸防未来 node_id 规则放松或 d 被污染(任何写入路径的
|
|
1382
|
+
# 最后一道闸:落盘位置必须在 root 内)。
|
|
1383
|
+
_real_node = os.path.realpath(path)
|
|
1384
|
+
_real_root = os.path.realpath(self.root)
|
|
1385
|
+
if _real_node != _real_root \
|
|
1386
|
+
and not _real_node.startswith(_real_root + os.sep):
|
|
1387
|
+
raise ValueError(
|
|
1388
|
+
f"node_id 落盘路径越界(realpath={_real_node},root={_real_root})"
|
|
1389
|
+
"——拒绝写入(P0-1 纵深闸)")
|
|
950
1390
|
created_at = extra.pop("created_at", time.time())
|
|
951
1391
|
# 条件论「观测时间」栏:写入时必须记录观测时间窗。
|
|
952
1392
|
# 调用方未提供 time_window 时,以写入时刻为锚、默认窗口 OBSERVATION_WINDOW_SEC。
|
|
@@ -967,6 +1407,8 @@ class MdCG:
|
|
|
967
1407
|
"evidence_count": 0, "positive_evidence": 0, "negative_evidence": 0,
|
|
968
1408
|
}
|
|
969
1409
|
fm.update(extra)
|
|
1410
|
+
if bucket_zh:
|
|
1411
|
+
fm["bucket_zh"] = bucket_zh
|
|
970
1412
|
# S1 大域先验:写入时固化「内容 → 大域」(契约 §3 S1)。
|
|
971
1413
|
# 为何在写入侧:检索侧要按域收敛,节点就必须带域;query 侧分类器已存在,
|
|
972
1414
|
# 缺的只是这一列节点元数据(审计偏差 4 的根因)。调用方可显式传入覆盖。
|
|
@@ -980,6 +1422,21 @@ class MdCG:
|
|
|
980
1422
|
_bd = None
|
|
981
1423
|
if _bd:
|
|
982
1424
|
fm["big_domain"] = _bd
|
|
1425
|
+
# CCG「不适用条件」写入口径:正文声明了 `# 不适用条件:` 而调用方没给列表时,
|
|
1426
|
+
# 按 ccgc 的同一拆分口径([;;])结构化进 frontmatter。否则 has_non_applicable()
|
|
1427
|
+
# (按正文文本判,health 的 neg_conditions_set 用它)与 non_applicable_conditions
|
|
1428
|
+
# (judge_qualification 的 REJECT 路径用它)会长期脱钩——实测历史数据修好后
|
|
1429
|
+
# 新写入节点立刻复现(1/73)。正文里字面 `…` 是写入期截断尾巴,按占位丢弃。
|
|
1430
|
+
if not fm.get("non_applicable_conditions"):
|
|
1431
|
+
_na_line = nodefile.ccg_field_value(content, "不适用条件")
|
|
1432
|
+
if _na_line:
|
|
1433
|
+
_na_vals = [s.strip() for s in re.split(r"[;;]", _na_line) if s.strip()]
|
|
1434
|
+
_na_vals = [re.sub(r"[..…\s,,;;、]+$", "", v).strip()
|
|
1435
|
+
for v in _na_vals]
|
|
1436
|
+
_na_vals = [v for v in _na_vals
|
|
1437
|
+
if v and not nodefile.is_placeholder_text(v)]
|
|
1438
|
+
if _na_vals:
|
|
1439
|
+
fm["non_applicable_conditions"] = _na_vals
|
|
983
1440
|
# 生命周期状态(② 显式状态机,真源 `lifecycle.py`):add 是**全量重建
|
|
984
1441
|
# fm** 而非增量更新,故必须显式处理状态——否则已定型/已降权节点会被静默
|
|
985
1442
|
# 打回 active(与 ④ rewrite 必须重传 branch_id 同构的坑)。口径:
|
|
@@ -1137,9 +1594,34 @@ class MdCG:
|
|
|
1137
1594
|
# 私有内容封装(默认恒等;MdCGSecure 覆盖为 AEAD 加密)。
|
|
1138
1595
|
# 索引派生同样基于落盘内容,保证与 _scan_nodes 重建结果一致。
|
|
1139
1596
|
sealed = self._write_node(node_id, path, fm, content)
|
|
1597
|
+
# 覆写且路由键变化时清理旧桶同 id 文件:不清理则磁盘留双文件,
|
|
1598
|
+
# rebuild_index() 后索引取哪个取决于 os.walk 枚举序——新内容可能被
|
|
1599
|
+
# 旧文件静默顶掉(违背「原文即真源」)。旧索引条目即旧路径唯一线索;
|
|
1600
|
+
# 双闸防误删:必须在 root 内(P2-20 同口径)且不等于本次新路径
|
|
1601
|
+
# (同桶覆写是同一文件,删了等于自毁)。先写新后删旧,顺序不可倒。
|
|
1602
|
+
if prev_entry and prev_entry.get("path"):
|
|
1603
|
+
try:
|
|
1604
|
+
_old_real = os.path.realpath(
|
|
1605
|
+
os.path.join(self.root, prev_entry["path"]))
|
|
1606
|
+
if _old_real != _real_node \
|
|
1607
|
+
and _old_real.startswith(_real_root + os.sep) \
|
|
1608
|
+
and os.path.isfile(_old_real):
|
|
1609
|
+
os.remove(_old_real)
|
|
1610
|
+
except OSError:
|
|
1611
|
+
pass # 删除失败不阻断主写路径(残留双文件退回旧行为,可重建兜底)
|
|
1612
|
+
# 旧桶计数递减(与 _unstage 同口径),否则 buckets 计数漂移;
|
|
1613
|
+
# 仅在桶变化时做——同桶覆写的 _stage 自增是既有口径,不在此对账。
|
|
1614
|
+
_ob = prev_entry.get("bucket")
|
|
1615
|
+
if _ob and _ob != bucket:
|
|
1616
|
+
_left = self.index["buckets"].get(_ob, 1) - 1
|
|
1617
|
+
if _left > 0:
|
|
1618
|
+
self.index["buckets"][_ob] = _left
|
|
1619
|
+
else:
|
|
1620
|
+
self.index["buckets"].pop(_ob, None)
|
|
1140
1621
|
self._stage(node_id, _strip_empty_gate_fields({
|
|
1141
1622
|
"path": os.path.relpath(path, self.root).replace("\\", "/"),
|
|
1142
1623
|
"layer": layer, "tags": tags, "bucket": bucket,
|
|
1624
|
+
"bucket_zh": bucket_zh or None,
|
|
1143
1625
|
"importance": importance, "created_at": fm["created_at"],
|
|
1144
1626
|
"verification_basis": verification_basis,
|
|
1145
1627
|
"has_neg_conditions": nodefile.has_non_applicable(sealed),
|
|
@@ -1243,6 +1725,18 @@ class MdCG:
|
|
|
1243
1725
|
from . import twophase
|
|
1244
1726
|
return twophase.pending(self, limit=limit)
|
|
1245
1727
|
|
|
1728
|
+
# 生效条件:每次调用都延迟导入 reconcile 并以原样 apply(含 False)转调 reconcile.reconcile_state(self, apply=apply) 并返回,自身不做参数回落;
|
|
1729
|
+
def reconcile_state(self, apply: bool = True) -> dict:
|
|
1730
|
+
"""启动对账 reconcile v0(v0.3 T9 落地,落地清单 P0-3):索引(派生
|
|
1731
|
+
态)vs 盘面真源的三类 diff 对账(多/缺索引条目、内容哈希漂移)。
|
|
1732
|
+
|
|
1733
|
+
与 `reconcile_writes`(写账本清账)分工见 `md_cg/reconcile.py` 模块
|
|
1734
|
+
文档串。`MDCG_RECONCILE=0` 关闭(缺省开,开关真源在 reconcile.enabled);
|
|
1735
|
+
`apply=False` 为 dry-run。真源自身损坏只告警留痕,不自动改写(T9 边界)。
|
|
1736
|
+
"""
|
|
1737
|
+
from . import reconcile # 延迟导入:与写路径解耦,避免模块环
|
|
1738
|
+
return reconcile.reconcile_state(self, apply=apply)
|
|
1739
|
+
|
|
1246
1740
|
# ------------------------------------------------------------------
|
|
1247
1741
|
# 边域窄原语(写路径收口:白箱工具写 md 真源的唯一正路)。
|
|
1248
1742
|
#
|
|
@@ -1493,7 +1987,7 @@ class MdCG:
|
|
|
1493
1987
|
"""当前活跃目标(默认最多 5 条)——检索定向的默认来源。"""
|
|
1494
1988
|
return self.list_goals(status="active", limit=limit)
|
|
1495
1989
|
|
|
1496
|
-
# 生效条件:status 不属于模块级 GOAL_STATUSES 时抛 ValueError;self.get(node_id) 取不到节点或其 frontmatter 的 layer 不等于 "goals" 时返回 None;否则写回 goal_status 与 status_changed_at
|
|
1990
|
+
# 生效条件:status 不属于模块级 GOAL_STATUSES 时抛 ValueError;self.get(node_id) 取不到节点或其 frontmatter 的 layer 不等于 "goals" 时返回 None;否则写回 goal_status 与 status_changed_at,置 self._dirty[node_id](读缓存代际失效,entry 取 index["nodes"] 现值或 {"path","layer":"goals"} 兜底)并返回 {"id": node_id, "status": status};
|
|
1497
1991
|
def set_goal_status(self, node_id: str, status: str):
|
|
1498
1992
|
"""目标状态机:active → done/dropped(可回退)。"""
|
|
1499
1993
|
if status not in GOAL_STATUSES:
|
|
@@ -1506,6 +2000,12 @@ class MdCG:
|
|
|
1506
2000
|
fm["status_changed_at"] = time.time()
|
|
1507
2001
|
self._write_node(node_id, os.path.join(self.root, node["path"]),
|
|
1508
2002
|
fm, node["content"])
|
|
2003
|
+
# 标脏(读缓存代际哨兵前提:写路径必须推进 write_gen——直写不标脏
|
|
2004
|
+
# 会让同实例 _goal_entry/检索读到旧 goal_status,p7「done 退出
|
|
2005
|
+
# active」陈旧形态)。不走 _stage:节点已在索引,_stage 会虚增
|
|
2006
|
+
# bucket 计数;这里只推代际,entry 同值幂等。
|
|
2007
|
+
self._dirty[node_id] = self.index["nodes"].get(node_id) \
|
|
2008
|
+
or {"path": node["path"], "layer": "goals"}
|
|
1509
2009
|
return {"id": node_id, "status": status}
|
|
1510
2010
|
|
|
1511
2011
|
# 生效条件:goal 为真值(非 None/空串/0)时返回 str(goal);goal 为假值时返回 self.active_goals(limit=limit) 中非空 goal 文本以空格拼接的字符串;
|
|
@@ -1668,7 +2168,7 @@ class MdCG:
|
|
|
1668
2168
|
st["written"] += 1
|
|
1669
2169
|
continue
|
|
1670
2170
|
fm["big_domain"] = dom
|
|
1671
|
-
self._write_node(nid,
|
|
2171
|
+
self._write_node(nid, self._node_disk_path(e), fm, content)
|
|
1672
2172
|
e["big_domain"] = dom
|
|
1673
2173
|
# 必须走 _stage:索引持久化靠 _dirty → flush → _index_log 重放,
|
|
1674
2174
|
# 只改内存 entry 会在重启后丢掉标签(S1 失效,且二次回填因 fm 已有标签而跳过)
|
|
@@ -1680,6 +2180,55 @@ class MdCG:
|
|
|
1680
2180
|
self.flush()
|
|
1681
2181
|
return st
|
|
1682
2182
|
|
|
2183
|
+
# 生效条件:遍历 index["nodes"](limit 截断);bucket 缺失/orphan/键含中文或可读键为空时计 not_needed 跳过;索引已有 bucket_zh 计 already;get() 不可读计 unreadable;别名=routing.bucket_zh_aliases(可读键, tags, content)(fm 已有则沿用)为空计 not_needed;dry_run 只计 written 不落盘,否则写 fm.bucket_zh + _write_node + _stage(e)(不动 content/其它字段),最终有写入即 flush,返回统计 dict。
|
|
2184
|
+
def backfill_bucket_zh(self, dry_run: bool = False, limit: int = None) -> dict:
|
|
2185
|
+
"""为英文桶键的存量节点补中文别名(issue #33;幂等、零翻译依赖)。
|
|
2186
|
+
|
|
2187
|
+
别名与写入侧同源(`routing.bucket_zh_aliases`:节点自身 tags ∪ 正文
|
|
2188
|
+
中文域词);只补 `fm.bucket_zh` 与索引快照,不动 content/条件空间;
|
|
2189
|
+
加密库走 get() → _write_node() 重封装安全路径(同 backfill_big_domain)。
|
|
2190
|
+
无中文面可用的节点跳过——其在 S1b 的 cross_lang_no_match 审计里可见。
|
|
2191
|
+
"""
|
|
2192
|
+
st = {"seen": 0, "already": 0, "written": 0, "not_needed": 0,
|
|
2193
|
+
"unreadable": 0, "dry_run": bool(dry_run)}
|
|
2194
|
+
for nid, e in list((self.index.get("nodes") or {}).items()):
|
|
2195
|
+
if limit and st["written"] >= limit:
|
|
2196
|
+
break
|
|
2197
|
+
st["seen"] += 1
|
|
2198
|
+
b = e.get("bucket")
|
|
2199
|
+
if not b or b == routing.ORPHAN:
|
|
2200
|
+
st["not_needed"] += 1
|
|
2201
|
+
continue
|
|
2202
|
+
kk = routing.bucket_key_readable(b)
|
|
2203
|
+
if not kk or routing._ZH_RE.search(kk):
|
|
2204
|
+
st["not_needed"] += 1 # 中文键无跨语言问题
|
|
2205
|
+
continue
|
|
2206
|
+
if e.get("bucket_zh"):
|
|
2207
|
+
st["already"] += 1
|
|
2208
|
+
continue
|
|
2209
|
+
got = self.get(nid)
|
|
2210
|
+
if not got or got.get("content") is None:
|
|
2211
|
+
st["unreadable"] += 1
|
|
2212
|
+
continue
|
|
2213
|
+
fm, content = got["frontmatter"], got["content"]
|
|
2214
|
+
aliases = fm.get("bucket_zh") or routing.bucket_zh_aliases(
|
|
2215
|
+
kk, fm.get("tags") or e.get("tags"), content)
|
|
2216
|
+
if not aliases:
|
|
2217
|
+
st["not_needed"] += 1 # 无中文面(cross_lang 审计兜底)
|
|
2218
|
+
continue
|
|
2219
|
+
if dry_run:
|
|
2220
|
+
st["written"] += 1
|
|
2221
|
+
continue
|
|
2222
|
+
fm["bucket_zh"] = aliases
|
|
2223
|
+
self._write_node(nid, self._node_disk_path(e),
|
|
2224
|
+
fm, content)
|
|
2225
|
+
e["bucket_zh"] = aliases
|
|
2226
|
+
self._stage(nid, e)
|
|
2227
|
+
st["written"] += 1
|
|
2228
|
+
if not dry_run and st["written"]:
|
|
2229
|
+
self.flush()
|
|
2230
|
+
return st
|
|
2231
|
+
|
|
1683
2232
|
# ---------- 读 ----------
|
|
1684
2233
|
|
|
1685
2234
|
# 生效条件:index["nodes"].get(node_id) 为假值时回落 self._dirty.get(node_id),仍为假值返回 None;打开 root 下 e["path"] 抛 OSError 返回 None;_open_content 返回 None(无密钥/身份不符)返回 None;否则返回 {id, frontmatter, content, path};
|
|
@@ -1687,7 +2236,7 @@ class MdCG:
|
|
|
1687
2236
|
e = self.index["nodes"].get(node_id) or self._dirty.get(node_id)
|
|
1688
2237
|
if not e:
|
|
1689
2238
|
return None
|
|
1690
|
-
p =
|
|
2239
|
+
p = self._node_disk_path(e)
|
|
1691
2240
|
try:
|
|
1692
2241
|
with open(p, encoding="utf-8") as f:
|
|
1693
2242
|
fm, content = nodefile.loads(f.read())
|
|
@@ -1698,15 +2247,27 @@ class MdCG:
|
|
|
1698
2247
|
return None # 有节点但无密钥 → 不可读即不存在
|
|
1699
2248
|
return {"id": node_id, "frontmatter": fm, "content": content, "path": e["path"]}
|
|
1700
2249
|
|
|
1701
|
-
#
|
|
2250
|
+
# 生效条件:经 _node_disk_path(entry) 定位(P2-20 越界抛 ValueError → 返回 (None, None),不可读即不存在——索引被污染时不得绕过统一校验读 root 外文件,2026-09-25 缺陷 #4),打开成功时返回 nodefile.loads 的 (fm, content);抛 OSError 时同样返回 (None, None);
|
|
1702
2251
|
def _read(self, entry):
|
|
1703
|
-
|
|
2252
|
+
try:
|
|
2253
|
+
p = self._node_disk_path(entry)
|
|
2254
|
+
except ValueError:
|
|
2255
|
+
return None, None
|
|
1704
2256
|
try:
|
|
1705
2257
|
with open(p, encoding="utf-8") as f:
|
|
1706
2258
|
return nodefile.loads(f.read())
|
|
1707
2259
|
except OSError:
|
|
1708
2260
|
return None, None
|
|
1709
2261
|
|
|
2262
|
+
def _doc_norm_bigrams(self, entry, c):
|
|
2263
|
+
"""文档侧归一化 bigram(`_score` 热点,批次 21 issue #31 钩子化)。
|
|
2264
|
+
|
|
2265
|
+
内容不变则派生物不变——readcache 启用时覆写为缓存版(随读缓存
|
|
2266
|
+
一并常驻,Rust `load_docs` 预计算 stripped/db_len 同款理论);
|
|
2267
|
+
基类默认即算即弃,行为与改动前逐位一致。
|
|
2268
|
+
"""
|
|
2269
|
+
return bigrams(normalize_en(c))
|
|
2270
|
+
|
|
1710
2271
|
# ---------- 资格判定(与性能 tier 正交)----------
|
|
1711
2272
|
|
|
1712
2273
|
@staticmethod
|
|
@@ -1868,6 +2429,11 @@ class MdCG:
|
|
|
1868
2429
|
ValueError(fail-closed 只针对调用方误用)。
|
|
1869
2430
|
"""
|
|
1870
2431
|
q = (query or "").strip()
|
|
2432
|
+
# 批次 15 统一口径(unify.py):任意语言 query → 标准原子序列
|
|
2433
|
+
# (统一翻译为中文→归一化到标准中文集→检索);纯中文原样、
|
|
2434
|
+
# MDCG_UNIFY_QUERY=0 可关、失败静默原样
|
|
2435
|
+
from .semantic.unify import unify_query
|
|
2436
|
+
q = unify_query(q)
|
|
1871
2437
|
if not q:
|
|
1872
2438
|
return [], {"tier": None, "reason": "empty_query", "scanned": 0}
|
|
1873
2439
|
pool_cfg = pooling.resolve(pooling.from_env(pools))
|
|
@@ -1880,9 +2446,17 @@ class MdCG:
|
|
|
1880
2446
|
# 把 rejected/unresolved 视作可参与召回的特殊「候选池」
|
|
1881
2447
|
# ——命中它们的结果会改变 meta 的 covered_neg(被负记忆覆盖的查询)
|
|
1882
2448
|
# 默认排除掉负记忆层的节点进入正排打分,仅作为「覆盖标记」用
|
|
1883
|
-
|
|
1884
|
-
|
|
1885
|
-
|
|
2449
|
+
# P2-2(批次 31):单次遍历双收集——负记忆层引用与正排候选
|
|
2450
|
+
# 同车收集,消除此前的第二遍全索引遍历。
|
|
2451
|
+
neg_layer_entries = []
|
|
2452
|
+
entries = []
|
|
2453
|
+
for e in self.index["nodes"].values():
|
|
2454
|
+
if include_neg and e["layer"] in ("rejected", "unresolved"):
|
|
2455
|
+
neg_layer_entries.append(e)
|
|
2456
|
+
if ((not layer or e["layer"] == layer)
|
|
2457
|
+
# '"*"' = 显式跨会话(读遍所有会话);缺省 None 同义
|
|
2458
|
+
and (not session or session == "*"
|
|
2459
|
+
or e.get("session") == session)
|
|
1886
2460
|
# 分支实验场:默认(branch=None)分支节点全部隐身;
|
|
1887
2461
|
# branch=<id> 时主支 + 本分支可见、其他分支仍隐身
|
|
1888
2462
|
and e.get("branch_id") in (None, branch)
|
|
@@ -1890,7 +2464,8 @@ class MdCG:
|
|
|
1890
2464
|
# (非法视图 ValueError——fail-closed 只针对调用方误用)
|
|
1891
2465
|
and (view is None or roleviews.matches(e, view))
|
|
1892
2466
|
# 时效:只在显式启用时排除已过期(not_yet 保留)
|
|
1893
|
-
and not (validity and trust.is_expired(e, now=now))
|
|
2467
|
+
and not (validity and trust.is_expired(e, now=now))):
|
|
2468
|
+
entries.append(e)
|
|
1894
2469
|
|
|
1895
2470
|
# 默认关:索引里可能残留门控字段(曾开启过 / 回填过)→ 返回前剥离,
|
|
1896
2471
|
# 保证候选 entry 形状与「从未启用过本功能」逐字节一致(独立复核 2026-09-19)。
|
|
@@ -1898,8 +2473,12 @@ class MdCG:
|
|
|
1898
2473
|
if (os.environ.get("MDCG_RETRIEVAL_PIPELINE") != "1" and entries
|
|
1899
2474
|
and any(("big_domain" in e) or ("observation_position" in e)
|
|
1900
2475
|
for e in entries)):
|
|
1901
|
-
|
|
1902
|
-
|
|
2476
|
+
# P2-3(批次 31)/ V21-5(批次 34,外部报告定案):**就地剥离**——
|
|
2477
|
+
# entries 里的 e 与 index["nodes"] 是同一对象,_strip_empty_gate_fields
|
|
2478
|
+
# 内部 del 直接落在索引条目上,天然自愈。批次 31 原写法 dict(e) 拷贝
|
|
2479
|
+
# 后按不存在的 "id" 键回写 = 永久 no-op(残留每查都在)。
|
|
2480
|
+
for e in entries:
|
|
2481
|
+
_strip_empty_gate_fields(e, enabled=False)
|
|
1903
2482
|
|
|
1904
2483
|
# ---- 时间算子(阶段二 4.1):候选**资格**过滤(在 S1/S2 收敛之前)----
|
|
1905
2484
|
# 与 validity 各司其职、互不替代:
|
|
@@ -1931,27 +2510,20 @@ class MdCG:
|
|
|
1931
2510
|
# 默认(MDCG_RETRIEVAL_PIPELINE 未设)全关 → 行为与改动前等价。
|
|
1932
2511
|
big_domain = routing.big_domain_classify(terms)
|
|
1933
2512
|
big_scores = routing.big_domain_score_breakdown(terms)
|
|
1934
|
-
gates = {}
|
|
1935
2513
|
# S3 子开关**必须显式 =1**(与 S1/S2 的「未设=开」不同,是有意的非对称):
|
|
1936
2514
|
# S3 会新增候选并引入新的 tier(TIER_SPREAD),在总开关开启时若默认随开,
|
|
1937
2515
|
# 会让「启用 S1/S2」的用户静默多出一层结果——独立复核(2026-09-19)要求显式启用。
|
|
1938
2516
|
# 参数:hops(默认 2)、decay(默认 0.5)、gain(默认 0.2)。
|
|
1939
2517
|
_s3 = (os.environ.get("MDCG_RETRIEVAL_PIPELINE") == "1"
|
|
1940
2518
|
and os.environ.get("MDCG_GATE_S3_SPREAD") == "1")
|
|
1941
|
-
# S4 层级激活优先级:同样必须显式 =1(新层加成会改变排序,属显式启用项)
|
|
1942
|
-
_s4 = (os.environ.get("MDCG_RETRIEVAL_PIPELINE") == "1"
|
|
1943
|
-
and os.environ.get("MDCG_GATE_S4_LAYER") == "1")
|
|
1944
2519
|
# S7 倒排候选层(只作候选生成器;召回与全表扫描一致);见 md_cg/postings.py
|
|
1945
2520
|
_s7 = (os.environ.get("MDCG_RETRIEVAL_PIPELINE") == "1"
|
|
1946
2521
|
and os.environ.get("MDCG_GATE_S7_POSTINGS") == "1")
|
|
1947
2522
|
_s7_narrowed = False
|
|
1948
|
-
|
|
1949
|
-
|
|
1950
|
-
|
|
1951
|
-
|
|
1952
|
-
_l = str(_e.get("layer") or "")
|
|
1953
|
-
_lc[_l] = _lc.get(_l, 0) + 1
|
|
1954
|
-
gates["s4"] = {"boosts": _bo, "layers": _lc}
|
|
2523
|
+
# S1/S1b/S2 收敛 + S4 审计走共享函数(issue #25:原先内联于此,被
|
|
2524
|
+
# MdCGOS.search 覆写悬空——生产路径从未生效;行为与内联版逐字节一致)
|
|
2525
|
+
entries, gates = apply_retrieval_gates(
|
|
2526
|
+
entries, terms, big_domain, context, min_results)
|
|
1955
2527
|
try:
|
|
1956
2528
|
_s3_hops = max(1, int(os.environ.get("MDCG_SPREAD_HOPS", "2")))
|
|
1957
2529
|
except ValueError:
|
|
@@ -1964,91 +2536,6 @@ class MdCG:
|
|
|
1964
2536
|
_s3_gain = min(1.0, max(0.0, float(os.environ.get("MDCG_SPREAD_GAIN", "0.2"))))
|
|
1965
2537
|
except ValueError:
|
|
1966
2538
|
_s3_gain = 0.2
|
|
1967
|
-
if os.environ.get("MDCG_RETRIEVAL_PIPELINE") == "1" and entries:
|
|
1968
|
-
if os.environ.get("MDCG_GATE_S1_DOMAIN", "1") != "0" and big_domain:
|
|
1969
|
-
# 域内 ∪ 未标域(兜底池):无域标签的历史节点绝不能因「域内够多」被丢
|
|
1970
|
-
#(契约 §3 S1 不变量:ORPHAN/未标域必须可被召回)
|
|
1971
|
-
same = [e for e in entries
|
|
1972
|
-
if not e.get("big_domain") or e.get("big_domain") == big_domain]
|
|
1973
|
-
# 召回安全:域内候选不足以支撑 min_results 时不收敛,留全量兜底
|
|
1974
|
-
if len(same) >= max(1, min_results):
|
|
1975
|
-
gates["s1"] = {"domain": big_domain, "in": len(same),
|
|
1976
|
-
"dropped": len(entries) - len(same)}
|
|
1977
|
-
entries = same
|
|
1978
|
-
else:
|
|
1979
|
-
gates["s1"] = {"domain": big_domain, "in": len(same),
|
|
1980
|
-
"dropped": 0, "fallback": "insufficient"}
|
|
1981
|
-
elif os.environ.get("MDCG_GATE_S1_DOMAIN", "1") != "0":
|
|
1982
|
-
gates["s1"] = {"domain": None, "reason": "no_domain_signal"}
|
|
1983
|
-
# ---- S1b 细口径桶收敛(设计见 docs/hive/检索收敛实测与S1b设计_v0.1.md)----
|
|
1984
|
-
# 为何:真实库 91.1% 节点有非 orphan 桶、期望扫描仅 2.1%(细口径),但 T0/T1 桶路
|
|
1985
|
-
# 只在调用方传 context 时可用;普通 search(q) 无 context 就只能全扫。S1b 让 query 侧
|
|
1986
|
-
# 自己推断候选桶(只用索引,不读文件),把这份收敛拿回来。
|
|
1987
|
-
# 召回安全:orphan/无桶节点恒留兜底;命中不足 min_results 即回退(不改 entries)。
|
|
1988
|
-
# 开关与参数就地定义(位于“总开关块”内,不在 S4 块内):S1b 只依赖总开关 + MDCG_GATE_S1B_BUCKET。
|
|
1989
|
-
_s1b = (os.environ.get("MDCG_GATE_S1B_BUCKET") == "1")
|
|
1990
|
-
try:
|
|
1991
|
-
_s1b_topk = int(os.environ.get("MDCG_BUCKET_TOPK", "3"))
|
|
1992
|
-
except ValueError:
|
|
1993
|
-
_s1b_topk = 3
|
|
1994
|
-
try:
|
|
1995
|
-
_s1b_minsim = min(1.0, max(0.0, float(os.environ.get("MDCG_BUCKET_MIN_SIM", "0.34"))))
|
|
1996
|
-
except ValueError:
|
|
1997
|
-
_s1b_minsim = 0.34
|
|
1998
|
-
if _s1b and entries:
|
|
1999
|
-
if _s1b_topk <= 0:
|
|
2000
|
-
gates["s1b"] = {"keys": [], "reason": "disabled_by_topk"}
|
|
2001
|
-
else:
|
|
2002
|
-
_sizes = {}
|
|
2003
|
-
for _e in entries:
|
|
2004
|
-
_b = _e.get("bucket")
|
|
2005
|
-
if _b and _b != routing.ORPHAN:
|
|
2006
|
-
_sizes[_b] = _sizes.get(_b, 0) + 1
|
|
2007
|
-
_best = {}
|
|
2008
|
-
for _b in _sizes:
|
|
2009
|
-
_kk = routing.bucket_key_readable(_b)
|
|
2010
|
-
if not _kk:
|
|
2011
|
-
continue
|
|
2012
|
-
_sim = 0.0
|
|
2013
|
-
for _t in terms:
|
|
2014
|
-
_sv = routing.domain_similarity(_t, _kk)
|
|
2015
|
-
if _sv > _sim:
|
|
2016
|
-
_sim = _sv
|
|
2017
|
-
if _sim >= _s1b_minsim:
|
|
2018
|
-
_best[_b] = _sim
|
|
2019
|
-
if not _best:
|
|
2020
|
-
gates["s1b"] = {"keys": [], "reason": "no_key_match",
|
|
2021
|
-
"in": len(entries), "buckets": len(_sizes)}
|
|
2022
|
-
else:
|
|
2023
|
-
_picked = sorted(_best.items(),
|
|
2024
|
-
key=lambda kv: (-kv[1], -_sizes[kv[0]],
|
|
2025
|
-
str(kv[0])))[:_s1b_topk]
|
|
2026
|
-
_keep = {_b for _b, _ in _picked}
|
|
2027
|
-
_kept = [e for e in entries
|
|
2028
|
-
if (e.get("bucket") in _keep)
|
|
2029
|
-
or not e.get("bucket")
|
|
2030
|
-
or e.get("bucket") == routing.ORPHAN]
|
|
2031
|
-
gates["s1b"] = {"keys": [_b for _b, _ in _picked],
|
|
2032
|
-
"sims": [round(_v, 4) for _, _v in _picked],
|
|
2033
|
-
"in": len(entries), "out": len(_kept),
|
|
2034
|
-
"sizes": {_b: _sizes[_b] for _b, _ in _picked}}
|
|
2035
|
-
if len(_kept) >= max(1, min_results):
|
|
2036
|
-
entries = _kept
|
|
2037
|
-
else:
|
|
2038
|
-
# 回退:entries 保持不变 → 审计的 out 必须记「真实输出规模」,
|
|
2039
|
-
# 命中桶本可保留的数量另存 would_keep(独立复核 2026-09-19 指出口径误导)
|
|
2040
|
-
gates["s1b"]["would_keep"] = len(_kept)
|
|
2041
|
-
gates["s1b"]["out"] = len(entries)
|
|
2042
|
-
gates["s1b"]["fallback"] = "insufficient"
|
|
2043
|
-
if os.environ.get("MDCG_GATE_S2_COND", "1") != "0" and isinstance(context, dict):
|
|
2044
|
-
kept = [e for e in entries if _cond_prefilter_pass(e, context)]
|
|
2045
|
-
gates["s2"] = {"in": len(entries), "out": len(kept),
|
|
2046
|
-
"dropped": len(entries) - len(kept)}
|
|
2047
|
-
# 门控清空则回退(宁多勿漏)
|
|
2048
|
-
if kept:
|
|
2049
|
-
entries = kept
|
|
2050
|
-
else:
|
|
2051
|
-
gates["s2"]["fallback"] = "empty"
|
|
2052
2539
|
if not entries:
|
|
2053
2540
|
_m = {"tier": None, "reason": "no_candidates", "scanned": 0}
|
|
2054
2541
|
if gates: # 默认关时 gates 为空 → 不落键(口径与改动前一致)
|
|
@@ -2058,21 +2545,27 @@ class MdCG:
|
|
|
2058
2545
|
# 负记忆覆盖:查询词是否已被否决议过
|
|
2059
2546
|
neg_coverage = []
|
|
2060
2547
|
if include_neg:
|
|
2061
|
-
|
|
2062
|
-
|
|
2063
|
-
|
|
2064
|
-
|
|
2065
|
-
|
|
2066
|
-
|
|
2067
|
-
|
|
2068
|
-
|
|
2069
|
-
|
|
2070
|
-
|
|
2071
|
-
|
|
2072
|
-
|
|
2073
|
-
|
|
2074
|
-
|
|
2075
|
-
|
|
2548
|
+
# P2-2:遍历第一轮同车收集的引用,不再全索引二遍
|
|
2549
|
+
for e in neg_layer_entries:
|
|
2550
|
+
if e["layer"] not in ("rejected", "unresolved"):
|
|
2551
|
+
continue
|
|
2552
|
+
# _index 用 path 存,直接按 path 读盘(批次 24 P2-1:删除
|
|
2553
|
+
# 原先的死调用 `got = self.get(...)`——结果从未使用却完成
|
|
2554
|
+
# 一次完整读盘+解析+解密,每次检索对每个负记忆节点双倍 IO)。
|
|
2555
|
+
# 批次 33 修复:P2-2 双收集重构时本段被整体缩进进 continue
|
|
2556
|
+
# 之后(不可达死代码)——neg_coverage 恒空 → s5 负记忆抑制
|
|
2557
|
+
# 全失效(test_retr_s5 8/17,Windows/Linux 同红;Docker Linux
|
|
2558
|
+
# 验证暴露后主机复跑定案为既有回归,非平台差异)。
|
|
2559
|
+
full_path = self._node_disk_path(e)
|
|
2560
|
+
if not os.path.exists(full_path):
|
|
2561
|
+
continue
|
|
2562
|
+
try:
|
|
2563
|
+
with open(full_path, encoding="utf-8") as f:
|
|
2564
|
+
fm, content = nodefile.loads(f.read())
|
|
2565
|
+
except OSError:
|
|
2566
|
+
continue
|
|
2567
|
+
if any(t in content for t in terms):
|
|
2568
|
+
neg_coverage.append(e)
|
|
2076
2569
|
|
|
2077
2570
|
stat = {"scanned": 0, "query": q, "gates": gates}
|
|
2078
2571
|
if _tf_audit:
|
|
@@ -2082,12 +2575,27 @@ class MdCG:
|
|
|
2082
2575
|
ctx = context if isinstance(context, dict) else {}
|
|
2083
2576
|
route_bucket = routing.bucket_dir(
|
|
2084
2577
|
routing.route_key(ctx, ctx.get("tags")))
|
|
2578
|
+
# 「无域信号」等价于「没声明条件」:此时**不施加桶约束**。否则 route_key
|
|
2579
|
+
# 回落的 orphan 会被当成一个真桶,下面 T0/T1 阶段只读该桶,并因
|
|
2580
|
+
# valid >= min_results 在 TIER_BUCKET_SCAN 直接 return —— **全量阶梯
|
|
2581
|
+
# 永不执行**,凡被标了 domain:(落在 cond_* 目录)的节点对这条查询
|
|
2582
|
+
# 整片不可见。
|
|
2583
|
+
# 实测(带 context 但不含 domain / observation_position):置 None 前
|
|
2584
|
+
# tier=T1_bucket_scan/scanned=10/目标召回丢失;置 None 后
|
|
2585
|
+
# tier=T2_global_like/scanned=75/召回命中。
|
|
2586
|
+
# 与 mdcos._path_bucket 的「无域信号弃权」(X1)同构,也贴合 test_p0
|
|
2587
|
+
# 的「桶路由平均 recall@10 ≥ 0.9(加 context 不丢召回)」不变量。
|
|
2588
|
+
if route_bucket == routing.ORPHAN:
|
|
2589
|
+
route_bucket = None
|
|
2085
2590
|
|
|
2086
2591
|
# 阶段 1 大域打分已在候选构建前算好(S1 门控要用);此处不再重复计算。
|
|
2087
2592
|
|
|
2088
2593
|
# 生效条件:docs 经 self._score(docs, q, qb, pool_cfg) 后分数 >0 的条数达到闭包阈值 min_results 时返回 self._emit(scored, k, tier, stat, route_bucket, record, len(docs), judge, context, neg_coverage, big_domain, big_scores, pool_cfg)(tier 原样透传);未达阈值返回 None;
|
|
2089
|
-
def try_stage(docs, tier):
|
|
2090
|
-
|
|
2594
|
+
def try_stage(docs, tier, scored=None):
|
|
2595
|
+
# P2-4(批次 31):支持传入已打分结果(reach 分支复用,
|
|
2596
|
+
# 免对同一批文档二次 _score)。
|
|
2597
|
+
if scored is None:
|
|
2598
|
+
scored = self._score(docs, q, qb, pool_cfg)
|
|
2091
2599
|
valid = sum(1 for _, s in scored if s > 0)
|
|
2092
2600
|
if valid >= min_results:
|
|
2093
2601
|
return self._emit(scored, k, tier, stat, route_bucket, record,
|
|
@@ -2134,11 +2642,27 @@ class MdCG:
|
|
|
2134
2642
|
stat["cap"] = GLOBAL_CAP
|
|
2135
2643
|
# 与 T2 **无条件**同序调用(不可按 cap 短路:cut_by_relevance 还会写
|
|
2136
2644
|
# 池账 pool_taken/cands/lost,短路会让 meta.pools.taken 缺失 → test_p43(13) 红)
|
|
2137
|
-
|
|
2645
|
+
# P2-4(批次 31):打分结果经 pre_scored 传给 try_stage,
|
|
2646
|
+
# 不再对同一 hits_r 二次 _score(报告 P2-4)。
|
|
2647
|
+
scored_r = self._score(hits_r, q, qb, pool_cfg)
|
|
2648
|
+
hits_r, _rep = cut_by_relevance(hits_r, scored_r,
|
|
2138
2649
|
GLOBAL_CAP, pools=pool_cfg,
|
|
2139
2650
|
key_of=pooling.doc_key, stat=stat)
|
|
2140
2651
|
pooling.record_audit(stat, _rep)
|
|
2141
|
-
|
|
2652
|
+
# V21-6(批次 34,外部报告定案):doc_key 返回 (node_id, entry),
|
|
2653
|
+
# 第二元是 dict 不可哈希——整键作字典键 = TypeError 死代码。
|
|
2654
|
+
# 取 [0](node_id)作键。缺陷迭代第 14 轮修(两处形态错误,
|
|
2655
|
+
# MDCG_REACH=1 实测 KeyError: 1 崩,stash 取证 HEAD 原生在位):
|
|
2656
|
+
# ① scored 元素是 (card, score),card 是 dict——对它调 doc_key
|
|
2657
|
+
# (形参须 (entry, fm, content) 三元组)= KeyError: 1;按
|
|
2658
|
+
# card["id"](与 doc_key(doc)[0] 同源同值)建键表;
|
|
2659
|
+
# ② 键表须存 (card, score) 原对再按 hits_r 序取回——自组
|
|
2660
|
+
# (doc, score) 对会让 _emit 排序键 x[0]["frontmatter"] 崩
|
|
2661
|
+
# (x[0] 须是 card dict)。
|
|
2662
|
+
_smap = {c.get("id"): (c, s) for c, s in scored_r}
|
|
2663
|
+
out = try_stage(hits_r, TIER_GLOBAL_LIKE,
|
|
2664
|
+
scored=[_smap[pooling.doc_key(d)[0]]
|
|
2665
|
+
for d in hits_r])
|
|
2142
2666
|
if out:
|
|
2143
2667
|
return out
|
|
2144
2668
|
# 收敛阶段未产出结果 → 继续走下方全量 T2/T3;此时必须**改写 reach 审计**,
|
|
@@ -2384,9 +2908,11 @@ class MdCG:
|
|
|
2384
2908
|
_boost = layer_boosts() if _s4_on else None
|
|
2385
2909
|
scored = []
|
|
2386
2910
|
for e, fm, c in docs:
|
|
2387
|
-
# 归一化 content 后取 bigram(与 query 侧 normalize_en
|
|
2388
|
-
|
|
2389
|
-
|
|
2911
|
+
# 归一化 content 后取 bigram(与 query 侧 normalize_en 对称);
|
|
2912
|
+
# 走可覆写钩子(批次 21,issue #31):内容不变则派生物不变——
|
|
2913
|
+
# readcache 启用时覆写为缓存版(Rust load_docs 预计算同款),
|
|
2914
|
+
# 基类默认即算即弃,行为与改动前逐位一致
|
|
2915
|
+
nb = self._doc_norm_bigrams(e, c)
|
|
2390
2916
|
sim = lexical_sim(qb, nb, mode)
|
|
2391
2917
|
tag_bonus = 0.05 if any(str(t) in q or q in str(t)
|
|
2392
2918
|
for t in (fm.get("tags") or [])) else 0.0
|
|
@@ -2513,11 +3039,16 @@ class MdCG:
|
|
|
2513
3039
|
out.append((r[0], r[1], qual))
|
|
2514
3040
|
# 把被负记忆覆盖的查询也作为结果条目返回(首条),方便调用方感知
|
|
2515
3041
|
for nc in neg_coverage[:3]:
|
|
2516
|
-
|
|
2517
|
-
|
|
2518
|
-
|
|
2519
|
-
|
|
2520
|
-
|
|
3042
|
+
# v9 留档残余项修复(缺陷迭代第 14 轮):改走 self._read——裸
|
|
3043
|
+
# open+loads 不在读缓存包装面(install 只包 cg._read),每条命中
|
|
3044
|
+
# 负层的查询恒 ≤3 次盘读(60+8 池实测 Q2 增量恰 3);并入读缓存
|
|
3045
|
+
# 后增量归零,负层主面 IO 早已同款覆盖(_neg_coverage → _read)。
|
|
3046
|
+
# _read 同款 _node_disk_path 边界闸(P2-20:索引被污染时不得绕过
|
|
3047
|
+
# 统一校验读 root 外文件)、OSError→(None,None) 与旧 continue 同义;
|
|
3048
|
+
# 负层写路径(add/add_rejected→_stage 标脏带 path)在脏集精确
|
|
3049
|
+
# 失效面内——覆写后尾部条目即时见新内容,不陈旧。
|
|
3050
|
+
fm, content = self._read(nc)
|
|
3051
|
+
if content is None:
|
|
2521
3052
|
continue
|
|
2522
3053
|
entry = {"id": nc["path"], "frontmatter": fm,
|
|
2523
3054
|
"content": content, "path": nc["path"]}
|
|
@@ -2700,14 +3231,17 @@ class MdCG:
|
|
|
2700
3231
|
return {"id": node_id, "from": from_layer, "to": to_layer,
|
|
2701
3232
|
"path": rel, "reason": reason}
|
|
2702
3233
|
|
|
2703
|
-
# 生效条件:verdict 不在 {confirmed,weakened,falsified} 时抛 ValueError;node_id 取不到节点返回 None;verdict=="falsified"
|
|
2704
|
-
def verify(self, node_id: str, evidence: str, verdict: str
|
|
3234
|
+
# 生效条件:verdict 不在 {confirmed,weakened,falsified} 时抛 ValueError;node_id 取不到节点返回 None;verdict=="falsified" 时先经 protect.guard_forget(受保护节点需显式 override=True——旧版本自动快照+留痕;拒绝抛 ProtectionError 且无任何副作用),通过后以 content[:200] 为假设写入 rejected 层、删除原文件并 _unstage,返回 {"action":"falsified","new_id":None,"evidence_count":None,"demoted":None};confirmed/weakened 时 confidence 分别 +0.05 / -0.15(夹到 [0,0.99] 并 round 2)、evidence_count 与 positive_evidence/negative_evidence 各 +1、追加 evidence_log,且仅当 weakened 后 confidence < DEMOTE_CONFIDENCE 且 layer=="knowledge" 时调 _move_layer 到 contextual 并 set_state("demoted"),否则 _write_node 回写并同步索引 evidence_count;
|
|
3235
|
+
def verify(self, node_id: str, evidence: str, verdict: str,
|
|
3236
|
+
override: bool = False):
|
|
2705
3237
|
"""验证单元(白箱第 3 篇第 13 章):对节点做一次外部验证裁决。
|
|
2706
3238
|
|
|
2707
3239
|
verdict ∈ {"confirmed", "weakened", "falsified"}
|
|
2708
3240
|
- confirmed:positive_evidence+1,confidence 上调 +0.05
|
|
2709
3241
|
- weakened:negative_evidence+1,confidence 下调 -0.15(反例权重更大)
|
|
2710
|
-
- falsified:节点移入 rejected/(幂等:相同 hypothesis
|
|
3242
|
+
- falsified:节点移入 rejected/(幂等:相同 hypothesis 不重复);
|
|
3243
|
+
受保护节点(self/anchor 层、protected 标记、importance≥0.7)
|
|
3244
|
+
不可一键证伪删除——需显式 override=True(先快照+留痕)
|
|
2711
3245
|
|
|
2712
3246
|
每次裁决累加 evidence_count(白箱第 5 篇第 4 章)。
|
|
2713
3247
|
weakened 后 confidence 跌破 DEMOTE_CONFIDENCE 且原属 knowledge 层 →
|
|
@@ -2719,6 +3253,15 @@ class MdCG:
|
|
|
2719
3253
|
if not node:
|
|
2720
3254
|
return None
|
|
2721
3255
|
if verdict == "falsified":
|
|
3256
|
+
# 写保护(N130,2026-09-25):证伪 = 对原节点的一次**硬删除**
|
|
3257
|
+
# (os.remove,无快照、不进 trash),与 forget/降级同受「不可遗忘」
|
|
3258
|
+
# 保护——self/anchor 层、protected 标记、importance≥0.7 不得被一次
|
|
3259
|
+
# verify 调用一键删除。对照 forget 路径(mdcos.py forget)同一闸口:
|
|
3260
|
+
# 受保护节点需显式 override=True(旧版本先快照进 _protected_history
|
|
3261
|
+
# 并留痕 _protected_audit.jsonl),拒绝抛 ProtectionError、原节点
|
|
3262
|
+
# 原样保留。守卫必须在 add_rejected 之前——拒绝路径零副作用。
|
|
3263
|
+
protect.guard_forget(self, node_id, override=override,
|
|
3264
|
+
actor=getattr(self, "actor", None))
|
|
2722
3265
|
# 移入 rejected:负记忆化
|
|
2723
3266
|
self.add_rejected(hypothesis=node["content"][:200],
|
|
2724
3267
|
reason=evidence,
|
|
@@ -2766,6 +3309,13 @@ class MdCG:
|
|
|
2766
3309
|
e = self.index["nodes"].get(node_id)
|
|
2767
3310
|
if e is not None:
|
|
2768
3311
|
e["evidence_count"] = fm["evidence_count"]
|
|
3312
|
+
# 落盘必须对索引增量日志(_dirty→flush→_index_log 重放)与
|
|
3313
|
+
# 读缓存代际可见:本分支写盘后若节点已在目标验证态,尾部
|
|
3314
|
+
# set_verification 走 noop 提前返回(trust.py stamp),不会
|
|
3315
|
+
# 经 _sync_index 标脏——不显式置 _dirty 会造成重开后索引
|
|
3316
|
+
# evidence_count 与盘面漂移、同实例 _read 命中旧 frontmatter
|
|
3317
|
+
#(口径同边域同步 _sync_edge_entry / goal 状态写回)。
|
|
3318
|
+
self._dirty[node_id] = e
|
|
2769
3319
|
# 验证态流转(可验证记忆单元):外部证据裁决**就是**验证态迁移。
|
|
2770
3320
|
# confirmed → verified(首次转正;已在 verified 则幂等 no-op)
|
|
2771
3321
|
# weakened → doubted(证据被削弱 ⇒ 存疑,待复核)
|
|
@@ -2868,7 +3418,7 @@ class MdCG:
|
|
|
2868
3418
|
continue
|
|
2869
3419
|
fm["access_count"] = int(fm.get("access_count") or 0) + c
|
|
2870
3420
|
fm["last_access"] = max(float(fm.get("last_access") or 0), last.get(nid, 0))
|
|
2871
|
-
self._write_node(nid,
|
|
3421
|
+
self._write_node(nid, self._node_disk_path(e), fm, content)
|
|
2872
3422
|
n += 1
|
|
2873
3423
|
with FileLock(self.access_log):
|
|
2874
3424
|
atomic_write(self.access_log, "")
|
|
@@ -2885,7 +3435,7 @@ class MdCG:
|
|
|
2885
3435
|
layer_stats = {}
|
|
2886
3436
|
neg_stats = {}
|
|
2887
3437
|
for e in self.index["nodes"].values():
|
|
2888
|
-
full_path =
|
|
3438
|
+
full_path = self._node_disk_path(e)
|
|
2889
3439
|
try:
|
|
2890
3440
|
with open(full_path, encoding="utf-8") as f:
|
|
2891
3441
|
fm, content = nodefile.loads(f.read())
|