@furongjun1999/dsh-memory 0.8.1 → 0.9.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (717) hide show
  1. package/README.md +662 -659
  2. package/codebuddy/README.md +1 -1
  3. package/data/policy.json +38 -27
  4. package/docs/README.md +18 -2
  5. package/docs/discipline/harnesses.yaml +1 -1
  6. package/docs/discipline/templates/spec.txt.tmpl +23 -0
  7. package/docs//350/247/204/350/214/203/AI/347/274/226/347/250/213/351/200/232/347/224/250/350/247/204/350/214/203.md +179 -0
  8. package/docs//350/247/204/350/214/203/AI/347/274/226/347/250/213/351/200/232/347/224/250/350/247/204/350/214/203_/347/201/265/346/236/242/351/200/202/351/205/215.md +45 -0
  9. package/docs//350/247/204/350/214/203/AI/347/274/226/347/250/213/351/200/232/347/224/250/350/247/204/350/214/203_/350/207/252/346/243/200/346/270/205/345/215/225.md +69 -0
  10. package/dsh/README.md +1 -1
  11. package/dsh/cordis.yml.example +139 -139
  12. package/hive/README.md +705 -0
  13. package/hive/id_charset_blocks.txt +147 -0
  14. package/lib/bridge.d.ts +3 -0
  15. package/lib/bridge.js +32 -11
  16. package/lib/cli.js +14 -17
  17. package/lib/hooks.d.ts +37 -3
  18. package/lib/hooks.js +176 -24
  19. package/lib/index.d.ts +2 -2
  20. package/lib/index.js +29 -6
  21. package/lib/init.js +21 -21
  22. package/lib/lib/datapath.d.ts +56 -0
  23. package/lib/lib/datapath.js +102 -9
  24. package/lib/lib/hook_audit.d.ts +0 -1
  25. package/lib/lib/hook_audit.js +51 -3
  26. package/lib/lib/mdcg_client.d.ts +4 -1
  27. package/lib/lib/mdcg_client.js +10 -2
  28. package/lib/lib/mutual.js +61 -15
  29. package/lib/lib/prompt_safety.js +28 -2
  30. package/lib/lib/roleplay_web.js +30 -2
  31. package/lib/lib/token_store.d.ts +25 -6
  32. package/lib/lib/token_store.js +224 -33
  33. package/lib/tools.d.ts +15 -2
  34. package/lib/tools.js +44 -4
  35. package/md_cg/_guard_kit.py +121 -0
  36. package/md_cg/admission.py +718 -718
  37. package/md_cg/anchor.py +114 -0
  38. package/md_cg/audit.py +37 -7
  39. package/md_cg/autonomy.py +433 -288
  40. package/md_cg/autonomy_modes.py +781 -642
  41. package/md_cg/bench_beir_bm25.py +154 -0
  42. package/md_cg/bench_beir_scifact.py +170 -0
  43. package/md_cg/bench_bm25_blockmax.py +134 -0
  44. package/md_cg/bench_bm25_incremental.py +246 -0
  45. package/md_cg/bench_ccg_heading.py +118 -0
  46. package/md_cg/bench_csre_correctness.py +145 -0
  47. package/md_cg/bench_excerpt_budget.py +94 -0
  48. package/md_cg/bench_fuzzy_substrings.py +299 -0
  49. package/md_cg/bench_memory_rsi.py +358 -0
  50. package/md_cg/bench_memory_rsi_latency.py +82 -0
  51. package/md_cg/bench_memory_rsi_public.py +116 -0
  52. package/md_cg/bench_memory_rsi_tools.py +229 -0
  53. package/md_cg/bench_rrf_candidates.py +212 -0
  54. package/md_cg/bench_rrf_public.py +100 -0
  55. package/md_cg/bench_text_projections.py +221 -0
  56. package/md_cg/bench_timeanchor.py +357 -0
  57. package/md_cg/bm25.py +601 -0
  58. package/md_cg/branches.py +404 -327
  59. package/md_cg/build_postings.py +74 -73
  60. package/md_cg/ccgc.py +4 -2
  61. package/md_cg/chain.py +573 -379
  62. package/md_cg/coldverify.py +531 -292
  63. package/md_cg/condaccept.py +263 -0
  64. package/md_cg/condgate.py +308 -0
  65. package/md_cg/config_registry.py +329 -0
  66. package/md_cg/config_registry_bulk.py +482 -0
  67. package/md_cg/config_registry_guard.py +487 -0
  68. package/md_cg/config_render.py +354 -0
  69. package/md_cg/config_validate.py +455 -0
  70. package/md_cg/conformance.py +747 -726
  71. package/md_cg/conformance_core.py +753 -0
  72. package/md_cg/consistency.py +1278 -955
  73. package/md_cg/consolidate.py +1793 -1633
  74. package/md_cg/crypto.py +346 -24
  75. package/md_cg/datapath.py +137 -12
  76. package/md_cg/eval_common.py +522 -575
  77. package/md_cg/evidence.py +946 -585
  78. package/md_cg/evidence_delivery.py +101 -0
  79. package/md_cg/experience_consolidation.py +317 -0
  80. package/md_cg/export.py +329 -222
  81. package/md_cg/forgetting.py +1050 -922
  82. package/md_cg/freshness.py +601 -548
  83. package/md_cg/fsutil.py +1120 -824
  84. package/md_cg/goal_gen.py +458 -0
  85. package/md_cg/hotcache.py +107 -9
  86. package/md_cg/hyperedge.py +262 -251
  87. package/md_cg/identity.py +573 -391
  88. package/md_cg/imgskill.py +1712 -0
  89. package/md_cg/insight.py +540 -521
  90. package/md_cg/interface_schema.py +146 -0
  91. package/md_cg/interop.py +20 -0
  92. package/md_cg/judgment_manifest.py +24 -1
  93. package/md_cg/lifecycle.py +426 -345
  94. package/md_cg/links.py +27 -6
  95. package/md_cg/logref.py +332 -327
  96. package/md_cg/mcp_server.py +5227 -4449
  97. package/md_cg/mdcg.py +1301 -203
  98. package/md_cg/mdcos.py +7108 -5440
  99. package/md_cg/memory_rsi.py +308 -0
  100. package/md_cg/metacognition.py +726 -591
  101. package/md_cg/nodefile.py +151 -3
  102. package/md_cg/op_table.py +1128 -0
  103. package/md_cg/postings.py +524 -300
  104. package/md_cg/predict.py +1103 -1100
  105. package/md_cg/probe_memory_rsi.py +95 -0
  106. package/md_cg/probe_memory_rsi_mcp.py +66 -0
  107. package/md_cg/protect.py +654 -494
  108. package/md_cg/protocol.py +492 -411
  109. package/md_cg/provenance.py +606 -600
  110. package/md_cg/qmd.py +477 -0
  111. package/md_cg/qmd_llm.py +334 -0
  112. package/md_cg/query_excerpt.py +171 -0
  113. package/md_cg/queue_health.py +130 -0
  114. package/md_cg/reach.py +470 -453
  115. package/md_cg/readcache.py +115 -3
  116. package/md_cg/reconcile.py +239 -228
  117. package/md_cg/refindex.py +1191 -1042
  118. package/md_cg/refine.py +613 -604
  119. package/md_cg/retrieve.py +128 -0
  120. package/md_cg/review_cli.py +204 -26
  121. package/md_cg/revise.py +140 -0
  122. package/md_cg/rollback.py +657 -420
  123. package/md_cg/rollback_cli.py +124 -104
  124. package/md_cg/rotate.py +347 -0
  125. package/md_cg/routing.py +490 -429
  126. package/md_cg/rrf_candidates.py +204 -0
  127. package/md_cg/run_tests.py +19 -1
  128. package/md_cg/scrub.py +177 -23
  129. package/md_cg/security.py +269 -17
  130. package/md_cg/self_state.py +1053 -1029
  131. package/md_cg/self_surface.py +50 -0
  132. package/md_cg/semantic/canonical.py +144 -144
  133. package/md_cg/semantic/export_en_zh_map.py +2 -2
  134. package/md_cg/semantic/zh_en_atoms.py +141 -139
  135. package/md_cg/session5.py +106 -0
  136. package/md_cg/signer.py +76 -2
  137. package/md_cg/sleep.py +1779 -1583
  138. package/md_cg/sources.py +1091 -833
  139. package/md_cg/state_events.py +75 -10
  140. package/md_cg/state_slots.py +312 -221
  141. package/md_cg/stg.py +85 -18
  142. package/md_cg/stgidx.py +537 -281
  143. package/md_cg/subgraph.py +851 -729
  144. package/md_cg/sustain.py +458 -19
  145. package/md_cg/task_memory.py +367 -0
  146. package/md_cg/test_action_derive.py +210 -203
  147. package/md_cg/test_audit_heartbeat_20261015.py +417 -0
  148. package/md_cg/test_audit_protection_20261015.py +483 -0
  149. package/md_cg/test_audit_rotate_parity.py +280 -0
  150. package/md_cg/test_audit_state_20261015.py +253 -0
  151. package/md_cg/test_audit_write_20261015.py +161 -0
  152. package/md_cg/test_auto_defaults.py +43 -4
  153. package/md_cg/test_autonomy_emotion.py +328 -0
  154. package/md_cg/test_autonomy_modes.py +2013 -1973
  155. package/md_cg/test_b1_foreign_read.py +245 -0
  156. package/md_cg/test_bm25.py +438 -0
  157. package/md_cg/test_bm25_hydration_refill.py +265 -0
  158. package/md_cg/test_bm25_incremental.py +241 -0
  159. package/md_cg/test_bm25_incremental_mutations.py +160 -0
  160. package/md_cg/test_bm25_mutations.py +205 -0
  161. package/md_cg/test_bm25_pool_view.py +196 -0
  162. package/md_cg/test_boot_openblas_guard.py +218 -197
  163. package/md_cg/test_boundary_hit.py +1 -1
  164. package/md_cg/test_branches_sensitivity.py +375 -0
  165. package/md_cg/test_c8_search_rrf_gates.py +1 -1
  166. package/md_cg/test_canonical_timezones.py +211 -0
  167. package/md_cg/test_ccg_heading_parser.py +73 -0
  168. package/md_cg/test_chain_best_first.py +403 -0
  169. package/md_cg/test_chain_candidate_scope.py +227 -0
  170. package/md_cg/test_cn_recall_grams.py +206 -0
  171. package/md_cg/test_code_kind_forbidden.py +101 -0
  172. package/md_cg/test_coldverify_drain.py +127 -0
  173. package/md_cg/test_condaccept.py +156 -0
  174. package/md_cg/test_condgate.py +141 -0
  175. package/md_cg/test_config_registry.py +55 -0
  176. package/md_cg/test_crypto_rotate.py +257 -0
  177. package/md_cg/test_csre_correctness.py +297 -0
  178. package/md_cg/test_csre_index_import_guard.py +161 -0
  179. package/md_cg/test_csre_mutations.py +34 -0
  180. package/md_cg/test_csre_session.py +69 -0
  181. package/md_cg/test_curiosity_budget.py +468 -0
  182. package/md_cg/test_d17_autostart_toolface.py +453 -0
  183. package/md_cg/test_datapath_tilde_home.py +271 -0
  184. package/md_cg/test_derived_cache_generations.py +256 -0
  185. package/md_cg/test_evidence_delivery_compact.py +54 -0
  186. package/md_cg/test_exact_text_cache.py +191 -0
  187. package/md_cg/test_excerpt_budget.py +188 -0
  188. package/md_cg/test_experience_consolidation.py +112 -0
  189. package/md_cg/test_export_default_collision.py +138 -0
  190. package/md_cg/test_export_snapshots.py +322 -0
  191. package/md_cg/test_fuzzy_substring_equivalence.py +127 -0
  192. package/md_cg/test_gain_gate.py +16 -5
  193. package/md_cg/test_goal_gen.py +355 -0
  194. package/md_cg/test_h2_session_view_norm.py +251 -233
  195. package/md_cg/test_h4_sustain_snapshot.py +1490 -1387
  196. package/md_cg/test_heartbeat_ledger.py +557 -0
  197. package/md_cg/test_hierarchy_audit_graph.py +237 -0
  198. package/md_cg/test_i50c_meta_passthrough.py +628 -608
  199. package/md_cg/test_i50e_readside_protection.py +658 -656
  200. package/md_cg/test_identity_attribution.py +96 -2
  201. package/md_cg/test_identity_contract.py +329 -0
  202. package/md_cg/test_imgskill.py +1444 -0
  203. package/md_cg/test_index_snapshot_shape.py +240 -0
  204. package/md_cg/test_interop_judgment.py +258 -228
  205. package/md_cg/test_issue111_gate_order.py +77 -0
  206. package/md_cg/test_issue111_scoring_contract.py +87 -0
  207. package/md_cg/test_issue121_calibration_bounds.py +293 -0
  208. package/md_cg/test_issue128_session_env_authority.py +246 -0
  209. package/md_cg/test_issue130_rrf_rank_credibility.py +217 -0
  210. package/md_cg/test_issue131_postings_content_staleness.py +424 -0
  211. package/md_cg/test_issue145_sleep_no_silent_rollback.py +168 -0
  212. package/md_cg/test_issue146_frontmatter_quarantine.py +428 -0
  213. package/md_cg/test_issue147_pack_id_escape.py +378 -0
  214. package/md_cg/test_issue148_merge_downgrade_leak.py +192 -0
  215. package/md_cg/test_issue149_accel_default_face.py +249 -0
  216. package/md_cg/test_issue150_empty_query_terms.py +328 -0
  217. package/md_cg/test_issue151_153_157_write_cache_index.py +152 -0
  218. package/md_cg/test_issue151b_base_add_importance_guard.py +166 -0
  219. package/md_cg/test_issue152_sealed_postings.py +309 -0
  220. package/md_cg/test_issue152b_decrypt_cache.py +218 -0
  221. package/md_cg/test_issue153b_cache_alias_and_canon.py +236 -0
  222. package/md_cg/test_issue153b_index_generation_invalidation.py +143 -0
  223. package/md_cg/test_issue157b_index_remainder.py +344 -0
  224. package/md_cg/test_issue189_conclusion_bidirectional.py +223 -0
  225. package/md_cg/test_issue190_tidy_idempotent.py +806 -0
  226. package/md_cg/test_issue191_import_trust.py +382 -0
  227. package/md_cg/test_issue192_move_layer_indices_guard.py +258 -0
  228. package/md_cg/test_issue193_consistency_knowledge_guard.py +524 -0
  229. package/md_cg/test_issue193_write_scope.py +378 -0
  230. package/md_cg/test_issue197_rank_credibility_property.py +201 -0
  231. package/md_cg/test_issue211_export_target.py +325 -0
  232. package/md_cg/test_issue212_writeface_gaps.py +349 -0
  233. package/md_cg/test_issue214_reject_sensitivity.py +267 -0
  234. package/md_cg/test_issue215_condition_source_gap.py +272 -0
  235. package/md_cg/test_issue216_clash_routes.py +214 -0
  236. package/md_cg/test_issue217_slot_routes.py +248 -0
  237. package/md_cg/test_issue218_postcondition_value_blind.py +222 -0
  238. package/md_cg/test_issue225_consistency_torn_log.py +433 -0
  239. package/md_cg/test_issue245_swarm_traversal.py +366 -0
  240. package/md_cg/test_issue258_pack_sensitivity.py +468 -0
  241. package/md_cg/test_issue265_coldverify_multiproc.py +438 -0
  242. package/md_cg/test_issue274_read_node_id_none.py +341 -0
  243. package/md_cg/test_issue275_layer_param.py +250 -0
  244. package/md_cg/test_issue277_node_id_traversal.py +384 -0
  245. package/md_cg/test_issue278_neg_separated_tension.py +271 -0
  246. package/md_cg/test_issue278_neg_window.py +294 -0
  247. package/md_cg/test_issue278_negative_layering.py +83 -0
  248. package/md_cg/test_issue279_signers_path_guard.py +652 -0
  249. package/md_cg/test_issue284_skip_not_pass.py +291 -0
  250. package/md_cg/test_issue294_flywheel_sensitivity.py +573 -0
  251. package/md_cg/test_issue295_sleep_root_binding.py +589 -0
  252. package/md_cg/test_issue297_rewrite_sensitivity.py +454 -0
  253. package/md_cg/test_issue298_consistency_ledger_sensitivity.py +421 -0
  254. package/md_cg/test_issue301_baseline_corpus_gate.py +245 -0
  255. package/md_cg/test_issue303_design_boundaries.py +262 -0
  256. package/md_cg/test_issue313_writelimit_sensitivity.py +404 -0
  257. package/md_cg/test_issue315_merge_branch_sensitivity.py +384 -0
  258. package/md_cg/test_issue315_sensitivity_passthrough_guard.py +514 -0
  259. package/md_cg/test_issue319_tenant_unregistered_bypass.py +257 -0
  260. package/md_cg/test_issue320_vstate_requires_verify.py +279 -0
  261. package/md_cg/test_issue321_mcp_deep_json.py +305 -0
  262. package/md_cg/test_issue322_node_write_lock.py +727 -0
  263. package/md_cg/test_issue323_aggregate_segment_boundary.py +365 -0
  264. package/md_cg/test_issue324_write_size_fields.py +215 -0
  265. package/md_cg/test_issue325_audit_poison.py +230 -0
  266. package/md_cg/test_issue326_append_jsonl_oserror.py +464 -0
  267. package/md_cg/test_issue326_ingest_auto_whitelist.py +331 -0
  268. package/md_cg/test_issue326_key_perms.py +355 -0
  269. package/md_cg/test_issue326_twin_parser_parity.py +323 -0
  270. package/md_cg/test_issue326_vm_call_frame.py +569 -0
  271. package/md_cg/test_issue345_writelimit_effective_sens.py +234 -0
  272. package/md_cg/test_issue346_overwrite_sensitivity_gate.py +424 -0
  273. package/md_cg/test_issue346_review_overwrite_guard.py +509 -0
  274. package/md_cg/test_issue348_insight_verify_op.py +200 -0
  275. package/md_cg/test_issue348_merge_rmw_lock.py +345 -0
  276. package/md_cg/test_issue350_casefold_collision.py +236 -0
  277. package/md_cg/test_issue351_ingest_root_fallback.py +202 -0
  278. package/md_cg/test_issue352_imgaudit.py +418 -0
  279. package/md_cg/test_issue352_pathsjson.py +537 -0
  280. package/md_cg/test_issue352_refindex.py +412 -0
  281. package/md_cg/test_issue352_rollback.py +604 -0
  282. package/md_cg/test_issue352_vstateinherit.py +196 -0
  283. package/md_cg/test_issue354_p1_writelimit_type.py +522 -0
  284. package/md_cg/test_issue354_p2_datapath_clock.py +381 -0
  285. package/md_cg/test_issue354_p2_writelimit_toctou.py +327 -0
  286. package/md_cg/test_issue354_rust_index_containment.py +571 -0
  287. package/md_cg/test_issue354_rust_wal_containment.py +606 -0
  288. package/md_cg/test_issue358_truncation_observable.py +247 -0
  289. package/md_cg/test_issue360_tidy_private_excerpt.py +196 -0
  290. package/md_cg/test_issue361_blindspot_not_silent_drop.py +494 -0
  291. package/md_cg/test_issue362_neg_condition.py +468 -0
  292. package/md_cg/test_issue363_lexical_idf.py +275 -0
  293. package/md_cg/test_issue370_readcache_assembly.py +488 -0
  294. package/md_cg/test_issue375_state_slot_retire.py +325 -0
  295. package/md_cg/test_issue376_heartbeat_failed.py +293 -0
  296. package/md_cg/test_issue377_layer_protect_stamp.py +324 -0
  297. package/md_cg/test_issue378_protected_floor.py +310 -0
  298. package/md_cg/test_issue383_state_event_write_gate.py +462 -0
  299. package/md_cg/test_issue397_sens_eff_wiring.py +435 -0
  300. package/md_cg/test_issue398_toolface_op_parity.py +511 -0
  301. package/md_cg/test_issue399_hive_json_shape.py +317 -0
  302. package/md_cg/test_issue400_hive_dirty_args.py +450 -0
  303. package/md_cg/test_issue401_wisdom_codegen.py +295 -0
  304. package/md_cg/test_issue404_heal_receipt.py +390 -0
  305. package/md_cg/test_issue405_git_pathspec.py +333 -0
  306. package/md_cg/test_issue408_eface_registry.py +291 -0
  307. package/md_cg/test_issue409_compact_index_completeness.py +509 -0
  308. package/md_cg/test_issue410_verify_demote_write.py +366 -0
  309. package/md_cg/test_issue411_scrub_idempotence.py +575 -0
  310. package/md_cg/test_issue412_discipline_producer_parity.py +312 -0
  311. package/md_cg/test_issue413_preimage_failclosed.py +239 -0
  312. package/md_cg/test_issue413_preimage_twin.py +340 -0
  313. package/md_cg/test_issue414_write_args_parity.py +613 -0
  314. package/md_cg/test_issue415_deps_gate_probe.py +358 -0
  315. package/md_cg/test_issue416_entry_key_subset.py +525 -0
  316. package/md_cg/test_issue417_twophase_merge_landed.py +423 -0
  317. package/md_cg/test_issue418_identity_role_type_gate.py +188 -0
  318. package/md_cg/test_issue418_role_type_gate.py +265 -0
  319. package/md_cg/test_issue419_pii_redaction.py +642 -0
  320. package/md_cg/test_issue424_merge_confirm_sensitivity.py +381 -0
  321. package/md_cg/test_issue425_ingest_library_path.py +331 -0
  322. package/md_cg/test_issue426a_dedup_receipt.py +242 -0
  323. package/md_cg/test_issue426b_root_clash_invisible.py +270 -0
  324. package/md_cg/test_issue426c_enqueue_not_silent.py +346 -0
  325. package/md_cg/test_issue426e_restore_overwrite_live.py +310 -0
  326. package/md_cg/test_issue426f_paths_gbk_failclosed.py +301 -0
  327. package/md_cg/test_issue427_serve_status_shape.py +191 -0
  328. package/md_cg/test_issue431_432_468_protect_single_point.py +322 -0
  329. package/md_cg/test_issue433_outlook_threshold_single_point.py +278 -0
  330. package/md_cg/test_issue434_435_float_gate.py +90 -0
  331. package/md_cg/test_issue439_sibling_sensitivity_inherit.py +448 -0
  332. package/md_cg/test_issue43_default_policy.py +874 -865
  333. package/md_cg/test_issue443a_state_event_length_gate.py +94 -0
  334. package/md_cg/test_issue443b_audit_record_length_gate.py +83 -0
  335. package/md_cg/test_issue443c_self_state_imgskill_length_gate.py +77 -0
  336. package/md_cg/test_issue444_scrub_evidence_plaintext.py +281 -0
  337. package/md_cg/test_issue445_metacognition_ledger_gate.py +454 -0
  338. package/md_cg/test_issue446_453_403_verified_evidence.py +382 -0
  339. package/md_cg/test_issue446_is_encrypted_envelope.py +166 -0
  340. package/md_cg/test_issue447_recent_sens_failclosed.py +270 -0
  341. package/md_cg/test_issue450_sensitivity_fallback_reason.py +51 -0
  342. package/md_cg/test_issue450_writegate_unknown_sens.py +203 -0
  343. package/md_cg/test_issue451_audit_actor.py +313 -0
  344. package/md_cg/test_issue452_ingest_root_hint.py +238 -0
  345. package/md_cg/test_issue452_ingest_root_sensitivity.py +299 -0
  346. package/md_cg/test_issue463_ghost_prune_guard.py +259 -0
  347. package/md_cg/test_issue466_basis_registry.py +469 -0
  348. package/md_cg/test_issue467_identity_sensitivity.py +204 -0
  349. package/md_cg/test_issue467_identity_sensitivity_parity.py +361 -0
  350. package/md_cg/test_issue469_scrub_override_passthrough.py +126 -0
  351. package/md_cg/test_issue470_wal_event_durable.py +427 -0
  352. package/md_cg/test_issue471_identity_readable.py +137 -0
  353. package/md_cg/test_issue472_audit_chain.py +91 -0
  354. package/md_cg/test_issue472_prune_ledger.py +106 -0
  355. package/md_cg/test_issue472_rollback_degraded.py +82 -0
  356. package/md_cg/test_issue491_compose_sandbox.py +460 -0
  357. package/md_cg/test_issue492_dup_readout_opt_in.py +292 -0
  358. package/md_cg/test_issue498_merge_trace_stale_index.py +280 -0
  359. package/md_cg/test_issue511_scoreboard_bad_files.py +256 -0
  360. package/md_cg/test_issue521_rotate_lock_suffix.py +110 -0
  361. package/md_cg/test_issue522_review_record_sensitivity.py +267 -0
  362. package/md_cg/test_issue524_orphan_noise_no_demote.py +88 -0
  363. package/md_cg/test_issue525_read_null_semantics.py +52 -0
  364. package/md_cg/test_issue538_hook_condition_space.py +97 -0
  365. package/md_cg/test_issue540_path_guard_whitespace.py +126 -0
  366. package/md_cg/test_issue541_tidy_protect_gate.py +141 -0
  367. package/md_cg/test_issue542_tenant_root_same.py +153 -0
  368. package/md_cg/test_issue543_issue_authorization.py +147 -0
  369. package/md_cg/test_issue544_sandbox_reflection_gate.py +175 -0
  370. package/md_cg/test_issue545_charset_vs_size_diagnostic.py +77 -0
  371. package/md_cg/test_issue546_ledger_concurrency.py +81 -0
  372. package/md_cg/test_issue547_append_jsonl_whitelist.py +115 -0
  373. package/md_cg/test_issue547_audit_durable.py +96 -0
  374. package/md_cg/test_issue549_durable_append.py +143 -0
  375. package/md_cg/test_issue557_r1_credential_value_forms.py +76 -0
  376. package/md_cg/test_issue557_r2_write_entries_size_gate.py +84 -0
  377. package/md_cg/test_issue557_r3_pem_block_redaction.py +75 -0
  378. package/md_cg/test_issue559_refine_prefix.py +70 -0
  379. package/md_cg/test_issue561_pending_review.py +109 -0
  380. package/md_cg/test_issue562_retired_faces.py +150 -0
  381. package/md_cg/test_issue66_scrub_semantics.py +107 -0
  382. package/md_cg/test_issue67_scrub_control_pool.py +233 -0
  383. package/md_cg/test_issue69_audit_boundary.py +129 -0
  384. package/md_cg/test_issue71_queue_backlog.py +377 -0
  385. package/md_cg/test_issue72_coldverify_controlled_drain.py +420 -0
  386. package/md_cg/test_issue73_doc_chapter_refs.py +326 -0
  387. package/md_cg/test_issue75_neural_failure.py +582 -0
  388. package/md_cg/test_issue78_export_concurrency.py +334 -0
  389. package/md_cg/test_issue80_residual_flush.py +169 -0
  390. package/md_cg/test_issue82_credential_rules.py +271 -0
  391. package/md_cg/test_issue82_layer_and_size.py +120 -0
  392. package/md_cg/test_issue84_portability.py +95 -0
  393. package/md_cg/test_issue85_export_overwrite.py +113 -0
  394. package/md_cg/test_issue86_runtime_revoke.py +110 -0
  395. package/md_cg/test_issue87_default_sensitivity_single.py +201 -0
  396. package/md_cg/test_issue88_conclusion_face.py +280 -0
  397. package/md_cg/test_issue88_execution_conflicts.py +87 -0
  398. package/md_cg/test_issue89_zero_score.py +169 -0
  399. package/md_cg/test_issue89a_score_cap_tiebreak.py +279 -0
  400. package/md_cg/test_issue91_verifier_sandbox.py +314 -0
  401. package/md_cg/test_issue91_verifier_timeout.py +528 -0
  402. package/md_cg/test_issue93_remember_redaction.py +282 -0
  403. package/md_cg/test_issue96_key_fallback.py +102 -0
  404. package/md_cg/test_issue98_tenant_root.py +104 -0
  405. package/md_cg/test_jsonl_commit_watermark.py +221 -0
  406. package/md_cg/test_live_validity_cache.py +256 -0
  407. package/md_cg/test_logref.py +1141 -1109
  408. package/md_cg/test_mcp_call_argument_types.py +99 -0
  409. package/md_cg/test_md_access_parity.py +330 -255
  410. package/md_cg/test_md_cg_no_hive_dep.py +140 -0
  411. package/md_cg/test_memory_rsi.py +215 -0
  412. package/md_cg/test_memory_rsi_adversarial.py +91 -0
  413. package/md_cg/test_mode_parity.py +1413 -1393
  414. package/md_cg/test_mutation_residue.py +120 -0
  415. package/md_cg/test_mutation_rollback.py +835 -805
  416. package/md_cg/test_n123_derive_expiry_chain.py +210 -205
  417. package/md_cg/test_n130_verify_falsified_protect.py +185 -185
  418. package/md_cg/test_n209_verify_write_face_gates.py +466 -461
  419. package/md_cg/test_n230_dirty_replay.py +52 -9
  420. package/md_cg/test_n62_tenant_bind_failclosed.py +200 -200
  421. package/md_cg/test_neg_coverage_budget.py +500 -0
  422. package/md_cg/test_neg_tail_honesty.py +733 -663
  423. package/md_cg/test_p0.py +16 -2
  424. package/md_cg/test_p14_root_fallback.py +373 -0
  425. package/md_cg/test_p1x_ref_root.py +47 -10
  426. package/md_cg/test_p21_tokens.py +294 -293
  427. package/md_cg/test_p24_evidence.py +232 -227
  428. package/md_cg/test_p26_refindex.py +20 -3
  429. package/md_cg/test_p27_docindex.py +792 -777
  430. package/md_cg/test_p28_refcheck.py +1053 -1032
  431. package/md_cg/test_p29_session_ingest_export.py +6 -0
  432. package/md_cg/test_p2_mcp.py +474 -453
  433. package/md_cg/test_p2_six_elements.py +476 -463
  434. package/md_cg/test_p3.py +370 -261
  435. package/md_cg/test_p33_ccg_wiring.py +300 -293
  436. package/md_cg/test_p3_legacy_closure.py +438 -433
  437. package/md_cg/test_p40_refine_worklist.py +244 -241
  438. package/md_cg/test_p47_session_view.py +25 -9
  439. package/md_cg/test_p4_freshness.py +781 -774
  440. package/md_cg/test_p83_jsonrpc_parse_error.py +124 -0
  441. package/md_cg/test_p9c_dedup_hints.py +288 -283
  442. package/md_cg/test_policy_required_ccg.py +444 -432
  443. package/md_cg/test_postings_digest_memo.py +139 -0
  444. package/md_cg/test_predict_beta.py +138 -135
  445. package/md_cg/test_qmd.py +500 -0
  446. package/md_cg/test_query_excerpt.py +209 -0
  447. package/md_cg/test_reach.py +456 -378
  448. package/md_cg/test_reach_snapshot_admission.py +218 -0
  449. package/md_cg/test_readcache_prodpath.py +4 -0
  450. package/md_cg/test_recall_face_guards.py +854 -812
  451. package/md_cg/test_recall_metadata.py +422 -0
  452. package/md_cg/test_retr_s5.py +312 -213
  453. package/md_cg/test_retr_score_once.py +14 -2
  454. package/md_cg/test_review_cli_attribution.py +11 -7
  455. package/md_cg/test_review_queue_batch.py +123 -0
  456. package/md_cg/test_review_queue_index.py +130 -0
  457. package/md_cg/test_rrf_candidates.py +292 -0
  458. package/md_cg/test_rrf_qualified_topk.py +214 -0
  459. package/md_cg/test_rrf_unique_votes.py +218 -0
  460. package/md_cg/test_s1_ast_judge.py +110 -0
  461. package/md_cg/test_scrub_ledger.py +413 -0
  462. package/md_cg/test_security_audit_b26.py +34 -3
  463. package/md_cg/test_security_audit_v21.py +1 -2
  464. package/md_cg/test_sensitivity_predicate_single_point.py +184 -0
  465. package/md_cg/test_session5_identity.py +93 -0
  466. package/md_cg/test_session_attribution_consistency.py +813 -0
  467. package/md_cg/test_session_identity_tristate.py +486 -0
  468. package/md_cg/test_sleep_p1.py +35 -3
  469. package/md_cg/test_state_event_op.py +521 -514
  470. package/md_cg/test_state_projection_streaming.py +136 -0
  471. package/md_cg/test_stg_interval_candidates.py +280 -0
  472. package/md_cg/test_stg_time_blocks.py +239 -0
  473. package/md_cg/test_stgidx_index_parity.py +1024 -1020
  474. package/md_cg/test_subgraph_bounded_bfs.py +227 -0
  475. package/md_cg/test_sustain_bounded.py +652 -624
  476. package/md_cg/test_task_memory.py +394 -0
  477. package/md_cg/test_text_projection_mutations.py +101 -0
  478. package/md_cg/test_time_anchor.py +421 -0
  479. package/md_cg/test_time_core_lint.py +1024 -975
  480. package/md_cg/test_tool_face.py +200 -189
  481. package/md_cg/test_trust_bounded_frontier.py +248 -0
  482. package/md_cg/test_trust_edge_reachability.py +405 -0
  483. package/md_cg/test_w5_path_fingerprint.py +381 -0
  484. package/md_cg/test_w7_redlines.py +659 -0
  485. package/md_cg/test_write_return_shape.py +919 -0
  486. package/md_cg/test_writepipe.py +228 -226
  487. package/md_cg/timeanchor.py +534 -0
  488. package/md_cg/tokens.py +68 -5
  489. package/md_cg/tool_face.py +273 -260
  490. package/md_cg/trust.py +1114 -1030
  491. package/md_cg/twophase.py +303 -231
  492. package/md_cg/weights.py +642 -638
  493. package/md_cg/whitebox.py +536 -527
  494. package/md_cg/whitebox_kb/aeis_core/core.py +5026 -5026
  495. package/md_cg/whitebox_kb/wisdom/chat_engine.py +1504 -1506
  496. package/md_cg/whitebox_kb/wisdom/code_compose.py +876 -837
  497. package/md_cg/whitebox_kb/wisdom/compiler_code_units.py +3040 -3033
  498. package/md_cg/whitebox_kb/wisdom/compose_engine.py +1634 -1633
  499. package/md_cg/whitebox_kb/wisdom/core/condition_vm.py +318 -298
  500. package/md_cg/whitebox_kb/wisdom/core/parser.py +1082 -1065
  501. package/md_cg/whitebox_kb/wisdom/csre.py +412 -264
  502. package/md_cg/whitebox_kb/wisdom/csre_session.py +97 -0
  503. package/md_cg/whitebox_kb/wisdom/md_access.py +490 -470
  504. package/md_cg/whitebox_kb/wisdom/migrate_wisdom.py +497 -476
  505. package/md_cg/whitebox_kb/wisdom/mini_python.py +995 -958
  506. package/md_cg/whitebox_kb/wisdom/navigate.py +249 -248
  507. package/md_cg/whitebox_kb/wisdom/net_units.py +2900 -2895
  508. package/md_cg/whitebox_kb/wisdom/neural_retrieve.py +100 -10
  509. package/md_cg/whitebox_kb/wisdom/semantic_translate.py +4060 -4054
  510. package/md_cg/whitebox_kb/wisdom/test_ecosystem_assembly.py +158 -158
  511. package/md_cg/whitebox_kb/wisdom/test_safe_module_proxy.py +147 -0
  512. package/md_cg/whitebox_kb/wisdom/verifier.py +1918 -1297
  513. package/md_cg/writelimit.py +584 -390
  514. package/md_cg/writepipe.py +1337 -826
  515. package/package.json +14 -9
  516. package/scripts/workspace_index.py +328 -0
  517. package/src/bridge.ts +34 -11
  518. package/src/cli.ts +75 -65
  519. package/src/hooks.ts +932 -776
  520. package/src/index.ts +30 -7
  521. package/src/init.ts +362 -361
  522. package/src/lib/datapath.ts +419 -326
  523. package/src/lib/hook_audit.ts +42 -3
  524. package/src/lib/mdcg_client.ts +469 -461
  525. package/src/lib/mutual.ts +481 -432
  526. package/src/lib/prompt_safety.ts +178 -149
  527. package/src/lib/roleplay_web.ts +30 -2
  528. package/src/lib/token_store.ts +274 -52
  529. package/src/tools.ts +271 -223
  530. package/zcode/README.md +1 -1
  531. package/docs/GBrain/345/217/257/345/200/237/351/211/264/347/202/271_/347/201/265/346/236/242/350/220/275/347/202/271/344/272/244/346/216/245_20260919.md +0 -169
  532. package/docs/Pi/345/217/257/345/255/246/344/271/240/344/274/230/347/202/271_/347/201/265/346/236/242/345/244/247/350/204/221/346/224/271/350/277/233/344/272/244/346/216/245_20260915.md +0 -144
  533. package/docs/eval/AGI/344/270/203/347/273/264/350/257/204/345/210/206/346/212/245/345/221/212_md_cg_v1.0.md +0 -299
  534. package/docs/eval/AGI/344/270/203/347/273/264/350/257/204/345/210/206/346/212/245/345/221/212_md_cg_v2.0.md +0 -239
  535. package/docs/eval/AGI/344/270/203/347/273/264/350/257/204/345/210/206/346/212/245/345/221/212_/345/205/255/345/256/266/350/256/260/345/277/206/347/263/273/347/273/237/345/220/214/345/260/272/345/256/236/346/265/213_v3.1.md +0 -340
  536. package/docs/eval/DSH/346/227/245/345/277/227/347/264/242/345/274/225v2_/345/217/202/350/200/203dsh-TUI_v1.0.md +0 -248
  537. package/docs/eval/DSH/346/227/245/345/277/227/347/264/242/345/274/225/346/225/210/346/236/234/351/252/214/350/257/201_v1.0.md +0 -209
  538. package/docs/eval/DSH/347/253/257/347/274/272/351/231/267/344/270/223/351/241/271_v1.0.md +0 -254
  539. package/docs/eval/N225_/347/264/242/345/274/225/346/227/245/345/277/227/351/235/236/345/257/271/350/261/241/350/243/205/350/275/275/351/235/242/347/261/273/345/236/213/351/227/270_v1.0.md +0 -418
  540. package/docs/eval/P1b2_/350/257/273/351/235/242/344/273/243/351/231/205/344/277/256/345/244/215_v1.0.md +0 -124
  541. package/docs/eval/bench_lingshu_self/bench_self.py +0 -140
  542. package/docs/eval/bench_lingshu_self/self_bench_result.json +0 -404
  543. package/docs/eval/bench_lingshu_self//347/201/265/346/236/242/350/207/252/345/272/223/347/253/257/345/210/260/347/253/257/346/243/200/347/264/242/345/256/236/346/265/213_v1.0.md +0 -38
  544. package/docs/eval/cons200_/345/206/262/347/252/201/346/243/200/346/265/213/351/200/211/351/235/242_/345/256/236/346/226/275/350/256/260/345/275/225_v1.0.md +0 -223
  545. package/docs/eval/cons200_/345/206/262/347/252/201/346/243/200/346/265/213/351/200/211/351/235/242_/345/256/236/346/226/275/350/256/260/345/275/225_v1.1.md +0 -340
  546. package/docs/eval/issue43_/351/273/230/350/256/244/347/255/226/347/225/245/344/270/216/351/224/256/347/261/273/345/236/213/351/227/270_v1.1.md +0 -441
  547. package/docs/eval/issue43_/351/273/230/350/256/244/347/255/226/347/225/245/345/212/240/350/275/275/344/270/216/345/207/255/346/215/256/346/230/216/346/226/207/351/230/237/345/210/227_v1.0.md +0 -398
  548. package/docs/eval/issue50_/345/205/203/346/225/260/346/215/256/351/200/217/344/274/240/344/270/216/345/205/234/345/272/225_/344/277/256/345/244/215/350/256/260/345/275/225_v1.0.md +0 -437
  549. package/docs/eval/issue50_/345/215/212/351/207/215/345/244/215/345/276/205/345/256/232/345/244/215/346/240/270_/344/277/256/345/244/215/350/256/260/345/275/225_v1.0.md +0 -488
  550. package/docs/eval/issue50_/345/276/205/345/256/232/345/244/215/346/240/270/345/205/245/351/230/237_/344/277/256/345/244/215/350/256/260/345/275/225_v1.0.md +0 -428
  551. package/docs/eval/issue50_/350/257/273/351/235/242/344/277/235/346/212/244/345/217/252/350/256/244/346/230/276/345/274/217/346/235/245/346/272/220_/344/277/256/345/244/215/350/256/260/345/275/225_v1.0.md +0 -271
  552. package/docs/eval/issue50_/351/207/215/350/246/201/345/272/246/345/220/214/346/272/220/344/270/216/344/277/235/346/212/244/350/257/255/344/271/211_/344/277/256/345/244/215/350/256/260/345/275/225_v1.0.md +0 -306
  553. package/docs/eval/issue51_/344/270/200/351/224/256/345/256/211/350/243/205/345/244/261/350/264/245_/345/275/222/345/261/236/345/210/244/345/256/232_v1.0.md +0 -58
  554. package/docs/eval/issue52_/346/235/241/344/273/266/345/205/210/350/241/214/344/270/216/346/210/252/346/226/255/345/217/257/350/247/202/346/265/213_/344/277/256/345/244/215/350/256/260/345/275/225_v1.0.md +0 -360
  555. package/docs/eval/issue64_replay_check_/346/240/270/351/252/214/344/270/216/344/277/256/345/244/215_v1.0.md +0 -284
  556. package/docs/eval/issue65_session_recall_/344/274/232/350/257/235/351/232/224/347/246/273_/346/240/270/351/252/214/344/270/216/344/277/256/345/244/215_v1.0.md +0 -329
  557. package/docs/eval//344/270/211/346/241/243/350/207/252/346/262/273_/346/255/245/351/252/244/342/221/241/346/241/243/344/275/215/345/215/225/344/270/200/345/205/245/345/217/243_/350/220/275/347/240/201/350/256/260/345/275/225_v1.0.md +0 -651
  558. package/docs/eval//344/270/211/346/241/243/350/207/252/346/262/273_/346/255/245/351/252/244/342/221/242/345/217/230/346/233/264/345/215/225/344/270/216/345/233/236/346/273/232/345/216/237/350/257/255_/350/220/275/347/240/201/350/256/260/345/275/225_v1.0.md +0 -439
  559. package/docs/eval//344/270/211/346/241/243/350/207/252/346/262/273_/346/255/245/351/252/244/342/221/243/345/207/206/345/205/245/350/257/273/346/225/260_/350/220/275/347/240/201/350/256/260/345/275/225_v1.0.md +0 -1337
  560. package/docs/eval//344/270/211/346/241/243/350/207/252/346/262/273_/346/255/245/351/252/244/342/221/244/346/224/266/345/256/230/344/270/216/345/205/250/351/223/276/351/252/214/346/224/266_/350/220/275/347/240/201/350/256/260/345/275/225_v1.0.md +0 -1817
  561. package/docs/eval//344/270/215/345/217/257/351/235/240/346/200/247/350/220/275/345/234/260_P0_v1.0.md +0 -185
  562. package/docs/eval//344/274/230/345/214/226/347/254/254/344/270/200/346/211/271_/346/216/245/347/272/277/344/270/216/347/255/211/344/273/267/345/217/230/346/215/242_v1.0.md +0 -498
  563. package/docs/eval//344/274/230/345/214/226/347/254/254/344/270/211/346/211/271_/351/227/250/347/246/201/350/275/254/346/255/243/344/270/216/350/260/203/345/272/246/346/255/242/350/241/200/344/270/216/351/227/250/346/216/247/346/224/266/345/217/243_v1.0.md +0 -669
  564. package/docs/eval//344/274/230/345/214/226/347/254/254/344/272/214/346/211/271_/344/276/235/350/265/226/351/200/217/344/274/240/344/270/216/345/257/271/346/213/215/345/217/243/345/276/204/344/270/216/350/264/237/347/274/223/345/255/230_v1.0.md +0 -683
  565. package/docs/eval//345/207/272/350/264/247/351/235/242/345/206/222/347/203/237_/350/277/233/350/264/247/351/227/250/347/246/201_v1.0.md +0 -460
  566. package/docs/eval//345/217/221/345/270/20306_/344/270/200/351/224/256/351/205/215/347/275/256/344/270/216DSH0172_v1.0.md +0 -171
  567. package/docs/eval//345/217/221/345/270/20307_/345/244/226/351/203/250/346/212/245/345/221/212/345/233/233/346/211/271/344/277/256/345/244/215/344/270/216/346/217/222/344/273/266/351/235/242/345/212/240/345/233/272_v1.0.md +0 -209
  568. package/docs/eval//345/217/221/345/270/20308_/350/207/252/350/277/255/344/273/243/344/270/216/347/235/241/347/234/240_/345/233/276/346/243/200/347/264/242/350/267/257/344/270/216/346/235/203/351/207/215_v1.0.md +0 -97
  569. package/docs/eval//345/217/221/345/270/20309_/346/243/200/347/264/242/351/235/242/344/270/211/346/211/271/346/224/266/345/217/243_v1.0.md +0 -66
  570. package/docs/eval//345/217/221/345/270/20310_/350/257/273/351/235/242/346/215/237/345/235/217UTF8/345/256/266/346/227/217/346/224/266/345/217/243_v1.0.md +0 -71
  571. package/docs/eval//345/217/221/345/270/20311_/345/244/226/346/212/245/345/205/255/350/277/236/344/277/256/344/270/216/344/270/226/347/225/214/346/250/241/345/236/213/345/272/225/345/272/247_v1.0.md +0 -91
  572. package/docs/eval//345/217/221/345/270/20312_/344/270/226/347/225/214/346/250/241/345/236/213/345/212/237/350/203/275/347/253/257_v1.0.md +0 -108
  573. package/docs/eval//345/217/221/345/270/20313_/344/270/212/344/270/213/346/226/207/350/207/252/347/256/241/347/220/206/346/234/272/345/210/266_v1.0.md +0 -114
  574. package/docs/eval//345/217/221/345/270/20314_/350/272/253/344/275/223/303/227/350/204/221/347/273/204/345/220/210_v1.0.md +0 -160
  575. package/docs/eval//345/217/221/345/270/20315_/344/274/232/350/257/235/351/232/224/347/246/273/344/277/256/345/244/215/344/270/216/350/272/253/344/275/223/345/272/223/345/261/225/347/244/272_v1.0.md +0 -88
  576. package/docs/eval//345/256/236/351/252/214/346/226/271/346/241/210_/345/255/246/344/271/240/351/227/255/347/216/257AB/344/270/216/346/250/252/350/257/204_v1.0.md +0 -174
  577. package/docs/eval//345/275/222/344/270/200/345/261/202/347/274/272/347/234/201/347/277/273/345/205/263_/344/277/256/345/244/215/350/256/260/345/275/225_v1.0.md +0 -458
  578. package/docs/eval//346/200/247/350/203/275/344/270/223/351/241/271_/345/206/267/346/237/245/350/257/242/344/270/216/345/206/205/345/255/230_v1.0.md +0 -218
  579. package/docs/eval//346/225/205/351/232/234/346/263/250/345/205/245/345/256/236/346/265/213_v1.0.md +0 -432
  580. package/docs/eval//346/250/252/350/257/204_/345/205/255/345/256/266100/351/242/230/344/270/255/350/213/261/345/217/214/346/237/245_v1.0.md +0 -223
  581. package/docs/eval//347/224/250/346/210/267/350/243/201/345/206/263/345/244/204/347/275/256_/347/254/254/344/270/200/346/211/271_v1.0.md +0 -195
  582. package/docs/eval//347/224/250/346/210/267/350/243/201/345/206/263/345/244/204/347/275/256_/347/254/254/344/272/214/346/211/271B6_/346/234/254/346/234/272/350/267/257/345/276/204/347/233/270/345/257/271/345/214/226_v1.0.md +0 -78
  583. package/docs/eval//347/253/257/345/210/260/347/253/257LoCoMoQA/345/220/214/345/217/243/345/276/204/345/257/271/347/205/247_v1.0.md +0 -100
  584. package/docs/eval//347/253/257/345/210/260/347/253/257/345/271/262/346/211/260/346/261/240/350/257/204/346/265/213_/347/241/256/345/256/232/346/200/247/350/243/201/345/206/263vsLLM_judge_v1.1.md +0 -197
  585. package/docs/eval//347/254/2543/345/261/202stg/347/273/223/346/236/204/347/264/242/345/274/225_/345/256/236/346/226/275/350/256/260/345/275/225_v1.0.md +0 -250
  586. package/docs/eval//347/254/254/344/270/211/346/226/271/351/252/214/350/257/201/346/212/245/345/221/212_LoCoMo_/347/201/265/346/236/242_.png +0 -0
  587. package/docs/eval//347/254/254/344/270/211/346/226/271/351/252/214/350/257/201/346/212/245/345/221/212_/347/201/265/346/236/242_vs_dejavu_/347/273/237/344/270/200/350/257/204/345/210/206_v7.md +0 -202
  588. package/docs/eval//347/274/226/347/240/201/351/235/242/345/211/215/347/275/256_/345/205/245/345/217/243/350/207/252/344/277/235/350/257/201UTF8/344/270/216/346/226/207/346/234/254open/345/256/210/345/215/253_v1.0.md +0 -646
  589. package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v1.md +0 -156
  590. package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v10.md +0 -210
  591. package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v11.md +0 -227
  592. package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v12.md +0 -203
  593. package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v13.md +0 -233
  594. package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v14.md +0 -191
  595. package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v15.md +0 -213
  596. package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v16.md +0 -214
  597. package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v17.md +0 -199
  598. package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v18.md +0 -183
  599. package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v19.md +0 -207
  600. package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v2.md +0 -156
  601. package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v20.md +0 -283
  602. package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v21.md +0 -224
  603. package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v22.md +0 -223
  604. package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v23.md +0 -293
  605. package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v24.md +0 -303
  606. package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v25.md +0 -230
  607. package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v26.md +0 -196
  608. package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v27.md +0 -259
  609. package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v28.md +0 -217
  610. package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v29.md +0 -270
  611. package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v3.md +0 -152
  612. package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v30.md +0 -231
  613. package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v31.md +0 -281
  614. package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v4.md +0 -128
  615. package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v5.md +0 -114
  616. package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v6.md +0 -192
  617. package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v7.md +0 -187
  618. package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v8.md +0 -207
  619. package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v9.md +0 -203
  620. package/docs/eval//350/257/255/344/271/211/346/227/266/347/251/272/345/233/276/350/241/245/345/205/250_P1_/346/212/275/345/217/226/345/231/250v4_/344/270/211/347/216/207/350/257/273/346/225/260_v1.0.md +0 -231
  621. package/docs/eval//351/262/270/345/250/230/344/270/203/347/261/273/347/212/266/346/200/201/350/277/275/350/270/252_v1.0.md +0 -176
  622. package/docs/hive//344/270/273/344/273/243/347/220/206/345/255/220/344/273/243/347/220/206/350/256/260/345/277/206/346/236/266/346/236/204/350/256/276/350/256/241.md +0 -385
  623. package/docs/hive//344/273/244/347/211/214/344/270/216/350/247/222/350/211/262/346/235/203/350/201/214/345/210/206/347/246/273_v0.1.md +0 -165
  624. package/docs/hive//344/273/244/347/211/214/350/257/255/344/271/211/344/277/256/346/255/243_/344/273/262/350/243/201/344/275/215_v0.1.md +0 -76
  625. package/docs/hive//345/244/232/347/253/257harness/351/200/232/344/277/241/345/245/221/347/272/246_v0.1.md +0 -42
  626. package/docs/hive//345/255/220/344/273/243/347/220/206/351/205/215/347/275/256/346/240/207/345/207/206_v0.5.md +0 -236
  627. package/docs/hive//345/256/211/345/205/250/345/256/241/350/256/241/345/256/236/351/224/232_v0.1.md +0 -202
  628. package/docs/hive//346/243/200/347/264/242/346/224/266/346/225/233/345/256/236/346/265/213/344/270/216S1b/350/256/276/350/256/241_v0.1.md +0 -43
  629. package/docs/hive//346/243/200/347/264/242/347/256/227/346/263/225/345/217/243/345/276/204/345/257/271/347/205/247_v0.1.md +0 -210
  630. package/docs/hive//346/243/200/347/264/242/350/267/257/345/276/204/344/270/216/350/256/244/347/237/245/347/273/223/346/236/204/345/245/221/347/272/246_v0.1.md +0 -152
  631. package/docs/hive//350/234/202/345/267/242M6_ingest/345/256/236/346/226/275/350/256/241/345/210/222_v0.1.md +0 -47
  632. package/docs/hive//350/234/202/345/267/242/345/217/214/345/256/236/344/276/213/344/272/222/351/252/214_/350/256/276/350/256/241/345/256/232/347/250/277.md +0 -519
  633. package/docs/hive//350/234/202/345/267/242/345/267/245/344/275/234/350/256/260/345/277/206_/351/241/271/347/233/256/350/256/241/345/210/222.md +0 -85
  634. package/docs/hive//350/234/202/345/267/242/350/256/276/350/256/241_/347/220/206/350/256/272/345/257/271/351/275/220_v0.1.md +0 -191
  635. package/docs/hive//350/234/202/345/267/242/350/277/255/344/273/243_/345/256/217/350/247/202/344/270/216/347/276/244/344/275/223/350/260/203/345/272/246_v0.1.md +0 -90
  636. package/docs/images/lingshu-moonlight-covenant-poster-preview.jpg +0 -0
  637. package/docs/images/lingshu-moonlight-covenant-poster.png +0 -0
  638. package/docs/mdcg/D_meta_/345/267/245/347/250/213/345/214/226/346/226/271/346/241/210_v0.2.md +0 -216
  639. package/docs/mdcg/README/350/257/246/347/273/206/347/211/210_v0.4.10.md +0 -746
  640. package/docs/mdcg/guardrail-charter.md +0 -164
  641. package/docs/mdcg/lingshu_tutorial.html +0 -14956
  642. package/docs/mdcg/memory-assessment.html +0 -923
  643. package/docs/mdcg/memory_score.html +0 -213
  644. package/docs/mdcg/memory_score.png +0 -0
  645. package/docs/mdcg/release_v0.3.0.md +0 -36
  646. package/docs/mdcg/release_v0.4.11.md +0 -49
  647. package/docs/mdcg/release_v0.4.5.md +0 -55
  648. package/docs/mdcg/tool_table_v0.3.0.md +0 -117
  649. package/docs/mdcg//344/270/226/347/225/214/346/250/241/345/236/213/345/212/237/350/203/275/347/253/257_/344/275/277/347/224/250/344/270/216/350/277/220/347/273/264_v1.0.md +0 -384
  650. package/docs/mdcg//344/273/243/347/240/201/350/257/204/345/256/241/344/270/216/346/235/241/344/273/266/345/214/226/346/263/250/351/207/212_/345/245/221/347/272/246_v0.1.md +0 -82
  651. package/docs/mdcg//345/205/250/345/272/223/344/273/243/347/240/201/350/257/204/345/256/241/344/270/216/346/235/241/344/273/266/345/214/226/346/263/250/351/207/212_/350/256/241/345/210/222_v0.1.md +0 -600
  652. package/docs/mdcg//345/212/237/350/203/275/350/260/203/347/224/250/346/230/240/345/260/204/350/241/250_v0.1.md +0 -266
  653. package/docs/mdcg//345/215/225/345/205/203/350/207/252/346/210/221/351/224/232/347/202/271_/347/263/273/347/273/237/346/217/220/347/244/272/350/257/215/346/240/207/345/207/206_v0.3.md +0 -275
  654. package/docs/mdcg//345/217/221/345/270/203/351/227/250/347/246/201/351/223/276_v0.1.md +0 -134
  655. package/docs/mdcg//346/272/220/347/240/201/347/272/247/346/236/266/346/236/204/345/256/241/350/256/241_GPT/346/211/271/350/257/204/345/257/271/347/205/247_v1.0.md +0 -130
  656. package/docs/mdcg//347/201/265/346/236/24282/345/267/245/345/205/267_/345/212/237/350/203/275/346/225/264/347/220/206/344/270/216/350/277/201/347/247/273/346/230/240/345/260/204_v0.1.md +0 -278
  657. package/docs/mdcg//347/201/265/346/236/242MCP/345/267/245/345/205/267/346/200/273/350/241/250_v3.4.md +0 -164
  658. package/docs/mdcg//347/201/265/346/236/242_/350/207/252/346/210/221/345/261/202/345/256/232/344/271/211.md +0 -43
  659. package/docs/mdcg//347/201/265/346/236/242/344/270/211/345/261/202/346/213/206/345/210/206/350/247/204/345/210/222_v0.1.md +0 -181
  660. package/docs/mdcg//347/201/265/346/236/242/350/256/260/345/277/206/345/212/250/350/257/215/345/215/217/350/256/256_v1.0-draft.md +0 -177
  661. package/docs/mdcg//347/216/257/344/272/214_/347/231/275/347/256/261/345/241/253/345/205/205/346/265/201/346/260/264/347/272/277_v0.1.md +0 -31
  662. package/docs/mdcg//347/235/241/347/234/240/345/221/250/346/234/237_/350/277/220/347/273/264/345/211/215/346/217/220/344/270/216/347/273/264/346/212/244/346/214/207/345/215/227_v1.0.md +0 -337
  663. package/docs/mdcg//347/274/272/345/217/243/345/215/225_P0/346/224/266/345/217/243_v0.1.md +0 -270
  664. package/docs/mdcg//350/256/244/347/237/245/345/233/276_G4-G8/347/274/272/345/217/243/350/243/201/345/256/232/345/215/225_v0.1.md +0 -491
  665. package/docs/mdcg//350/256/244/347/237/245/345/233/276_MD/347/233/256/345/275/225/346/226/271/346/241/210_v0.1.md +0 -552
  666. package/docs/mdcg//350/256/244/347/237/245/345/233/276_/346/235/241/344/273/266/347/251/272/351/227/264/345/220/210/346/210/220/344/270/216/347/224/237/346/225/210/346/235/241/344/273/266/345/217/243/345/276/204_v0.1.md +0 -340
  667. package/docs/mdcg//350/256/244/347/237/245/345/233/276_/347/264/242/345/274/225/344/270/216/345/267/245/347/250/213/350/247/204/350/214/203/345/214/226_/350/256/241/345/210/222_v0.1.md +0 -624
  668. package/docs/mdcg//350/256/244/347/237/245/345/233/276/344/275/234/344/270/272/350/256/260/345/277/206/346/223/215/344/275/234/347/263/273/347/273/237_/350/257/204/344/274/260/344/270/216/350/267/257/347/272/277_v0.1.md +0 -94
  669. package/docs/mdcg//350/256/260/345/277/206/346/223/215/344/275/234/347/263/273/347/273/237_MdCGOS/344/270/216MCP/346/216/245/345/205/245_v0.1.md +0 -281
  670. package/docs/mdcg//350/267/250/347/253/257/351/252/214/350/257/201/344/270/216/345/220/214/346/255/245/345/215/217/350/256/256_v0.1.md +0 -93
  671. package/docs/plans/GridWorld/346/234/200/345/260/217/351/227/255/347/216/257/350/247/204/346/240/274_v0.1.md +0 -176
  672. package/docs/plans/stg/346/235/241/344/273/266/345/214/226/344/270/216/347/273/223/346/236/204/347/264/242/345/274/225_/350/256/276/350/256/241_v0.1.md +0 -124
  673. package/docs/plans//345/205/250/344/270/255/346/226/207/347/274/226/347/240/201/344/270/216/350/234/202/345/267/242/344/273/273/345/212/241/346/240/207/350/257/206/345/245/221/347/272/246_v2.0.md +0 -220
  674. package/docs/plans//345/221/275/345/220/215/346/262/273/347/220/206_/351/241/271/347/233/256/350/256/241/345/210/222.md +0 -81
  675. package/docs/plans//345/244/232/344/270/273/344/275/223/344/270/226/347/225/214/346/250/241/345/236/213_/345/257/271/351/275/220/350/257/204/344/274/260/344/270/216/350/220/275/345/234/260/350/256/276/350/256/241_v0.1.md +0 -141
  676. package/docs/plans//347/201/265/346/236/242/350/272/253/344/275/223/303/227/350/204/221_/344/270/226/347/225/214/346/250/241/345/236/213/345/257/271/346/216/245/350/256/276/350/256/241_v0.1.md +0 -119
  677. package/docs/plans//347/234/237/346/272/220/347/264/242/345/274/225_/351/200/232/347/224/250/346/234/272/345/210/266_v0.3.md +0 -227
  678. package/docs/plans//347/235/241/347/234/240/344/270/216/350/207/252/350/277/255/344/273/243_/345/212/237/350/203/275/344/274/230/345/214/226/350/256/276/350/256/241_v0.4.md +0 -547
  679. package/docs/plans//347/237/245/350/257/206/345/233/276/350/260/261/351/241/271/347/233/256_/344/272/244/346/216/245/346/226/207/346/241/243_20260914.md +0 -308
  680. package/docs/plans//350/234/202/345/267/242/346/250/241/345/236/213/345/257/206/351/222/245/351/205/215/347/275/256/351/235/242_v1.0.md +0 -385
  681. package/docs/plans//350/256/260/345/277/206/350/207/252/345/244/204/347/220/206/344/270/211/346/241/243/350/207/252/346/262/273_/350/256/276/350/256/241_v0.2.md +0 -207
  682. package/docs/plans//350/257/255/344/271/211/346/227/266/347/251/272/345/233/276/350/241/245/345/205/250_/344/270/226/347/225/214/346/250/241/345/236/213/345/212/237/350/203/275/347/253/257_/350/256/276/350/256/241_v0.1.md +0 -107
  683. package/docs/promo/bilibili-cover.jpg +0 -0
  684. package/docs/promo/promo-visual-1.png +0 -0
  685. package/docs/promo/promo-visual-2.png +0 -0
  686. package/docs/promo/promo-visual-3.png +0 -0
  687. package/docs/promo/promo-visual-4.png +0 -0
  688. package/docs/promo/promo-visual-5.png +0 -0
  689. package/docs/swarm//350/234/202/347/276/244/344/272/222/350/201/224_v0.1.md +0 -704
  690. package/docs/swarm//350/234/202/347/276/244/345/220/214/351/224/231/346/243/200/346/265/213/345/256/236/351/252/214/345/215/217/350/256/256_v0.1.md +0 -242
  691. package/docs/swarm//350/234/202/347/276/244/345/244/232/346/231/272/350/203/275/344/275/223_/344/275/277/347/224/250/346/212/245/345/221/212_20260913.md +0 -68
  692. package/docs/swarm//350/234/202/347/276/244/345/244/232/346/231/272/350/203/275/344/275/223_/345/212/237/350/203/275/350/257/264/346/230/216_v0.6.md +0 -130
  693. package/docs/swarm//350/234/202/347/276/244/347/233/262/345/214/272/346/240/207/350/256/260_v1.0.md +0 -89
  694. package/docs/theory//344/270/215/345/217/257/351/235/240/345/256/232/347/220/206/344/270/216/345/244/261/346/225/210/344/274/230/345/205/210/346/241/206/346/236/266_v0.3.md +0 -365
  695. package/docs/theory//344/277/241/346/201/257/345/267/256/344/270/272/344/273/200/344/271/210/345/277/205/347/204/266/345/255/230/345/234/250/344/270/224/350/207/252/347/204/266/346/211/251/345/244/247.md +0 -239
  696. package/docs/theory//345/215/225/347/272/277/347/250/213/344/270/216/346/263/250/346/204/217/345/212/233/351/233/206/344/270/255_/346/227/240/344/272/211/350/256/256/347/220/206/350/256/272/346/226/207/346/241/243_v1.0.md +0 -129
  697. package/docs/theory//345/271/266/345/217/221/345/277/205/347/204/266/346/200/247/347/220/206/350/256/272_v0.2.md +0 -134
  698. package/docs/theory//346/231/272/350/203/275/347/232/204/345/205/254/347/220/206/345/214/226/345/237/272/347/237/263.md +0 -789
  699. package/docs/theory//346/231/272/350/203/275/347/232/204/350/256/244/347/237/245/350/277/207/347/250/213.md +0 -379
  700. package/docs/theory//346/231/272/350/203/275/350/256/2723.4.md +0 -5260
  701. package/docs/theory//346/246/202/345/277/265/345/210/206/345/261/202/345/257/271/351/275/220/350/241/250_v0.1.md +0 -145
  702. package/docs/theory//347/220/206/350/256/272_/346/234/272/345/210/266_/344/273/243/347/240/201_/345/256/236/351/252/214_/347/274/272/345/217/243/347/237/251/351/230/265_v0.1.md +0 -144
  703. package/docs/theory//347/220/206/350/256/272/344/273/223/346/213/206/345/210/206/344/270/216/347/231/275/347/256/261/347/237/245/350/257/206/345/272/223/345/206/205/350/277/201_v0.1.md +0 -198
  704. package/docs/theory//347/231/275/347/256/261/346/231/272/350/203/275/346/230/257/344/273/200/344/271/210/357/274/237.md +0 -812
  705. package/docs/theory//347/231/275/347/256/261/346/231/272/350/203/275/347/263/273/345/210/227/302/267/347/254/254/344/272/224/347/257/207/357/274/232/350/256/251AI/347/234/237/346/255/243/350/243/205/344/270/212/350/256/260/345/277/206.md +0 -421
  706. package/docs/theory//350/256/244/347/237/245/344/273/243/347/220/206/344/270/216/346/224/266/346/225/233/347/273/223/346/236/204_/346/235/241/344/273/266/350/256/272/351/207/215/346/236/204_v0.1.md +0 -221
  707. package/docs/world_model//344/270/226/347/225/214/346/250/241/345/236/213_/347/245/236/347/273/217/347/275/221/347/273/234/345/272/225/345/261/202/346/236/266/346/236/204_v1.0.md +0 -237
  708. package/docs//344/270/215/345/217/257/351/235/240/346/200/247/347/220/206/350/256/272_v0.1.md +0 -343
  709. package/docs//345/217/221/345/270/203/344/273/266/345/233/236/346/272/257/350/257/264/346/230/216_v0.1.md +0 -82
  710. package/docs//345/255/230/347/256/227/344/270/200/344/275/223/346/236/266/346/236/204/345/255/246/344/271/240_/345/244/226/351/203/250/347/220/206/350/256/272/345/257/271/347/205/247/344/270/216/350/267/257/347/272/277/344/272/244/346/216/245_20260916.md +0 -98
  711. package/docs//345/255/230/347/256/227/344/270/200/344/275/223/347/245/236/347/273/217/347/275/221/347/273/234/346/236/266/346/236/204_/346/200/273/347/272/262/344/270/216/344/272/244/346/216/245_20260916.md +0 -109
  712. package/docs//347/201/265/346/236/242/350/207/252/346/210/221/346/224/271/350/277/233/345/267/245/344/275/234/350/256/241/345/210/222_/345/244/226/351/203/250/347/240/224/347/251/266/347/263/273/345/210/227/345/220/270/346/224/266_v1_20260919.md +0 -286
  713. package/docs//347/231/275/347/256/261/345/214/226/347/272/262/351/242/206_/344/270/211/351/241/271/347/233/256/345/255/246/344/271/240/346/200/273/346/236/266/346/236/204/344/270/216Ghidra/344/272/244/346/216/245_20260916.md +0 -104
  714. package/docs//347/247/244_/350/256/260/345/277/206/347/263/273/347/273/237/350/257/204/346/265/213/350/247/204/350/214/203_v2.1.md +0 -225
  715. package/docs//350/256/260/345/277/206/347/263/273/347/273/237/345/215/263/347/231/275/347/256/261/345/214/226/346/231/272/350/203/275_/346/236/266/346/236/204/345/257/271/347/205/247_v0.2.md +0 -121
  716. package/docs//350/256/260/345/277/206/347/263/273/347/273/237/345/215/263/347/231/275/347/256/261/345/214/226/346/231/272/350/203/275_/346/236/266/346/236/204/345/257/271/347/205/247_v0.3.md +0 -117
  717. package/docs//350/256/260/345/277/206/350/257/204/345/256/241/347/263/273/347/273/237_/347/253/213/351/241/271/350/256/276/350/256/241/344/270/216/346/226/275/345/267/245/344/272/244/346/216/245_20260915.md +0 -183
package/md_cg/sources.py CHANGED
@@ -1,834 +1,1092 @@
1
- # -*- coding: utf-8 -*-
2
- """md_cg · 设备驱动(记忆 OS #3):把外部会话流接进认知图
3
-
4
- 设计:
5
- Source(事件源)—— 把某种外部存储解析成统一事件流:
6
- {"t": epoch 秒, "seq": 序号, "role": ..., "text": ..., "session": ..., "cwd": ...}
7
-
8
- ⚠ **单位纪律(issue #23)**:`t` 一律 **epoch 秒**,与图内 `created_at` /
9
- `condition_space.time_window` / `trust.parse_time` 同口径。外部源里的 13 位毫秒
10
- (DSH 的 `time` 字段、通用 JSONL 的毫秒戳)在**源适配层**经 `trust.epoch_seconds`
11
- 归一后即不外溢——否则 `condition_space.time_window` 会落毫秒,把
12
- `stg(op=timeline)` 的倒序头部整片占满并顶掉 auto-recall。
13
-
14
- Ingestor(摄取器)—— 增量 watermark + 去重 + 写节点 + 自动 fix-pair 挖掘。
15
-
16
- 内置源:
17
- · JsonlSource —— 通用 JSONL(每行一个事件对象)
18
- · DSHSessionSource —— DeepSeek Harness 会话(~/.dsh/sessions/**/session.jsonl[.zstd])
19
- zstd 为**可选**依赖:缺失时优雅降级(跳过 .zstd 文件并告警)
20
-
21
- 默认敏感度:会话内容是私有记忆 → sensitivity="private"(避免落入公开根)。
22
-
23
- 零第三方依赖(zstd 为可选增强)。
24
- """
25
- from __future__ import annotations
26
-
27
- import glob
28
- import json
29
- import os
30
- import time
31
-
32
- from . import trust
33
- from .fsutil import publish
34
- from .security import DEFAULT_SENSITIVITY
35
-
36
- # 会话事件的默认落层与敏感度
37
- SESSION_LAYER = "contextual"
38
- SESSION_SENSITIVITY = "private"
39
-
40
-
41
- # --------------------------------------------------------------------------
42
- # 事件源
43
- # --------------------------------------------------------------------------
44
-
45
- # 生效条件:无必填构造形参,类常量 name="source" 即实例默认;仅当子类覆写 events() 时才产出事件,基类 events() 恒抛 NotImplementedError。
46
- class Source:
47
- """事件源基类。"""
48
-
49
- name = "source"
50
-
51
- # 生效条件:任何调用都直接 raise NotImplementedError(基类占位,无其它分支)。
52
- def events(self):
53
- raise NotImplementedError
54
-
55
- # 生效条件:无前置;返回类常量 self.name(基类为 "source"),作为 watermark 的稳定标识,不含路径与运行期状态;
56
- def key(self):
57
- """源的稳定标识(用于 watermark)。"""
58
- return self.name
59
-
60
-
61
- # 生效条件:required 形参 path 总被存入 self.path;可选形参 name 为假值(None/空串)时 self.name 回落到 "jsonl:"+os.path.basename(path),为真值时用 name 本身,t_key/role_key/text_key/default_role 原样存入属性。
62
- class JsonlSource(Source):
63
- """通用 JSONL 会话源。
64
-
65
- 每行是事件对象;字段映射可配置:
66
- t_key —— 时间戳字段(默认 "time",也接受 ISO 字符串)
67
- role_key —— 角色字段(默认 "role")
68
- text_key —— 文本字段(默认 "text")
69
- """
70
-
71
- # 生效条件:传入 path;name 为假值(None/空串)时回落为 "jsonl:"+os.path.basename(path),t_key/role_key/text_key/default_role 原样存为属性(默认值 "time"/"role"/"text"/"user")。
72
- def __init__(self, path: str, name: str = None, t_key="time", role_key="role",
73
- text_key="text", default_role="user"):
74
- self.path = path
75
- self.name = name or ("jsonl:" + os.path.basename(path))
76
- self.t_key, self.role_key, self.text_key = t_key, role_key, text_key
77
- self.default_role = default_role
78
-
79
- # 生效条件:无前置;返回 self.name(构造时已回落为 "jsonl:"+basename(path)),只随构造参数变化、不随文件内容变化;
80
- def key(self):
81
- return self.name
82
-
83
- # 生效条件:o.get(self.t_key) 经 float 可转数值时返回 trust.epoch_seconds(该值)(秒值原样、13 位毫秒 /1000 归一到**秒**);float 抛 TypeError/ValueError 且 v 为字符串时按 v[:19] 与 "%Y-%m-%dT%H:%M:%S" 解析,成功返回 mktime(**秒**)、抛 ValueError 返回 0.0;键缺失/None/其它不可转类型返回 0.0。
84
- def _ts(self, o):
85
- v = o.get(self.t_key)
86
- try:
87
- return trust.epoch_seconds(float(v)) or 0.0
88
- except (TypeError, ValueError):
89
- pass
90
- if isinstance(v, str):
91
- try:
92
- return time.mktime(time.strptime(v[:19], "%Y-%m-%dT%H:%M:%S"))
93
- except ValueError:
94
- return 0.0
95
- return 0.0
96
-
97
- # 生效条件:os.path.exists(self.path) 为真时逐行产出,空行、json.loads 抛 ValueError、o.get(self.text_key) 为假的行被跳过,产出项为 t=self._ts(o)、seq=o.get("seq", i)、role=o.get(self.role_key) or self.default_role、text=str(text)、session/cwd 取 o 同名键;path 不存在时直接 return 不产出。
98
- def events(self):
99
- if not os.path.exists(self.path):
100
- return
101
- with open(self.path, encoding="utf-8", errors="replace") as f:
102
- for i, line in enumerate(f):
103
- line = line.strip()
104
- if not line:
105
- continue
106
- try:
107
- o = json.loads(line)
108
- except ValueError:
109
- continue
110
- text = o.get(self.text_key)
111
- if not text:
112
- continue
113
- yield {"t": self._ts(o), "seq": o.get("seq", i),
114
- "role": o.get(self.role_key) or self.default_role,
115
- "text": str(text), "session": o.get("session"),
116
- "cwd": o.get("cwd")}
117
-
118
-
119
- # 生效条件:path 指向的内容可读且 zstandard 可导入时返回 StringIO(raw.decode("utf-8", errors="replace"));ImportError 时返回 None。
120
- def _zstd_reader(path):
121
- """返回可读的文本迭代器;zstd 不可用返回 None。"""
122
- import io
123
- try:
124
- import zstandard as zstd
125
- except ImportError:
126
- return None
127
- with open(path, "rb") as f:
128
- raw = zstd.ZstdDecompressor().stream_reader(f).read()
129
- return io.StringIO(raw.decode("utf-8", errors="replace"))
130
-
131
-
132
- # 生效条件:required 形参 path 总被存入 self.path,可选形参 include_reasoning 原样存入,self.name 恒为 "dsh:"+os.path.basename(os.path.dirname(path))(与 include_reasoning 取值无关)。
133
- class DSHSessionSource(Source):
134
- """DeepSeek Harness 会话源。
135
-
136
- 事件映射:
137
- user/message → role=user
138
- assistant/message → role=assistant(只取 content[].text,reasoning 默认丢弃)
139
- tool/call → role=command("name(args)" 形式)
140
- tool/result → role=tool-output
141
- """
142
-
143
- # 生效条件:传入 path 即成立,include_reasoning 原样存为属性(默认 False),name 固定为 "dsh:"+os.path.basename(os.path.dirname(path))。
144
- def __init__(self, path: str, include_reasoning: bool = False):
145
- self.path = path
146
- self.include_reasoning = include_reasoning
147
- self.name = "dsh:" + os.path.basename(os.path.dirname(path))
148
-
149
- # 生效条件:无前置;返回 self.name(构造时固定为 "dsh:"+basename(dirname(path))),与 include_reasoning 取值无关;
150
- def key(self):
151
- return self.name
152
-
153
- @staticmethod
154
- # 生效条件:可选形参 root 为假值(None/空串)时改用默认目录 os.path.join(expanduser("~"),".dsh","sessions"),否则用传入 root;对 root 下递归 glob 到的 session.jsonl 与 session.jsonl.zstd 逐条 os.stat(抛 OSError 的条目跳过),按 (-st_size, path) 升序键排序(两处 glob 均无命中时为空列表),可选形参 limit 为假值(None/0)时返回全部 rows,否则返回 rows[:limit];每行为 (path, size, mtime) 三元组。
155
- def discover_detailed(root: str = None, limit: int = None):
156
- """候选会话 → `[(path, size, mtime), ...]`(**排序唯一真源**,`discover` 复用)。
157
-
158
- 排序:会话**文件体积**降序 → `path` 升序(确定性终键)。
159
- **刻意不按最近活动排序**——这正是 `source='auto'` 可能选中很久以前会话的
160
- 原因;把 size/mtime 一并透出,让「为什么选它」在返回体里可辨
161
- (issue #23 附带建议:先让依据可见,是否换策略另议)。
162
- """
163
- root = root or os.path.join(os.path.expanduser("~"), ".dsh", "sessions")
164
- files = glob.glob(os.path.join(root, "**", "session.jsonl"), recursive=True)
165
- files += glob.glob(os.path.join(root, "**", "session.jsonl.zstd"), recursive=True)
166
- rows = []
167
- for p in files:
168
- try:
169
- st = os.stat(p)
170
- except OSError:
171
- continue # 竞态删除:跳过而非整体失败
172
- rows.append((p, st.st_size, st.st_mtime))
173
- rows.sort(key=lambda r: (-r[1], r[0]))
174
- return rows[:limit] if limit else rows
175
-
176
- @staticmethod
177
- # 生效条件:可选形参 root/limit 原样转交 discover_detailed,返回其结果的 path 列(同为体积降序、同确定性终键),limit 为假值时返回全部。
178
- def discover(root: str = None, limit: int = None):
179
- """候选会话路径(体积降序)——`discover_detailed` 的路径投影。"""
180
- return [p for p, _s, _m in DSHSessionSource.discover_detailed(root, limit)]
181
-
182
- # 生效条件:self.path 以 ".zstd" 结尾时经 _zstd_reader 逐行产出(其返回 None 时 raise RuntimeError),否则以 utf-8/errors=replace 打开 self.path 逐行产出。
183
- def _lines(self):
184
- if self.path.endswith(".zstd"):
185
- fh = _zstd_reader(self.path)
186
- if fh is None:
187
- raise RuntimeError("zstd 不可用(pip install zstandard),跳过该会话")
188
- try:
189
- yield from fh
190
- finally:
191
- fh.close()
192
- else:
193
- with open(self.path, encoding="utf-8", errors="replace") as f:
194
- yield from f
195
-
196
- @staticmethod
197
- # 生效条件:required 形参 content 为 str 时原样返回 content;为 list 时收集其中 str 元素及 type 属于 ("text","input-text") 且 text 为真值的 dict 元素(取 str(c["text"])),以 "\n" 连接返回(无可收集元素时为空串 "");既非 str 也非 list 时返回 ""。
198
- def _text_of(content):
199
- """content 可能是 [{type,text}] 或字符串。"""
200
- if isinstance(content, str):
201
- return content
202
- if isinstance(content, list):
203
- parts = []
204
- for c in content:
205
- if isinstance(c, dict):
206
- if c.get("type") in ("text", "input-text") and c.get("text"):
207
- parts.append(str(c["text"]))
208
- elif isinstance(c, str):
209
- parts.append(c)
210
- return "\n".join(parts)
211
- return ""
212
-
213
- # 生效条件:逐行解析后按 o.get("type") 分派——"session" 只更新 sess/cwd 不产出;"user/message" 产出 role=user 与 _text_of(data.get("content"));"assistant/message" 产出 role=assistant 与 _text_of(msg.get("content")),include_reasoning 为真时再把 content 中 type=="reasoning" 且有 text 的项追加 "\n[reasoning] "+str(c["text"]);"tool/call" 产出 role=command 与 f"{name}({arguments or ''})";"tool/result" 产出 role=tool-output,仅当 inner[0] 为 dict 时 text=_text_of(inner[0].get("content"));其它 type 或 ev 中 text 为空的事件不产出。
214
- def events(self):
215
- sess = None
216
- cwd = None
217
- for line in self._lines():
218
- line = line.strip()
219
- if not line:
220
- continue
221
- try:
222
- o = json.loads(line)
223
- except ValueError:
224
- continue
225
- t = o.get("type")
226
- data = o.get("data") or {}
227
- if t == "session":
228
- sess, cwd = o.get("id"), o.get("cwd")
229
- continue
230
- ev = {"t": trust.epoch_seconds(float(o.get("time") or 0)) or 0.0,
231
- "seq": o.get("seq"),
232
- "session": sess, "cwd": cwd}
233
- if t == "user/message":
234
- ev["role"], ev["text"] = "user", self._text_of(data.get("content"))
235
- elif t == "assistant/message":
236
- msg = data.get("message") or {}
237
- ev["role"] = "assistant"
238
- ev["text"] = self._text_of(msg.get("content"))
239
- if self.include_reasoning:
240
- for c in (msg.get("content") or []):
241
- if isinstance(c, dict) and c.get("type") == "reasoning" \
242
- and c.get("text"):
243
- ev["text"] = (ev["text"] or "") + "\n[reasoning] " + str(c["text"])
244
- elif t == "tool/call":
245
- ev["role"] = "command"
246
- ev["text"] = f"{data.get('name')}({data.get('arguments') or ''})"
247
- elif t == "tool/result":
248
- msg = data.get("message") or {}
249
- inner = msg.get("content") or []
250
- txt = ""
251
- if inner and isinstance(inner[0], dict):
252
- txt = self._text_of(inner[0].get("content"))
253
- ev["role"], ev["text"] = "tool-output", txt
254
- else:
255
- continue
256
- if ev.get("text"):
257
- yield ev
258
-
259
-
260
- # --------------------------------------------------------------------------
261
- # 蜂巢任务事件源(M6:hive/jobs → contextual,外部设计稿 蜂巢记忆架构设计.md §5)
262
- # --------------------------------------------------------------------------
263
-
264
- class HiveJobsSource(Source):
265
- """蜂巢任务事件源。
266
-
267
- 事件映射(设计稿 §5.3):
268
- start → role=user,任务开始(task 摘要)
269
- tool → **跳过**(防流水账爆炸;tool_trace 已在 result 有摘要)
270
- handoff → role=assistant,续跑卡
271
- error → role=assistant,「任务失败(job=…):…」(fix-pair 前件)
272
- final → role=assistant,content 头
273
- result.json 终态 → **权威确认事件**(progress 可能因强杀缺失 final):
274
- done → 「任务完成(job=…)」;error/timeout/killed → 「任务失败(job=…)」
275
-
276
- 排序:**job_id 名升序**(`sorted(listdir)`,确定性),**不是**时间序——
277
- 契约 v2(`docs/plans/全中文编码与蜂巢任务标识契约_v2.0.md` §四.5)起 id 改为
278
- 语义四槽 `h_<身份>_<任务>_<单元>_<编号>`,名序不再等于提交时序:旧形态
279
- `h<unix_ms>_<pid>` 才是「名序 = 时间序」的**巧合代理**。据实订正(C2:本仓
280
- 禁止把代理当结构保证)——本源的时序由来源行自带的 ts 承载,排序只求
281
- **跨 job 全序且确定**(seq 由本源按枚举顺序递增,ingest 去重键 = (session, seq),
282
- 名序的确定性保证该键跨轮稳定)。
283
- 诚实边界(**已发现、未在本批动码**):本源的 `_jobs()` 若需要**真时间序**,
284
- 应改走与 Rust 侧同判据的 created_ts 真值(`hive/src/job.rs::list_jobs_by_created`
285
- 的语义,Python 侧对应 `mcp_server._list_jobs_by_created`)——契约 §四.5 的消费者
286
- 清单未列本源,且改动牵动 md_cg 摄入链语义与其守卫,故留待单独裁决,不静默改。
287
- 解析失败的行/条目计入 self.skipped,不终杀批次。
288
- """
289
-
290
- name = "hive_jobs"
291
-
292
- def __init__(self, root: str, error_sensitivity: str = "private"):
293
- self.root = os.path.abspath(root)
294
- self.skipped = 0
295
- # §5.5「error:true 建议 private」——默认 private;调用方 clearance 不足时
296
- # 可显式降级 error_sensitivity="internal"(显式权衡:接受失败细节入 internal,
297
- # 换取误差归因原料不丢)。能否落盘由写入者 clearance 裁决(写隔离)。
298
- self.error_sensitivity = error_sensitivity
299
-
300
- def key(self):
301
- """多仓/多池隔离:root 参与键(watermark 按源路径各自推进)。"""
302
- return f"hive_jobs:{self.root}"
303
-
304
- def _jobs(self):
305
- try:
306
- return sorted(d for d in os.listdir(self.root)
307
- if d.startswith("h")
308
- and os.path.isdir(os.path.join(self.root, d)))
309
- except OSError:
310
- return []
311
-
312
- def _read_json(self, path):
313
- try:
314
- with open(path, encoding="utf-8") as f:
315
- return json.load(f)
316
- except (OSError, ValueError):
317
- self.skipped += 1
318
- return None
319
-
320
- def events(self):
321
- seq = 0
322
- for job_id in self._jobs():
323
- jdir = os.path.join(self.root, job_id)
324
- sess = f"hive:{job_id}"
325
- spec = self._read_json(os.path.join(jdir, "spec.json")) or {}
326
- task = str(spec.get("user_prompt") or "")[:200]
327
-
328
- has_final = False
329
- prog = os.path.join(jdir, "progress.jsonl")
330
- if os.path.isfile(prog):
331
- try:
332
- fh = open(prog, encoding="utf-8", errors="replace")
333
- except OSError:
334
- fh = None
335
- if fh is not None:
336
- with fh:
337
- for line in fh:
338
- line = line.strip()
339
- if not line:
340
- continue
341
- try:
342
- o = json.loads(line)
343
- except ValueError:
344
- self.skipped += 1
345
- continue
346
- kind = o.get("kind")
347
- t = trust.epoch_seconds(float(o.get("ts") or 0)) or 0.0
348
- if kind == "start":
349
- seq += 1
350
- yield {"t": t, "seq": seq, "session": sess,
351
- "role": "user",
352
- "text": f"任务开始(job={job_id}):{task}"}
353
- elif kind == "handoff":
354
- seq += 1
355
- has_final = True
356
- yield {"t": t, "seq": seq, "session": sess,
357
- "role": "assistant",
358
- "text": f"续跑卡(job={job_id}):"
359
- f"{str(o.get('summary') or '')[:400]}"}
360
- elif kind == "error":
361
- seq += 1
362
- ev = {"t": t, "seq": seq, "session": sess,
363
- "role": "assistant",
364
- "text": f"任务失败(job={job_id}):"
365
- f"{str(o.get('error') or '')[:300]}"}
366
- if self.error_sensitivity:
367
- ev["sensitivity"] = self.error_sensitivity
368
- yield ev
369
- elif kind == "final":
370
- seq += 1
371
- has_final = True
372
- yield {"t": t, "seq": seq, "session": sess,
373
- "role": "assistant",
374
- "text": str(o.get("content_head") or "")[:400]}
375
- # tool / budget_stop / force_final / 其它 → 纯跳过
376
- # (§5.3 原设计为聚合计数,实现从简——终态由 result.json
377
- # 权威确认承载,无需逐条累计;防流水账爆炸目标不变。
378
- # 注释曾照抄设计稿口径称「仅聚合计数」,与实现漂移,
379
- # zcode 外评抓出,2026-09-23 修正)
380
-
381
- # result.json 终态权威确认(progress 可能因强杀缺失 final)
382
- res = self._read_json(os.path.join(jdir, "result.json")) or {}
383
- state = "done" if res.get("ok") is True else \
384
- ("error" if res.get("ok") is False else None)
385
- if state is None:
386
- continue
387
- t = trust.epoch_seconds(float(res.get("finished_ts") or 0))
388
- if not t:
389
- # 兜底(批次8 实测缺陷):exec_cmd 旧版 result 无 finished_ts,
390
- # t=0 会被水位整片过滤(确定性任务事件永远摄不进来)——
391
- # 回落 result.json 的 mtime(产物诞生时间,同「产物说了算」族)
392
- rpath = os.path.join(jdir, "result.json")
393
- t = trust.epoch_seconds(os.path.getmtime(rpath)) if \
394
- os.path.isfile(rpath) else 0.0
395
- t = t or 0.0
396
- head = str(res.get("content") or "")[:300]
397
- err = str(res.get("error") or "")[:300]
398
- seq += 1
399
- if state == "done":
400
- # 前缀独立成行:content 的行首结构(命令行等)不被破坏,
401
- # 保证 mine_fix_pairs 的 _FIX_RE 行首启发式仍能命中修复证据
402
- text = f"任务完成(job={job_id}):\n{head}"
403
- # 重放命令行(批次8):cmd 任务(exec_cmd)的 result.steps 携带
404
- # 原始 argv——附到事件正文,事件自身携带「做了什么」的可复放
405
- # 信息,同时命令形态可被 _FIX_RE 行首启发式命中(fix-pair 原料)
406
- steps = res.get("steps")
407
- if isinstance(steps, list) and steps:
408
- argv = (steps[-1] or {}).get("command") or []
409
- if argv:
410
- text += "\n" + " ".join(str(a) for a in argv)
411
- yield {"t": t, "seq": seq, "session": sess, "role": "assistant",
412
- "text": text}
413
- else:
414
- tag = "超时强杀" if st_err_is_timeout(err) else "失败"
415
- ev = {"t": t, "seq": seq, "session": sess, "role": "assistant",
416
- "text": f"任务{tag}(job={job_id}):{err or head}"
417
- + ("" if has_final else "(progress 缺 final,"
418
- "本条为 result 终态补位)")}
419
- if self.error_sensitivity:
420
- ev["sensitivity"] = self.error_sensitivity
421
- yield ev
422
-
423
-
424
- def st_err_is_timeout(err: str) -> bool:
425
- return "超时" in (err or "")
426
-
427
-
428
- # --------------------------------------------------------------------------
429
- # 摄取器
430
- # --------------------------------------------------------------------------
431
-
432
- # 生效条件:required 形参 cg 总被存入并以其 cg.root 拼出 self.path=os.path.join(cg.root,"_sources.json");layer/sensitivity 原样存入,未传时取模块级常量 SESSION_LAYER、SESSION_SENSITIVITY 作为默认值。
433
- class Ingestor:
434
- """增量摄取:watermark + 去重 + 写节点 + 自动 fix-pair 挖掘。
435
-
436
- watermark 文件:<root>/_sources.json
437
- {source_key: {"t": 最后时间戳(epoch 秒), "seq": 最后序号, "count": 已摄取条数}}
438
-
439
- 存量水位兼容:**旧版写入的是毫秒**,读回时经 `trust.epoch_seconds` 归一到秒,
440
- 故升级后无需清 `_sources.json` 重建(否则 `ev["t"] < last_t` 恒真 → 新事件全被跳过)。
441
- """
442
-
443
- # 生效条件:传入带 root 的 cg 即成立,layer/sensitivity 默认 SESSION_LAYER/SESSION_SENSITIVITY 并原样存为属性,路径为 os.path.join(cg.root, "_sources.json")。
444
- def __init__(self, cg, layer: str = SESSION_LAYER,
445
- sensitivity: str = SESSION_SENSITIVITY):
446
- self.cg = cg
447
- self.layer = layer
448
- self.sensitivity = sensitivity
449
- self.path = os.path.join(cg.root, "_sources.json")
450
-
451
- # ---- watermark ----
452
-
453
- # 生效条件:self.path 存在、json.load 成功且结果为 dict 时返回该 dict;path 不存在、抛 ValueError/OSError 或结果非 dict 时返回 {"schema": 1, "sources": {}}。
454
- def _load(self):
455
- if os.path.exists(self.path):
456
- try:
457
- with open(self.path, encoding="utf-8") as f:
458
- d = json.load(f)
459
- if isinstance(d, dict):
460
- return d
461
- except (ValueError, OSError):
462
- pass
463
- return {"schema": 1, "sources": {}}
464
-
465
- # 生效条件:传入 d 时以 ensure_ascii=False/indent=1 写入 self.path+".tmp",再 publish(带 Windows 短重试的 os.replace)覆盖 self.path。
466
- def _save(self, d):
467
- tmp = self.path + ".tmp"
468
- with open(tmp, "w", encoding="utf-8") as f:
469
- json.dump(d, f, ensure_ascii=False, indent=1)
470
- publish(tmp, self.path)
471
-
472
- # 生效条件:key 命中 self._load()["sources"] 时返回其值,缺 key 时返回 {}(_load 结果缺 "sources" 键则抛 KeyError)。
473
- def watermark(self, key: str):
474
- return self._load()["sources"].get(key, {})
475
-
476
- # 生效条件:调用即返回 dict(self._load()["sources"]) 的浅拷贝(_load 结果缺 "sources" 键则抛 KeyError)。
477
- def watermarks(self):
478
- return dict(self._load()["sources"])
479
-
480
- # ---- 摄取 ----
481
-
482
- # 生效条件:对必需形参 source,先按 watermark(source.key()) 跳过 t<last_t(wm 的 "t" 为假值时视作 0)及 t==last_t 且 last_seq 非 None 且 ev["seq"] 为 int 且 <= last_seq 的事件、并按 (session, seq) 去重,max_events 为真值(非 0/None)时取满即停;dry_run 为假时逐条 cg.add(nid 已在 cg.index["nodes"] 中则跳过,cg.add 抛异常则 denied+=1、记 last_error 并继续),denied 与 last_error 同时成立时补 last_error/hint 与 denied_events 明细,mine_fix_pairs 为真且 new_events 非空且非 dry_run 时以 as_proposals=True 调 mine_fix_pairs(自动产物走审核队列)并结果附 fix_pairs,new_events 非空且非 dry_run 时以末事件写回水位(count 累加 written),最后返回 result。
483
- def ingest(self, source, mine_fix_pairs: bool = False, max_events: int = None,
484
- dry_run: bool = False):
485
- """摄取一个源的新事件。返回统计。
486
-
487
- 去重键:(session, seq) —— 同一事件不重复入库。
488
- 增量:只处理 (t, seq) 大于 watermark 的事件。
489
- mine_fix_pairs 默认 False(先落账后挖矿——§5.5 系统纪律化,
490
- zcode 外评 break#2:默认 True 曾使 file/jsonl/mdcg_ingest 三入口
491
- 绕过审核队列直写 knowledge 层);显式开启时产物走 propose 队列。
492
- """
493
- key = source.key()
494
- wm = self.watermark(key)
495
- # 水位单位归一:**存量水位是毫秒**(旧版 `_ts` 产出),不归一会让
496
- # `ev["t"] < last_t` 恒真 → 升级后新事件被整片误判为「已处理」而跳过。
497
- last_t, last_seq = trust.epoch_seconds(float(wm.get("t") or 0)) or 0.0, \
498
- wm.get("seq")
499
- new_events, seen = [], set()
500
- denied_events = [] # 被拒事件明细(session/seq/原因)——可观测可重放
501
- for ev in source.events():
502
- if ev.get("t", 0) < last_t:
503
- continue
504
- if ev.get("t", 0) == last_t and last_seq is not None \
505
- and isinstance(ev.get("seq"), int) and ev["seq"] <= last_seq:
506
- continue
507
- dedup = (ev.get("session"), ev.get("seq"))
508
- if dedup in seen:
509
- continue
510
- seen.add(dedup)
511
- new_events.append(ev)
512
- if max_events and len(new_events) >= max_events:
513
- break
514
-
515
- written, ids, denied = 0, [], 0
516
- if not dry_run:
517
- for ev in new_events:
518
- nid = "src_%s_%s" % (_sig(key)[:6], _sig(
519
- f"{ev.get('session')}:{ev.get('seq')}")[:10])
520
- if nid in self.cg.index["nodes"]:
521
- continue # 幂等
522
- body = ("# 功能名:会话事件\n"
523
- f"# 生效条件:检索「{str(ev.get('text'))[:20]}」\n"
524
- "# 子功能:记录会话事件\n"
525
- f"# 执行:{str(ev.get('text'))[:80]}\n"
526
- "# 验证方式:data(会话原始记录)\n"
527
- "# 不适用条件:其它会话\n\n"
528
- f"{ev.get('text')}\n")
529
- try:
530
- self.cg.add(nid, body, layer=self.layer, role=ev.get("role"),
531
- tags=["session", key],
532
- # 事件级密级覆盖(M6 §5.5):error 事件建议 private
533
- # (失败细节可能含路径/配置),缺省回落源级默认
534
- sensitivity=ev.get("sensitivity") or self.sensitivity,
535
- verification_basis="data",
536
- condition_space={"observation_position": key,
537
- "observation_tool": "会话流",
538
- "time_window": [ev.get("t") or 0,
539
- ev.get("t") or 0]})
540
- except Exception as exc: # noqa: BLE001 —— 权限/层错误不中断整批
541
- denied += 1
542
- # 诚实化(M6):denied 事件虽被水位跳过,但明细必须可见——
543
- # 静默丢失会吞掉误差归因闭环的原料(error 事件恰是原料)
544
- denied_events.append({
545
- "session": ev.get("session"), "seq": ev.get("seq"),
546
- "error": str(exc)[:150]})
547
- self.last_error = f"{type(exc).__name__}: {exc}"
548
- continue
549
- ids.append(nid)
550
- written += 1
551
-
552
- result = {"source": key, "new_events": len(new_events), "written": written,
553
- "denied": denied, "ids": ids, "dry_run": dry_run,
554
- "sensitivity": self.sensitivity}
555
- if denied_events:
556
- result["denied_events"] = denied_events
557
- if denied and getattr(self, "last_error", None):
558
- result["last_error"] = self.last_error
559
- result["hint"] = ("会话内容默认 sensitivity=private;"
560
- "调用方需 MDCG_CLEARANCE=private 才能写入")
561
- # 自动 fix-pair 挖掘(对标 deja-vu:错误→修复)——as_proposals=True:
562
- # 自动管线产物走审核队列,绝不直写 knowledge 层(§5.5 系统纪律)
563
- if mine_fix_pairs and new_events and not dry_run:
564
- result["fix_pairs"] = self.cg.mine_fix_pairs(
565
- [{"role": e.get("role"), "text": e.get("text")}
566
- for e in new_events], as_proposals=True)
567
-
568
- if new_events and not dry_run:
569
- d = self._load()
570
- last = new_events[-1]
571
- d["sources"][key] = {
572
- "t": last.get("t") or last_t,
573
- "seq": last.get("seq"),
574
- "count": (wm.get("count") or 0) + written,
575
- "updated_at": time.time(),
576
- "path": getattr(source, "path", None),
577
- }
578
- self._save(d)
579
- return result
580
-
581
-
582
- # 生效条件:text 为 None 或假值时按 "" 参与 sha1;n 默认 12,返回 hexdigest 前 n 位(n=0 得空串)。
583
- def _sig(text: str, n: int = 12) -> str:
584
- import hashlib
585
- return hashlib.sha1((text or "").encode("utf-8")).hexdigest()[:n]
586
-
587
-
588
- # --------------------------------------------------------------------------
589
- # 摄取分派(P0 · ingest op):按扩展名选摄取方式,单一入口吃多种文件
590
- # --------------------------------------------------------------------------
591
- #
592
- # 三条摄取链:
593
- # session —— 会话流(.jsonl):走 Ingestor(watermark + 去重 + fix-pair)
594
- # doc —— 文档(.md/.txt/...):走 docindex.extract + refindex.add_items
595
- # code —— 代码(.py/.ts/...):走 codeindex.extract + refindex.add_items
596
- #
597
- # 设计要点:
598
- # - 注册表 INGEST_REGISTRY 是唯一真源:新增后缀只改这里。
599
- # - dir 动作分链处理:目录里 doc / code / jsonl 混放时各链互不干扰。
600
- # - 幂等:沿用 refindex.Ledger(size+mtime 水位)与 Ingestor watermark。
601
- # - 预演:dry_run=True 只统计、不写入(对应计划「可预演」要求)。
602
-
603
- INGEST_ACTIONS = ("file", "dir", "jsonl", "stat", "hive")
604
-
605
- INGEST_REGISTRY = {
606
- # 会话流
607
- ".jsonl": "session", ".ndjson": "session",
608
- # 文档
609
- ".md": "doc", ".markdown": "doc", ".txt": "doc", ".rst": "doc",
610
- ".html": "doc", ".htm": "doc",
611
- # 代码
612
- ".py": "code", ".js": "code", ".mjs": "code", ".ts": "code",
613
- ".tsx": "code", ".jsx": "code", ".go": "code", ".rs": "code",
614
- ".java": "code", ".kt": "code", ".swift": "code", ".rb": "code",
615
- ".php": "code", ".cs": "code", ".cpp": "code", ".cc": "code",
616
- ".c": "code", ".h": "code", ".hpp": "code",
617
- }
618
-
619
-
620
- # 生效条件:path 为 None 或空串时 splitext 得 "" 且未登记 → 返回 None;扩展名(小写)存在于 INGEST_REGISTRY 时返回其 kind。
621
- def dispatch_of(path: str):
622
- """按扩展名返回摄取方式(session / doc / code);未登记返回 None。"""
623
- return INGEST_REGISTRY.get(os.path.splitext(path or "")[1].lower())
624
-
625
-
626
- # 生效条件:required 形参 cg 总被存入;可选形参 sensitivity 为假值(None/空串)时内部 Ingestor 的 sensitivity 回落模块级常量 SESSION_SENSITIVITY,为真值时用传入的 sensitivity。
627
- class FileDispatcher:
628
- """单一入口吃多种文件:按扩展名分派到会话流 / 文档 / 代码三条摄取链。"""
629
-
630
- # 生效条件:传入 cg 即成立,sensitivity 为假值(None/空串)时所用 Ingestor 回落 SESSION_SENSITIVITY,否则用传入值。
631
- def __init__(self, cg, sensitivity=None):
632
- self.cg = cg
633
- # 会话链默认 sensitivity=private;调用方可显式覆盖(测试/受限环境)
634
- self.ingestor = Ingestor(cg, sensitivity=sensitivity or SESSION_SENSITIVITY)
635
-
636
- # ---- stat:看水位与支持面 ----
637
-
638
- # 生效条件:from . import refindex 与 refindex.Ledger(self.cg.root).summary() 均不抛异常时返回该 summary 结果,抛任何异常时返回 {}。
639
- def _ledger_stat(self):
640
- # `Ledger` 只有 `summary()`、**没有** `stat()`:此前写 `.stat()` 撞
641
- # AttributeError 被下面的 except 吞成 `{}`,于是 `ingest stat` 的水位面
642
- # 永远是空且无人知晓(唯一实现面是 `refindex.Ledger.summary`)。
643
- try:
644
- from . import refindex
645
- return refindex.Ledger(self.cg.root).summary()
646
- except Exception: # noqa: BLE001
647
- return {}
648
-
649
- # 生效条件:调用即返回由 INGEST_REGISTRY 按 kind 分组并排序后的 extensions、list(INGEST_ACTIONS)、self.ingestor.watermarks()、self._ledger_stat() 与固定 note。
650
- def stat(self):
651
- kinds = {}
652
- for ext, kind in INGEST_REGISTRY.items():
653
- kinds.setdefault(kind, []).append(ext)
654
- return {"ok": True, "kind": "stat", "actions": list(INGEST_ACTIONS),
655
- "extensions": {k: sorted(v) for k, v in sorted(kinds.items())},
656
- "watermarks": self.ingestor.watermarks(),
657
- "ledger": self._ledger_stat(),
658
- "note": ("注册表是唯一真源:新增后缀只改 INGEST_REGISTRY。"
659
- "watermarks=会话流水位;ledger=文档/代码文件水位。")}
660
-
661
- # ---- 单文件 ----
662
-
663
- # 生效条件:dispatch_of(path) 为 None 时返回 ok=False 的「不支持的后缀」结果;path 不是文件时返回 ok=False 的「文件不存在」结果;kind=="session" 时转 ingest_jsonl(path, dry_run=dry_run)(layer/sensitivity 不参与);其它 kind 转 _ingest_doc_or_code(path, kind, layer=layer, sensitivity=sensitivity, dry_run=dry_run)。
664
- def ingest_file(self, path, layer=None, sensitivity=None, dry_run=False):
665
- kind = dispatch_of(path)
666
- if kind is None:
667
- ext = os.path.splitext(path or "")[1] or "(无后缀)"
668
- return {"ok": False, "path": path, "error": f"不支持的后缀:{ext}",
669
- "supported": sorted(INGEST_REGISTRY)}
670
- if not os.path.isfile(path):
671
- return {"ok": False, "path": path, "error": "文件不存在"}
672
- if kind == "session":
673
- return self.ingest_jsonl(path, dry_run=dry_run)
674
- return self._ingest_doc_or_code(path, kind, layer=layer,
675
- sensitivity=sensitivity, dry_run=dry_run)
676
-
677
- # 生效条件:kind=="code" 用 codeindex 否则用 docindex;mod.extract 抛 ValueError 时返回 ok=False 的「抽取失败」;dry_run 为真时只返回 items 计数与前 20 个 node_id 不写盘;否则经 refindex.add_items(kind 为 code_ref/doc_ref)写入并返回 indexed、ids[:20] 与 sensitivity。
678
- def _ingest_doc_or_code(self, path, kind, layer=None, sensitivity=None,
679
- dry_run=False):
680
- from . import codeindex, docindex, refindex
681
- mod = codeindex if kind == "code" else docindex
682
- rel = os.path.basename(path)
683
- with open(path, encoding="utf-8", errors="replace") as f:
684
- src = f.read()
685
- try:
686
- items = mod.extract(src, path=rel,
687
- suffix=os.path.splitext(path)[1].lower())
688
- except ValueError as exc:
689
- return {"ok": False, "path": path, "error": f"抽取失败:{exc}"}
690
- items = items or []
691
- if dry_run:
692
- return {"ok": True, "dry_run": True, "kind": kind, "path": path,
693
- "items": len(items),
694
- "ids": [mod.node_id(i) for i in items[:20]],
695
- "note": "预演:只抽取计数,未写盘"}
696
- ref_kind = "code_ref" if kind == "code" else "doc_ref"
697
- ids, sens = refindex.add_items(
698
- self.cg, items, kind=ref_kind,
699
- root=os.path.dirname(path) or ".", layer=layer,
700
- sensitivity=sensitivity)
701
- return {"ok": True, "kind": kind, "path": path, "items": len(items),
702
- "indexed": len(ids), "ids": ids[:20], "sensitivity": sens}
703
-
704
- # ---- 目录 ----
705
-
706
- # 生效条件:调用即 os.walk(root) 统计每个文件名经 dispatch_of 得到的 kind(无匹配记 "unsupported")并返回 dry_run 预演计数结果。
707
- def _dry_dir(self, root):
708
- counts = {}
709
- for _dp, _dn, fns in os.walk(root):
710
- for name in fns:
711
- k = dispatch_of(name) or "unsupported"
712
- counts[k] = counts.get(k, 0) + 1
713
- return {"ok": True, "dry_run": True, "kind": "dir", "root": root,
714
- "counts": counts,
715
- "note": "预演:仅统计各链文件数,未做任何写入"}
716
-
717
- # 生效条件:root 非目录时返回 ok=False 的「目录不存在」;dry_run 为真时返回 _dry_dir(root);否则对 doc_ref/code_ref 两链各以 patterns/max_files/max_items/incremental/ledger 调 refindex.index_dir(commit=False) 与 add_items 后 ledger.save(),并把 root 下 **/*.jsonl 前 max_files 个逐个 ingest_jsonl 后返回 out。
718
- def ingest_dir(self, root, layer=None, sensitivity=None, patterns=None,
719
- max_files=500, max_items=2000, incremental=False,
720
- dry_run=False):
721
- from . import refindex
722
- if not os.path.isdir(root):
723
- return {"ok": False, "error": f"目录不存在:{root}"}
724
- if dry_run:
725
- return self._dry_dir(root)
726
- ledger = refindex.Ledger(self.cg.root)
727
- out = {"ok": True, "kind": "dir", "root": root, "chains": {}}
728
- for ref_kind, key in (("doc_ref", "doc"), ("code_ref", "code")):
729
- items, errors, stats = refindex.index_dir(
730
- root, kind=ref_kind, patterns=patterns, max_files=max_files,
731
- max_items=max_items, incremental=incremental, ledger=ledger,
732
- commit=False)
733
- ids, sens = refindex.add_items(self.cg, items, kind=ref_kind,
734
- root=root, layer=layer,
735
- sensitivity=sensitivity)
736
- # 写序(同 op=index_code/index_doc):节点先落盘,水位随后——
737
- # 中途被杀只会留下「节点新 + 水位旧」,下次增量重切,不会静默漏漂移。
738
- ledger.save()
739
- out["chains"][key] = {
740
- "indexed": len(ids), "errors": len(errors),
741
- "files": stats.get("files"), "truncated": stats.get("truncated"),
742
- "empty_scan": stats.get("empty_scan"),
743
- "skipped_unchanged": stats.get("skipped_unchanged", 0),
744
- "skipped_suffixes": stats.get("skipped_suffixes", []),
745
- "sensitivity": sens}
746
- # 会话流(.jsonl)逐个增量摄取
747
- jsons = sorted(glob.glob(os.path.join(root, "**", "*.jsonl"),
748
- recursive=True))[:max_files]
749
- ses = []
750
- for p in jsons:
751
- r = self.ingest_jsonl(p)
752
- ses.append({"path": p, "written": r.get("written", 0),
753
- "new_events": r.get("new_events", 0)})
754
- out["chains"]["session"] = {"files": len(jsons), "results": ses}
755
- return out
756
-
757
- # ---- 会话流 ----
758
-
759
- @staticmethod
760
- # 生效条件:required 形参 path 能被 open(...,encoding="utf-8",errors="replace") 打开时读前 50000 字符,head 含 '"user/message"'、'"assistant/message"'、'"tool/call"' 任一标记则返回 DSHSessionSource(path),否则返回 JsonlSource(path);打开抛 OSError 时直接返回 JsonlSource(path)。
761
- def _auto_source(path):
762
- """通用 JSONL vs DSH 会话:按内容探测,避免调用方选错源类型。"""
763
- try:
764
- with open(path, encoding="utf-8", errors="replace") as f:
765
- head = f.read(50000)
766
- except OSError:
767
- return JsonlSource(path)
768
- for marker in ('"user/message"', '"assistant/message"', '"tool/call"'):
769
- if marker in head:
770
- return DSHSessionSource(path)
771
- return JsonlSource(path)
772
-
773
- # 生效条件:path 不是文件时返回 ok=False 的「文件不存在」;否则经 _auto_source(path) 选源后调 self.ingestor.ingest(src, dry_run=dry_run, max_events=max_events),并补上 ok=True/kind/path/source_class 后返回。
774
- def ingest_jsonl(self, path, dry_run=False, max_events=None):
775
- if not os.path.isfile(path):
776
- return {"ok": False, "error": f"文件不存在:{path}"}
777
- src = self._auto_source(path)
778
- res = self.ingestor.ingest(src, dry_run=dry_run, max_events=max_events)
779
- res.update({"ok": True, "kind": "session", "path": path,
780
- "source_class": type(src).__name__})
781
- return res
782
-
783
-
784
- # 生效条件:cg 必填;action 为 None/空串时 (action or "stat") 归为 stat;file/jsonl 缺 path 返回 ok=False;dir 的 max_files/max_items 走 int(x or 500)/int(x or 2000),传 0 也变 500/2000;未知 action 抛 ValueError。
785
- def run(cg, action: str = "stat", **kw):
786
- """ingest op 唯一入口。"""
787
- act = (action or "stat").strip().lower()
788
- d = FileDispatcher(cg, sensitivity=kw.get("sensitivity"))
789
- if act == "stat":
790
- return d.stat()
791
- if act == "file":
792
- p = kw.get("path")
793
- if not p:
794
- return {"ok": False, "error": "file 动作需要 path"}
795
- return d.ingest_file(p, layer=kw.get("layer"),
796
- sensitivity=kw.get("sensitivity"),
797
- dry_run=bool(kw.get("dry_run")))
798
- if act == "dir":
799
- p = kw.get("path") or cg.root
800
- return d.ingest_dir(p, layer=kw.get("layer"),
801
- sensitivity=kw.get("sensitivity"),
802
- patterns=kw.get("patterns"),
803
- max_files=int(kw.get("max_files") or 500),
804
- max_items=int(kw.get("max_items") or 2000),
805
- incremental=bool(kw.get("incremental")),
806
- dry_run=bool(kw.get("dry_run")))
807
- if act == "jsonl":
808
- p = kw.get("path")
809
- if not p:
810
- return {"ok": False, "error": "jsonl 动作需要 path"}
811
- return d.ingest_jsonl(p, dry_run=bool(kw.get("dry_run")),
812
- max_events=kw.get("max_events"))
813
- if act == "hive":
814
- # M6:蜂巢任务事件源(hive/jobs → contextual)。root 解析链:显式 path >
815
- # env MDCG_HIVE_JOBS > 报错指引(fail-closed,不猜仓库布局)。
816
- # §5.5 硬纪律:mine_fix_pairs=False(先落账后挖矿——自动挖掘产物直写
817
- # knowledge 层违反双轨制,实测 0→2 污染);默认密级 internal,
818
- # error 事件由源层 per-event 覆写 private(失败细节可能含路径/配置)。
819
- root = kw.get("path") or os.environ.get("MDCG_HIVE_JOBS")
820
- if not root:
821
- return {"ok": False, "error": (
822
- "hive 动作需要 path(hive/jobs 目录),或设 env MDCG_HIVE_JOBS——"
823
- "不猜测仓库布局(fail-closed)")}
824
- src = HiveJobsSource(root, error_sensitivity=kw.get("error_sensitivity")
825
- if kw.get("error_sensitivity") is not None else "private")
826
- rep = Ingestor(cg, layer=kw.get("layer") or "contextual",
827
- sensitivity=kw.get("sensitivity") or "internal"
828
- ).ingest(src, mine_fix_pairs=False,
829
- max_events=kw.get("max_events"),
830
- dry_run=bool(kw.get("dry_run")))
831
- rep["source"] = src.key()
832
- rep["skipped"] = src.skipped
833
- return rep
1
+ # -*- coding: utf-8 -*-
2
+ """md_cg · 设备驱动(记忆 OS #3):把外部会话流接进认知图
3
+
4
+ 设计:
5
+ Source(事件源)—— 把某种外部存储解析成统一事件流:
6
+ {"t": epoch 秒, "seq": 序号, "role": ..., "text": ..., "session": ..., "cwd": ...}
7
+
8
+ ⚠ **单位纪律(issue #23)**:`t` 一律 **epoch 秒**,与图内 `created_at` /
9
+ `condition_space.time_window` / `trust.parse_time` 同口径。外部源里的 13 位毫秒
10
+ (DSH 的 `time` 字段、通用 JSONL 的毫秒戳)在**源适配层**经 `trust.epoch_seconds`
11
+ 归一后即不外溢——否则 `condition_space.time_window` 会落毫秒,把
12
+ `stg(op=timeline)` 的倒序头部整片占满并顶掉 auto-recall。
13
+
14
+ Ingestor(摄取器)—— 增量 watermark + 去重 + 写节点 + 自动 fix-pair 挖掘。
15
+
16
+ 内置源:
17
+ · JsonlSource —— 通用 JSONL(每行一个事件对象)
18
+ · DSHSessionSource —— DeepSeek Harness 会话(~/.dsh/sessions/**/session.jsonl[.zstd])
19
+ zstd 为**可选**依赖:缺失时优雅降级(跳过 .zstd 文件并告警)
20
+
21
+ 默认敏感度:会话内容是私有记忆 → sensitivity="private"(避免落入公开根)。
22
+
23
+ 零第三方依赖(zstd 为可选增强)。
24
+ """
25
+ from __future__ import annotations
26
+
27
+ import glob
28
+ import json
29
+ import os
30
+ import time
31
+
32
+ from . import trust
33
+ from .fsutil import publish
34
+ from .security import DEFAULT_SENSITIVITY
35
+
36
+ # 会话事件的默认落层与敏感度
37
+ SESSION_LAYER = "contextual"
38
+ SESSION_SENSITIVITY = "private"
39
+
40
+
41
+ # --------------------------------------------------------------------------
42
+ # 事件源
43
+ # --------------------------------------------------------------------------
44
+
45
+ # 生效条件:无必填构造形参,类常量 name="source" 即实例默认;仅当子类覆写 events() 时才产出事件,基类 events() 恒抛 NotImplementedError。
46
+ class Source:
47
+ """事件源基类。"""
48
+
49
+ name = "source"
50
+
51
+ #: #466:**来源声明面**(本件新增)。外部内容经 ingest 落库时,基底/密级取自来源声明,
52
+ #: 不再由写入点硬编码——「同一份外部文本走不同链得到不同基底」的根因就是硬编码。
53
+ #: 基底缺省 `untrusted`:在 `nodefile.VERIFICATION_BASIS`(**可接受集**)内,
54
+ #: 但**不在** `forgetting.VERIFIED_BASES`(**已验**集)里,也不在理科/文科两档内——
55
+ #: 外部内容本就不应作为可信基底进入权威记忆。
56
+ #: 密级缺省 `internal`(**本件不降级**:降 public 会放宽外带面,与本件方向相反)。
57
+ default_basis = "untrusted"
58
+ default_sensitivity = "internal"
59
+
60
+ # 生效条件:任何调用都直接 raise NotImplementedError(基类占位,无其它分支)。
61
+ def events(self):
62
+ raise NotImplementedError
63
+
64
+ # 生效条件:无前置;返回类常量 self.name(基类为 "source"),作为 watermark 的稳定标识,不含路径与运行期状态;
65
+ def key(self):
66
+ """源的稳定标识(用于 watermark)。"""
67
+ return self.name
68
+
69
+
70
+ # 生效条件:required 形参 path 总被存入 self.path;可选形参 name 为假值(None/空串)时 self.name 回落到 "jsonl:"+os.path.basename(path),为真值时用 name 本身,t_key/role_key/text_key/default_role 原样存入属性。
71
+ class JsonlSource(Source):
72
+ """通用 JSONL 会话源。
73
+
74
+ 每行是事件对象;字段映射可配置:
75
+ t_key —— 时间戳字段(默认 "time",也接受 ISO 字符串)
76
+ role_key —— 角色字段(默认 "role")
77
+ text_key —— 文本字段(默认 "text")
78
+ """
79
+
80
+ default_basis = "untrusted" # #466:外部 JSONL 会话 ⇒ 未验基底
81
+ default_sensitivity = "internal"
82
+
83
+ # 生效条件:传入 path;name 为假值(None/空串)时回落为 "jsonl:"+os.path.basename(path),t_key/role_key/text_key/default_role 原样存为属性(默认值 "time"/"role"/"text"/"user")。
84
+ def __init__(self, path: str, name: str = None, t_key="time", role_key="role",
85
+ text_key="text", default_role="user"):
86
+ self.path = path
87
+ self.name = name or ("jsonl:" + os.path.basename(path))
88
+ self.t_key, self.role_key, self.text_key = t_key, role_key, text_key
89
+ self.default_role = default_role
90
+
91
+ # 生效条件:无前置;返回 self.name(构造时已回落为 "jsonl:"+basename(path)),只随构造参数变化、不随文件内容变化;
92
+ def key(self):
93
+ return self.name
94
+
95
+ # 生效条件:o.get(self.t_key) 经 float 可转数值时返回 trust.epoch_seconds(该值)(秒值原样、13 位毫秒 /1000 归一到**秒**);float 抛 TypeError/ValueError 且 v 为字符串时按 v[:19] 与 "%Y-%m-%dT%H:%M:%S" 解析,成功返回 mktime(**秒**)、抛 ValueError 返回 0.0;键缺失/None/其它不可转类型返回 0.0。
96
+ def _ts(self, o):
97
+ v = o.get(self.t_key)
98
+ try:
99
+ return trust.epoch_seconds(float(v)) or 0.0
100
+ except (TypeError, ValueError):
101
+ pass
102
+ if isinstance(v, str):
103
+ try:
104
+ return time.mktime(time.strptime(v[:19], "%Y-%m-%dT%H:%M:%S"))
105
+ except ValueError:
106
+ return 0.0
107
+ return 0.0
108
+
109
+ # 生效条件:os.path.exists(self.path) 为真时逐行产出,空行、json.loads 抛 ValueError、o.get(self.text_key) 为假的行被跳过,产出项为 t=self._ts(o)、seq=o.get("seq", i)、role=o.get(self.role_key) or self.default_role、text=str(text)、session/cwd 取 o 同名键;path 不存在时直接 return 不产出。
110
+ def events(self):
111
+ if not os.path.exists(self.path):
112
+ return
113
+ with open(self.path, encoding="utf-8", errors="replace") as f:
114
+ for i, line in enumerate(f):
115
+ line = line.strip()
116
+ if not line:
117
+ continue
118
+ try:
119
+ o = json.loads(line)
120
+ except ValueError:
121
+ continue
122
+ text = o.get(self.text_key)
123
+ if not text:
124
+ continue
125
+ yield {"t": self._ts(o), "seq": o.get("seq", i),
126
+ "role": o.get(self.role_key) or self.default_role,
127
+ "text": str(text), "session": o.get("session"),
128
+ "cwd": o.get("cwd")}
129
+
130
+
131
+ # 生效条件:path 指向的内容可读且 zstandard 可导入时返回 StringIO(raw.decode("utf-8", errors="replace"));ImportError 时返回 None。
132
+ def _zstd_reader(path):
133
+ """返回可读的文本迭代器;zstd 不可用返回 None。"""
134
+ import io
135
+ try:
136
+ import zstandard as zstd
137
+ except ImportError:
138
+ return None
139
+ with open(path, "rb") as f:
140
+ raw = zstd.ZstdDecompressor().stream_reader(f).read()
141
+ return io.StringIO(raw.decode("utf-8", errors="replace"))
142
+
143
+
144
+ # 生效条件:required 形参 path 总被存入 self.path,可选形参 include_reasoning 原样存入,self.name 恒为 "dsh:"+os.path.basename(os.path.dirname(path))(与 include_reasoning 取值无关)。
145
+ class DSHSessionSource(Source):
146
+ """DeepSeek Harness 会话源。
147
+
148
+ 事件映射:
149
+ user/message → role=user
150
+ assistant/message → role=assistant(只取 content[].text,reasoning 默认丢弃)
151
+ tool/call → role=command("name(args)" 形式)
152
+ tool/result → role=tool-output
153
+ """
154
+
155
+ default_basis = "untrusted" # #466:外部会话转录 ⇒ 未验基底
156
+ default_sensitivity = "internal"
157
+
158
+ # 生效条件:传入 path 即成立,include_reasoning 原样存为属性(默认 False),name 固定为 "dsh:"+os.path.basename(os.path.dirname(path))。
159
+ def __init__(self, path: str, include_reasoning: bool = False):
160
+ self.path = path
161
+ self.include_reasoning = include_reasoning
162
+ self.name = "dsh:" + os.path.basename(os.path.dirname(path))
163
+
164
+ # 生效条件:无前置;返回 self.name(构造时固定为 "dsh:"+basename(dirname(path))),与 include_reasoning 取值无关;
165
+ def key(self):
166
+ return self.name
167
+
168
+ @staticmethod
169
+ # 生效条件:可选形参 root 为假值(None/空串)时改用默认目录 os.path.join(expanduser("~"),".dsh","sessions"),否则用传入 root;对 root 下递归 glob 到的 session.jsonl 与 session.jsonl.zstd 逐条 os.stat(抛 OSError 的条目跳过),按 (-st_size, path) 升序键排序(两处 glob 均无命中时为空列表),可选形参 limit 为假值(None/0)时返回全部 rows,否则返回 rows[:limit];每行为 (path, size, mtime) 三元组。
170
+ def discover_detailed(root: str = None, limit: int = None):
171
+ """候选会话 → `[(path, size, mtime), ...]`(**排序唯一真源**,`discover` 复用)。
172
+
173
+ 排序:会话**文件体积**降序 → `path` 升序(确定性终键)。
174
+ **刻意不按最近活动排序**——这正是 `source='auto'` 可能选中很久以前会话的
175
+ 原因;把 size/mtime 一并透出,让「为什么选它」在返回体里可辨
176
+ (issue #23 附带建议:先让依据可见,是否换策略另议)。
177
+ """
178
+ root = root or os.path.join(os.path.expanduser("~"), ".dsh", "sessions")
179
+ files = glob.glob(os.path.join(root, "**", "session.jsonl"), recursive=True)
180
+ files += glob.glob(os.path.join(root, "**", "session.jsonl.zstd"), recursive=True)
181
+ rows = []
182
+ for p in files:
183
+ try:
184
+ st = os.stat(p)
185
+ except OSError:
186
+ continue # 竞态删除:跳过而非整体失败
187
+ rows.append((p, st.st_size, st.st_mtime))
188
+ rows.sort(key=lambda r: (-r[1], r[0]))
189
+ return rows[:limit] if limit else rows
190
+
191
+ @staticmethod
192
+ # 生效条件:可选形参 root/limit 原样转交 discover_detailed,返回其结果的 path 列(同为体积降序、同确定性终键),limit 为假值时返回全部。
193
+ def discover(root: str = None, limit: int = None):
194
+ """候选会话路径(体积降序)——`discover_detailed` 的路径投影。"""
195
+ return [p for p, _s, _m in DSHSessionSource.discover_detailed(root, limit)]
196
+
197
+ # 生效条件:self.path 以 ".zstd" 结尾时经 _zstd_reader 逐行产出(其返回 None 时 raise RuntimeError),否则以 utf-8/errors=replace 打开 self.path 逐行产出。
198
+ def _lines(self):
199
+ if self.path.endswith(".zstd"):
200
+ fh = _zstd_reader(self.path)
201
+ if fh is None:
202
+ raise RuntimeError("zstd 不可用(pip install zstandard),跳过该会话")
203
+ try:
204
+ yield from fh
205
+ finally:
206
+ fh.close()
207
+ else:
208
+ with open(self.path, encoding="utf-8", errors="replace") as f:
209
+ yield from f
210
+
211
+ @staticmethod
212
+ # 生效条件:required 形参 content 为 str 时原样返回 content;为 list 时收集其中 str 元素及 type 属于 ("text","input-text") 且 text 为真值的 dict 元素(取 str(c["text"])),以 "\n" 连接返回(无可收集元素时为空串 "");既非 str 也非 list 时返回 ""。
213
+ def _text_of(content):
214
+ """content 可能是 [{type,text}] 或字符串。"""
215
+ if isinstance(content, str):
216
+ return content
217
+ if isinstance(content, list):
218
+ parts = []
219
+ for c in content:
220
+ if isinstance(c, dict):
221
+ if c.get("type") in ("text", "input-text") and c.get("text"):
222
+ parts.append(str(c["text"]))
223
+ elif isinstance(c, str):
224
+ parts.append(c)
225
+ return "\n".join(parts)
226
+ return ""
227
+
228
+ # 生效条件:逐行解析后按 o.get("type") 分派——"session" 只更新 sess/cwd 不产出;"user/message" 产出 role=user 与 _text_of(data.get("content"));"assistant/message" 产出 role=assistant 与 _text_of(msg.get("content")),include_reasoning 为真时再把 content 中 type=="reasoning" 且有 text 的项追加 "\n[reasoning] "+str(c["text"]);"tool/call" 产出 role=command 与 f"{name}({arguments or ''})";"tool/result" 产出 role=tool-output,仅当 inner[0] 为 dict 时 text=_text_of(inner[0].get("content"));其它 type 或 ev 中 text 为空的事件不产出。
229
+ def events(self):
230
+ sess = None
231
+ cwd = None
232
+ for line in self._lines():
233
+ line = line.strip()
234
+ if not line:
235
+ continue
236
+ try:
237
+ o = json.loads(line)
238
+ except ValueError:
239
+ continue
240
+ t = o.get("type")
241
+ data = o.get("data") or {}
242
+ if t == "session":
243
+ sess, cwd = o.get("id"), o.get("cwd")
244
+ continue
245
+ ev = {"t": trust.epoch_seconds(float(o.get("time") or 0)) or 0.0,
246
+ "seq": o.get("seq"),
247
+ "session": sess, "cwd": cwd}
248
+ if t == "user/message":
249
+ ev["role"], ev["text"] = "user", self._text_of(data.get("content"))
250
+ elif t == "assistant/message":
251
+ msg = data.get("message") or {}
252
+ ev["role"] = "assistant"
253
+ ev["text"] = self._text_of(msg.get("content"))
254
+ if self.include_reasoning:
255
+ for c in (msg.get("content") or []):
256
+ if isinstance(c, dict) and c.get("type") == "reasoning" \
257
+ and c.get("text"):
258
+ ev["text"] = (ev["text"] or "") + "\n[reasoning] " + str(c["text"])
259
+ elif t == "tool/call":
260
+ ev["role"] = "command"
261
+ ev["text"] = f"{data.get('name')}({data.get('arguments') or ''})"
262
+ elif t == "tool/result":
263
+ msg = data.get("message") or {}
264
+ inner = msg.get("content") or []
265
+ txt = ""
266
+ if inner and isinstance(inner[0], dict):
267
+ txt = self._text_of(inner[0].get("content"))
268
+ ev["role"], ev["text"] = "tool-output", txt
269
+ else:
270
+ continue
271
+ if ev.get("text"):
272
+ yield ev
273
+
274
+
275
+ # --------------------------------------------------------------------------
276
+ # 蜂巢任务事件源(M6:hive/jobs → contextual,外部设计稿 蜂巢记忆架构设计.md §5)
277
+ # --------------------------------------------------------------------------
278
+
279
+ class HiveJobsSource(Source):
280
+ """蜂巢任务事件源。
281
+
282
+ 事件映射(设计稿 §5.3):
283
+ start → role=user,任务开始(task 摘要)
284
+ tool → **跳过**(防流水账爆炸;tool_trace 已在 result 有摘要)
285
+ handoff → role=assistant,续跑卡
286
+ error → role=assistant,「任务失败(job=…):…」(fix-pair 前件)
287
+ final → role=assistant,content 头
288
+ result.json 终态 → **权威确认事件**(progress 可能因强杀缺失 final):
289
+ done → 「任务完成(job=…)」;error/timeout/killed → 「任务失败(job=…)」
290
+
291
+ 排序:**job_id 名升序**(`sorted(listdir)`,确定性),**不是**时间序——
292
+ 契约 v2(`docs/plans/全中文编码与蜂巢任务标识契约_v2.0.md` §四.5)起 id 改为
293
+ 语义四槽 `h_<身份>_<任务>_<单元>_<编号>`,名序不再等于提交时序:旧形态
294
+ `h<unix_ms>_<pid>` 才是「名序 = 时间序」的**巧合代理**。据实订正(C2:本仓
295
+ 禁止把代理当结构保证)——本源的时序由来源行自带的 ts 承载,排序只求
296
+ **跨 job 全序且确定**(seq 由本源按枚举顺序递增,ingest 去重键 = (session, seq),
297
+ 名序的确定性保证该键跨轮稳定)。
298
+ 诚实边界(**已发现、未在本批动码**):本源的 `_jobs()` 若需要**真时间序**,
299
+ 应改走与 Rust 侧同判据的 created_ts 真值(`hive/src/job.rs::list_jobs_by_created`
300
+ 的语义,Python 侧对应 `mcp_server._list_jobs_by_created`)——契约 §四.5 的消费者
301
+ 清单未列本源,且改动牵动 md_cg 摄入链语义与其守卫,故留待单独裁决,不静默改。
302
+ 解析失败的行/条目计入 self.skipped,不终杀批次。
303
+ """
304
+
305
+ name = "hive_jobs"
306
+
307
+ #: #466:hive 任务面同为**外部**内容 ⇒ 基底缺省 untrusted;
308
+ #: 密级保持 internal **不变**(本件不带宽外带面),只是改为可声明。
309
+ default_basis = "untrusted"
310
+ default_sensitivity = "internal"
311
+
312
+ def __init__(self, root: str, error_sensitivity: str = "private"):
313
+ self.root = os.path.abspath(root)
314
+ self.skipped = 0
315
+ # §5.5「error:true 建议 private」——默认 private;调用方 clearance 不足时
316
+ # 可显式降级 error_sensitivity="internal"(显式权衡:接受失败细节入 internal,
317
+ # 换取误差归因原料不丢)。能否落盘由写入者 clearance 裁决(写隔离)。
318
+ self.error_sensitivity = error_sensitivity
319
+
320
+ def key(self):
321
+ """多仓/多池隔离:root 参与键(watermark 按源路径各自推进)。"""
322
+ return f"hive_jobs:{self.root}"
323
+
324
+ def _jobs(self):
325
+ try:
326
+ return sorted(d for d in os.listdir(self.root)
327
+ if d.startswith("h")
328
+ and os.path.isdir(os.path.join(self.root, d)))
329
+ except OSError:
330
+ return []
331
+
332
+ def _read_json(self, path):
333
+ try:
334
+ with open(path, encoding="utf-8") as f:
335
+ return json.load(f)
336
+ except (OSError, ValueError):
337
+ self.skipped += 1
338
+ return None
339
+
340
+ def events(self):
341
+ seq = 0
342
+ for job_id in self._jobs():
343
+ jdir = os.path.join(self.root, job_id)
344
+ sess = f"hive:{job_id}"
345
+ spec = self._read_json(os.path.join(jdir, "spec.json")) or {}
346
+ task = str(spec.get("user_prompt") or "")[:200]
347
+
348
+ has_final = False
349
+ prog = os.path.join(jdir, "progress.jsonl")
350
+ if os.path.isfile(prog):
351
+ try:
352
+ fh = open(prog, encoding="utf-8", errors="replace")
353
+ except OSError:
354
+ fh = None
355
+ if fh is not None:
356
+ with fh:
357
+ for line in fh:
358
+ line = line.strip()
359
+ if not line:
360
+ continue
361
+ try:
362
+ o = json.loads(line)
363
+ except ValueError:
364
+ self.skipped += 1
365
+ continue
366
+ kind = o.get("kind")
367
+ t = trust.epoch_seconds(float(o.get("ts") or 0)) or 0.0
368
+ if kind == "start":
369
+ seq += 1
370
+ yield {"t": t, "seq": seq, "session": sess,
371
+ "role": "user",
372
+ "text": f"任务开始(job={job_id}):{task}"}
373
+ elif kind == "handoff":
374
+ seq += 1
375
+ has_final = True
376
+ yield {"t": t, "seq": seq, "session": sess,
377
+ "role": "assistant",
378
+ "text": f"续跑卡(job={job_id}):"
379
+ f"{str(o.get('summary') or '')[:400]}"}
380
+ elif kind == "error":
381
+ seq += 1
382
+ ev = {"t": t, "seq": seq, "session": sess,
383
+ "role": "assistant",
384
+ "text": f"任务失败(job={job_id}):"
385
+ f"{str(o.get('error') or '')[:300]}"}
386
+ if self.error_sensitivity:
387
+ ev["sensitivity"] = self.error_sensitivity
388
+ yield ev
389
+ elif kind == "final":
390
+ seq += 1
391
+ has_final = True
392
+ yield {"t": t, "seq": seq, "session": sess,
393
+ "role": "assistant",
394
+ "text": str(o.get("content_head") or "")[:400]}
395
+ # tool / budget_stop / force_final / 其它 → 纯跳过
396
+ # (§5.3 原设计为聚合计数,实现从简——终态由 result.json
397
+ # 权威确认承载,无需逐条累计;防流水账爆炸目标不变。
398
+ # 注释曾照抄设计稿口径称「仅聚合计数」,与实现漂移,
399
+ # zcode 外评抓出,2026-09-23 修正)
400
+
401
+ # result.json 终态权威确认(progress 可能因强杀缺失 final)
402
+ res = self._read_json(os.path.join(jdir, "result.json")) or {}
403
+ state = "done" if res.get("ok") is True else \
404
+ ("error" if res.get("ok") is False else None)
405
+ if state is None:
406
+ continue
407
+ t = trust.epoch_seconds(float(res.get("finished_ts") or 0))
408
+ if not t:
409
+ # 兜底(批次8 实测缺陷):exec_cmd 旧版 result 无 finished_ts,
410
+ # t=0 会被水位整片过滤(确定性任务事件永远摄不进来)——
411
+ # 回落 result.json 的 mtime(产物诞生时间,同「产物说了算」族)
412
+ rpath = os.path.join(jdir, "result.json")
413
+ t = trust.epoch_seconds(os.path.getmtime(rpath)) if \
414
+ os.path.isfile(rpath) else 0.0
415
+ t = t or 0.0
416
+ head = str(res.get("content") or "")[:300]
417
+ err = str(res.get("error") or "")[:300]
418
+ seq += 1
419
+ if state == "done":
420
+ # 前缀独立成行:content 的行首结构(命令行等)不被破坏,
421
+ # 保证 mine_fix_pairs 的 _FIX_RE 行首启发式仍能命中修复证据
422
+ text = f"任务完成(job={job_id}):\n{head}"
423
+ # 重放命令行(批次8):cmd 任务(exec_cmd)的 result.steps 携带
424
+ # 原始 argv——附到事件正文,事件自身携带「做了什么」的可复放
425
+ # 信息,同时命令形态可被 _FIX_RE 行首启发式命中(fix-pair 原料)
426
+ steps = res.get("steps")
427
+ if isinstance(steps, list) and steps:
428
+ argv = (steps[-1] or {}).get("command") or []
429
+ if argv:
430
+ text += "\n" + " ".join(str(a) for a in argv)
431
+ yield {"t": t, "seq": seq, "session": sess, "role": "assistant",
432
+ "text": text}
433
+ else:
434
+ tag = "超时强杀" if st_err_is_timeout(err) else "失败"
435
+ ev = {"t": t, "seq": seq, "session": sess, "role": "assistant",
436
+ "text": f"任务{tag}(job={job_id}):{err or head}"
437
+ + ("" if has_final else "(progress 缺 final,"
438
+ "本条为 result 终态补位)")}
439
+ if self.error_sensitivity:
440
+ ev["sensitivity"] = self.error_sensitivity
441
+ yield ev
442
+
443
+
444
+ def st_err_is_timeout(err: str) -> bool:
445
+ return "超时" in (err or "")
446
+
447
+
448
+ # --------------------------------------------------------------------------
449
+ # 摄取器
450
+ # --------------------------------------------------------------------------
451
+
452
+ # 生效条件:required 形参 cg 总被存入并以其 cg.root 拼出 self.path=os.path.join(cg.root,"_sources.json");layer/sensitivity 原样存入,未传时取模块级常量 SESSION_LAYER、SESSION_SENSITIVITY 作为默认值。
453
+ class Ingestor:
454
+ """增量摄取:watermark + 去重 + 写节点 + 自动 fix-pair 挖掘。
455
+
456
+ watermark 文件:<root>/_sources.json
457
+ {source_key: {"t": 最后时间戳(epoch 秒), "seq": 最后序号, "count": 已摄取条数}}
458
+
459
+ 存量水位兼容:**旧版写入的是毫秒**,读回时经 `trust.epoch_seconds` 归一到秒,
460
+ 故升级后无需清 `_sources.json` 重建(否则 `ev["t"] < last_t` 恒真 → 新事件全被跳过)。
461
+ """
462
+
463
+ # 生效条件:传入带 root 的 cg 即成立,layer/sensitivity 默认 SESSION_LAYER/SESSION_SENSITIVITY 并原样存为属性,路径为 os.path.join(cg.root, "_sources.json")。
464
+ def __init__(self, cg, layer: str = SESSION_LAYER,
465
+ sensitivity: str = SESSION_SENSITIVITY):
466
+ self.cg = cg
467
+ self.layer = layer
468
+ self.sensitivity = sensitivity
469
+ self.path = os.path.join(cg.root, "_sources.json")
470
+
471
+ # ---- watermark ----
472
+
473
+ # 生效条件:self.path 存在、json.load 成功且结果为 dict 时返回该 dict;path 不存在、抛 ValueError/OSError 或结果非 dict 时返回 {"schema": 1, "sources": {}}。
474
+ def _load(self):
475
+ if os.path.exists(self.path):
476
+ try:
477
+ with open(self.path, encoding="utf-8") as f:
478
+ d = json.load(f)
479
+ if isinstance(d, dict):
480
+ return d
481
+ except (ValueError, OSError):
482
+ pass
483
+ return {"schema": 1, "sources": {}}
484
+
485
+ # 生效条件:传入 d 时以 ensure_ascii=False/indent=1 写入 self.path+".tmp",再 publish(带 Windows 短重试的 os.replace)覆盖 self.path。
486
+ def _save(self, d):
487
+ tmp = self.path + ".tmp"
488
+ with open(tmp, "w", encoding="utf-8") as f:
489
+ json.dump(d, f, ensure_ascii=False, indent=1)
490
+ publish(tmp, self.path)
491
+
492
+ # 生效条件:key 命中 self._load()["sources"] 时返回其值,缺 key 时返回 {}(_load 结果缺 "sources" 键则抛 KeyError)。
493
+ def watermark(self, key: str):
494
+ return self._load()["sources"].get(key, {})
495
+
496
+ # 生效条件:调用即返回 dict(self._load()["sources"]) 的浅拷贝(_load 结果缺 "sources" 键则抛 KeyError)。
497
+ def watermarks(self):
498
+ return dict(self._load()["sources"])
499
+
500
+ # ---- 摄取 ----
501
+
502
+ # 生效条件:对必需形参 source,先按 watermark(source.key()) 跳过 t<last_t(wm 的 "t" 为假值时视作 0)及 t==last_t 且 last_seq 非 None 且 ev["seq"] 为 int 且 <= last_seq 的事件、并按 (session, seq) 去重,max_events 为真值(非 0/None)时取满即停;dry_run 为假时逐条 cg.add(nid 已在 cg.index["nodes"] 中则跳过,cg.add 抛异常则 denied+=1、记 last_error 并继续),denied 与 last_error 同时成立时补 last_error/hint 与 denied_events 明细,mine_fix_pairs 为真且 new_events 非空且非 dry_run 时以 as_proposals=True 调 mine_fix_pairs(自动产物走审核队列)并结果附 fix_pairs,new_events 非空且非 dry_run 时以末事件写回水位(count 累加 written),最后返回 result。
503
+ def ingest(self, source, mine_fix_pairs: bool = False, max_events: int = None,
504
+ dry_run: bool = False):
505
+ """摄取一个源的新事件。返回统计。
506
+
507
+ 去重键:(session, seq) —— 同一事件不重复入库。
508
+ 增量:只处理 (t, seq) 大于 watermark 的事件。
509
+ mine_fix_pairs 默认 False(先落账后挖矿——§5.5 系统纪律化,
510
+ zcode 外评 break#2:默认 True 曾使 file/jsonl/mdcg_ingest 三入口
511
+ 绕过审核队列直写 knowledge 层);显式开启时产物走 propose 队列。
512
+ """
513
+ key = source.key()
514
+ wm = self.watermark(key)
515
+ # 水位单位归一:**存量水位是毫秒**(旧版 `_ts` 产出),不归一会让
516
+ # `ev["t"] < last_t` 恒真 → 升级后新事件被整片误判为「已处理」而跳过。
517
+ last_t, last_seq = trust.epoch_seconds(float(wm.get("t") or 0)) or 0.0, \
518
+ wm.get("seq")
519
+ new_events, seen = [], set()
520
+ denied_events = [] # 被拒事件明细(session/seq/原因)——可观测可重放
521
+ for ev in source.events():
522
+ if ev.get("t", 0) < last_t:
523
+ continue
524
+ if ev.get("t", 0) == last_t and last_seq is not None \
525
+ and isinstance(ev.get("seq"), int) and ev["seq"] <= last_seq:
526
+ continue
527
+ dedup = (ev.get("session"), ev.get("seq"))
528
+ if dedup in seen:
529
+ continue
530
+ seen.add(dedup)
531
+ new_events.append(ev)
532
+ if max_events and len(new_events) >= max_events:
533
+ break
534
+
535
+ written, ids, denied = 0, [], 0
536
+ if not dry_run:
537
+ for ev in new_events:
538
+ nid = "src_%s_%s" % (_sig(key)[:6], _sig(
539
+ f"{ev.get('session')}:{ev.get('seq')}")[:10])
540
+ if nid in self.cg.index["nodes"]:
541
+ continue # 幂等
542
+ # #466(Lead 裁定):基底取自**来源声明**(事件级 basis 优先,否则源级
543
+ # default_basis)。取值主体是形参 `source`(= Source 实例)、**不是 `self`**:
544
+ # 本方法是 `Ingestor.ingest(self, source, …)`,Ingestor 只是**搬运者**、不该
545
+ # 自己声明基底。写 `self.default_basis` 会让 Ingestor 无此属性 ⇒ AttributeError,
546
+ # 且被下方 except Exception 吞成 denied 计数 ⇒ **外部内容静默不落库**
547
+ # (2026-10-15 回归事故根因)。
548
+ # ★两条通道**必须同源**:CCG 正文行的「验证方式:」与 fm 的
549
+ # verification_basis 共用**同一个 basis 变量**——正文声明可验、枚举说未验
550
+ # 会让同一节点自相矛盾(守卫 H01 钉这一点)。
551
+ basis = ev.get("basis") or source.default_basis
552
+ body = ("# 功能名:会话事件\n"
553
+ f"# 生效条件:检索「{str(ev.get('text'))[:20]}」\n"
554
+ "# 子功能:记录会话事件\n"
555
+ f"# 执行:{str(ev.get('text'))[:80]}\n"
556
+ f"# 验证方式:{basis}(会话原始记录)\n"
557
+ "# 不适用条件:其它会话\n\n"
558
+ f"{ev.get('text')}\n")
559
+
560
+ try:
561
+ self.cg.add(nid, body, layer=self.layer, role=ev.get("role"),
562
+ tags=["session", key],
563
+ # 事件级密级覆盖(M6 §5.5):error 事件建议 private
564
+ # (失败细节可能含路径/配置),缺省回落源级默认
565
+ sensitivity=ev.get("sensitivity") or self.sensitivity,
566
+ # #466:不再硬编码 "data"(外部内容不得自带权威基底)。
567
+ verification_basis=basis,
568
+ condition_space={"observation_position": key,
569
+ "observation_tool": "会话流",
570
+ "time_window": [ev.get("t") or 0,
571
+ ev.get("t") or 0]})
572
+ except Exception as exc: # noqa: BLE001 —— 权限/层错误不中断整批
573
+ denied += 1
574
+ # 诚实化(M6):denied 事件虽被水位跳过,但明细必须可见——
575
+ # 静默丢失会吞掉误差归因闭环的原料(error 事件恰是原料)
576
+ denied_events.append({
577
+ "session": ev.get("session"), "seq": ev.get("seq"),
578
+ "error": str(exc)[:150]})
579
+ self.last_error = f"{type(exc).__name__}: {exc}"
580
+ continue
581
+ ids.append(nid)
582
+ written += 1
583
+
584
+ result = {"source": key, "new_events": len(new_events), "written": written,
585
+ "denied": denied, "ids": ids, "dry_run": dry_run,
586
+ "sensitivity": self.sensitivity}
587
+ if denied_events:
588
+ result["denied_events"] = denied_events
589
+ if denied and getattr(self, "last_error", None):
590
+ result["last_error"] = self.last_error
591
+ result["hint"] = ("会话内容默认 sensitivity=private;"
592
+ "调用方需 MDCG_CLEARANCE=private 才能写入")
593
+ # 自动 fix-pair 挖掘(对标 deja-vu:错误→修复)——as_proposals=True:
594
+ # 自动管线产物走审核队列,绝不直写 knowledge 层(§5.5 系统纪律)
595
+ if mine_fix_pairs and new_events and not dry_run:
596
+ result["fix_pairs"] = self.cg.mine_fix_pairs(
597
+ [{"role": e.get("role"), "text": e.get("text")}
598
+ for e in new_events], as_proposals=True)
599
+
600
+ if new_events and not dry_run:
601
+ d = self._load()
602
+ last = new_events[-1]
603
+ d["sources"][key] = {
604
+ "t": last.get("t") or last_t,
605
+ "seq": last.get("seq"),
606
+ "count": (wm.get("count") or 0) + written,
607
+ "updated_at": time.time(),
608
+ "path": getattr(source, "path", None),
609
+ }
610
+ self._save(d)
611
+ return result
612
+
613
+
614
+ # 生效条件:text 为 None 或假值时按 "" 参与 sha1;n 默认 12,返回 hexdigest 前 n 位(n=0 得空串)。
615
+ def _sig(text: str, n: int = 12) -> str:
616
+ import hashlib
617
+ return hashlib.sha1((text or "").encode("utf-8")).hexdigest()[:n]
618
+
619
+
620
+ # --------------------------------------------------------------------------
621
+ # 摄取分派(P0 · ingest op):按扩展名选摄取方式,单一入口吃多种文件
622
+ # --------------------------------------------------------------------------
623
+ #
624
+ # 三条摄取链:
625
+ # session —— 会话流(.jsonl):走 Ingestor(watermark + 去重 + fix-pair)
626
+ # doc —— 文档(.md/.txt/...):走 docindex.extract + refindex.add_items
627
+ # code —— 代码(.py/.ts/...):走 codeindex.extract + refindex.add_items
628
+ #
629
+ # 设计要点:
630
+ # - 注册表 INGEST_REGISTRY 是唯一真源:新增后缀只改这里。
631
+ # - dir 动作分链处理:目录里 doc / code / jsonl 混放时各链互不干扰。
632
+ # - 幂等:沿用 refindex.Ledger(size+mtime 水位)与 Ingestor watermark。
633
+ # - 预演:dry_run=True 只统计、不写入(对应计划「可预演」要求)。
634
+
635
+ INGEST_ACTIONS = ("file", "dir", "jsonl", "stat", "hive")
636
+
637
+ INGEST_REGISTRY = {
638
+ # 会话流
639
+ ".jsonl": "session", ".ndjson": "session",
640
+ # 文档
641
+ ".md": "doc", ".markdown": "doc", ".txt": "doc", ".rst": "doc",
642
+ ".html": "doc", ".htm": "doc",
643
+ # 代码
644
+ ".py": "code", ".js": "code", ".mjs": "code", ".ts": "code",
645
+ ".tsx": "code", ".jsx": "code", ".go": "code", ".rs": "code",
646
+ ".java": "code", ".kt": "code", ".swift": "code", ".rb": "code",
647
+ ".php": "code", ".cs": "code", ".cpp": "code", ".cc": "code",
648
+ ".c": "code", ".h": "code", ".hpp": "code",
649
+ }
650
+
651
+
652
+ # 生效条件:path 为 None 或空串时 splitext 得 "" 且未登记 → 返回 None;扩展名(小写)存在于 INGEST_REGISTRY 时返回其 kind。
653
+ def dispatch_of(path: str):
654
+ """按扩展名返回摄取方式(session / doc / code);未登记返回 None。"""
655
+ return INGEST_REGISTRY.get(os.path.splitext(path or "")[1].lower())
656
+
657
+
658
+ # --------------------------------------------------------------------------
659
+ # 库内路径闸(issue #425):库内节点文件 / 回收站 / 保护历史**不是外部文档**
660
+ # --------------------------------------------------------------------------
661
+ # 病象:`ingest dir path=<库根>` 或 `ingest file path=trash/<id>.md` 时,遍历把
662
+ # 各层节点文件、回收站、保护历史当外部 md 重新切成 doc_* 节点——
663
+ # · restricted 节点正文被抄进 internal 节点(无令牌访客可检索);
664
+ # · 已 forget+accept 删除的节点从 trash/ 复活成活节点。
665
+ # 根因(条件层):摄取链只判「path 非空」,不判「它是不是记忆库自己」;而 doc 节点
666
+ # 的密级判据(`docindex.sensitivity_for`)只看路径段,源节点 frontmatter 里的
667
+ # `sensitivity: restricted` 根本不参与 ⇒ 重索引必然降级。库内节点文件的密级真源
668
+ # 是节点 frontmatter(`cg.get` 面),**不是**路径——本链不读它,故这里 **fail-closed**:
669
+ # 库根本身、各层目录、trash/、_protected_history/、hippocampus/ 一律拒绝/剪枝,
670
+ # 不猜、不降级。库根下**普通子目录**仍可作为外部资料根(保留既有用法)。
671
+ # 姊妹判据:export 侧已有 `security.check_export_target`(#211),本闸是 ingest 侧对应的一道。
672
+
673
+ #: 库根下不得被 ingest 当外部文档的子树(相对库根首段)。
674
+ #: 层目录从 `mdcg.LAYERS` 取(唯一真源),不在此复制一份。
675
+ _LIBRARY_EXTRA_RESERVED = ("trash", "_protected_history", "hippocampus",
676
+ "_index_log", "export")
677
+
678
+
679
+ def library_reserved_dirs():
680
+ """库根下保留子树的首段名(层目录 + 回收站/保护历史/海马体等)。
681
+
682
+ 层目录**只从 `mdcg.LAYERS` 取**(唯一真源,不在此复制一份);取不到即
683
+ fail-closed 抛错——宁可本次摄取失败,也不静默放宽成「层目录可被当外部文档」。
684
+ """
685
+ from .mdcg import LAYERS as _LAYERS
686
+ return tuple(_LAYERS) + _LIBRARY_EXTRA_RESERVED
687
+
688
+
689
+ def library_rel(cg, path):
690
+ """path 相对记忆库根的路径(正斜杠);不在库内返回 None,等于库根返回 ""。
691
+
692
+ realpath + normcase 后比较:符号链接/大小写不改变「是否库内」的结论。
693
+ """
694
+ root = getattr(cg, "root", None)
695
+ if not root or not path or not str(path).strip():
696
+ return None
697
+ try:
698
+ lib = os.path.normcase(os.path.realpath(root))
699
+ real = os.path.normcase(os.path.realpath(os.path.expanduser(str(path))))
700
+ except (OSError, ValueError):
701
+ return None
702
+ if real == lib:
703
+ return ""
704
+ if real.startswith(lib.rstrip(os.sep) + os.sep):
705
+ return os.path.relpath(real, lib).replace("\\", "/")
706
+ return None
707
+
708
+
709
+ def library_reserved(cg, rel):
710
+ """相对库根的路径是否落在保留子树(含库根本身)——这些不是外部文档。"""
711
+ if rel is None:
712
+ return False
713
+ if rel == "":
714
+ return True
715
+ return rel.replace("\\", "/").split("/")[0] in library_reserved_dirs()
716
+
717
+
718
+ def library_skip_rel(cg, root):
719
+ """root 是库根**祖先**时,返回库根相对 root 的路径(遍历时整枝剪掉);否则 None。
720
+
721
+ issue #425 C7:`MDCG_INGEST_ROOT` 设成库根上级目录时,`dir` 空 path 会从上级
722
+ 开始 walk,把整个记忆库扫进来。剪枝比「逐文件判」便宜且不漏(层目录、trash、
723
+ _protected_history 都在库根之下)。
724
+ """
725
+ try:
726
+ lib = os.path.normcase(os.path.realpath(cg.root))
727
+ r = os.path.normcase(os.path.realpath(root))
728
+ except (OSError, ValueError):
729
+ return None
730
+ if lib == r:
731
+ return None
732
+ rel = os.path.relpath(lib, r).replace("\\", "/")
733
+ if rel == ".." or rel.startswith("../"):
734
+ return None
735
+ return rel
736
+
737
+
738
+ # 生效条件:cg 无 root 或 path 为空时返回 None(无路径参数不判);否则按 realpath 判——path 等于库根或落在库根保留子树(层目录/trash/_protected_history/hippocampus 等)内时返回 ok=False 的拒绝回执(error 点名 issue #425 与原因);库外或库根下普通子目录返回 None(放行)。
739
+ def library_ingest_guard(cg, path, what="ingest"):
740
+ """库内路径闸的**单点**判定(issue #425):返回拒绝回执或 None。"""
741
+ rel = library_rel(cg, path)
742
+ if not library_reserved(cg, rel):
743
+ return None
744
+ where = "记忆库根" if rel == "" else f"记忆库保留子树「{rel.split('/')[0]}」"
745
+ return {"ok": False, "path": path, "error": (
746
+ f"{what} 拒绝:{path} 落在{where}内——库内节点文件/回收站/保护历史"
747
+ f"**不是外部文档**(issue #425:重新索引会把 restricted 正文抄成 internal、"
748
+ f"把已删除节点复活)。外部资料请放在**库根之外**的根再摄取"
749
+ f"(设 MDCG_INGEST_ROOT);库内节点的维护/重建不走 ingest 面。")}
750
+
751
+
752
+ # --------------------------------------------------------------------------
753
+ # 来源密级继承(issue #452):判据面必须同时看**绝对路径**与相对路径的段
754
+ # --------------------------------------------------------------------------
755
+ # 病象:`docindex.index_dir` 产出的 item["path"] 是**相对被摄取根**的;而
756
+ # `refindex.add_items` 用 `docindex.sensitivity_for(it["path"])` 判密级。
757
+ # 当**摄取根本身就是敏感目录**(`ingest(path=<…>/private)`)时,`private` 这一段
758
+ # 正是 root 自身,被 relpath 抹掉 ⇒ 提示永远命中不了,敏感文档以 internal 落库,
759
+ # internal 档身份读得回其正文摘要。同一内容换成父目录摄取则正确判 private 并
760
+ # fail-closed——同一份文档、同一个写者,只因 path 参数不同而一紧一松。
761
+ #
762
+ # 根因(条件层):判据的**输入面**取错了路径(相对路径丢了根自身的段),
763
+ # 不是 sensitivity_for 的实现有问题。故修法:判据仍只有 `docindex.sensitivity_for`
764
+ # 一处,但喂给它的是**源文件的绝对路径**(绝对路径的段集合 ⊇ 相对路径的段集合,
765
+ # 只会更严、不会更松)。item["path"] 保持相对 root 不变——节点 id / doc_ref 的
766
+ # 稳定标识与回读口径不受影响(换 id 会动存量幂等,不在本条范围)。
767
+ def source_sensitivity(path, explicit=None):
768
+ """源文件密级判据的**唯一取值点**(issue #452):路径面取绝对路径。
769
+
770
+ 判据本体仍只有 `docindex.sensitivity_for` 一处;本函数只补两件事:
771
+ ① **路径面**喂绝对路径(绝对路径的段集合 ⊇ 相对路径,只会更严);
772
+ ② **来源声明面**(issue #425 修复建议 3):源文件 frontmatter 里若带
773
+ `sensitivity`,取「路径判据」与「来源声明」中**更严**的一档——
774
+ 否则把一份 restricted 节点文件复制到库外再摄取,密级就凭空降成
775
+ internal(「删掉的比留下的更泄露」同族)。缺键/值域非法时不猜,
776
+ 按路径判据(既有语义);库内节点文件由 `library_ingest_guard` 直接拒,
777
+ 走不到这里。
778
+ """
779
+ from . import docindex, nodefile
780
+ from .security import SENSITIVITY_ORDER
781
+ s, _basis = docindex.sensitivity_for(path, explicit)
782
+ cands = [s]
783
+ try:
784
+ with open(path, encoding="utf-8", errors="replace") as f:
785
+ fm, _content = nodefile.loads(f.read())
786
+ except (OSError, ValueError):
787
+ fm = {}
788
+ fm_s = fm.get("sensitivity") if isinstance(fm, dict) else None
789
+ if isinstance(fm_s, str) and fm_s in SENSITIVITY_ORDER:
790
+ cands.append(fm_s)
791
+ return max(cands, key=SENSITIVITY_ORDER.index)
792
+
793
+
794
+ def add_items_with_source_sensitivity(cg, items, *, kind, root, layer=None,
795
+ sensitivity=None, conflicts=None):
796
+ """按**来源文件**分组调 `refindex.add_items`,每组带该文件继承来的密级。
797
+
798
+ `refindex.add_items` 只接受一个批次级 `sensitivity`,而密级提示可能落在
799
+ **被摄取根自身**的名字上(#452)。故按 item["path"](相对 root 的源文件路径)
800
+ 分组,每组以 `source_sensitivity(join(root, rel), sensitivity)` 作为该组的
801
+ 密级——判据仍是 `docindex.sensitivity_for` 单点,只是输入面补上了绝对路径。
802
+ doc 之外(code_ref)不涉密级继承,保持原批次一次调用。
803
+ 返回 `(ids, sens_counts)`,与 `refindex.add_items` 同形。
804
+ """
805
+ from . import refindex
806
+ if kind != "doc_ref" or not items:
807
+ return refindex.add_items(cg, items, kind=kind, root=root, layer=layer,
808
+ sensitivity=sensitivity, conflicts=conflicts)
809
+ groups = {}
810
+ for it in items:
811
+ groups.setdefault(it.get("path") or "", []).append(it)
812
+ if len(groups) <= 1:
813
+ rel = next(iter(groups), "")
814
+ s = source_sensitivity(os.path.join(root, rel), sensitivity) if rel else sensitivity
815
+ return refindex.add_items(cg, items, kind=kind, root=root, layer=layer,
816
+ sensitivity=s, conflicts=conflicts)
817
+ ids, sens = [], {}
818
+ for rel, grp in groups.items():
819
+ s = source_sensitivity(os.path.join(root, rel), sensitivity)
820
+ _ids, _sens = refindex.add_items(cg, grp, kind=kind, root=root, layer=layer,
821
+ sensitivity=s, conflicts=conflicts)
822
+ ids.extend(_ids)
823
+ for k, v in (_sens or {}).items():
824
+ sens[k] = sens.get(k, 0) + v
825
+ return ids, sens
826
+
827
+
828
+ # 生效条件:required 形参 cg 总被存入;可选形参 sensitivity 为假值(None/空串)时内部 Ingestor 的 sensitivity 回落模块级常量 SESSION_SENSITIVITY,为真值时用传入的 sensitivity。
829
+ class FileDispatcher:
830
+ """单一入口吃多种文件:按扩展名分派到会话流 / 文档 / 代码三条摄取链。"""
831
+
832
+ # 生效条件:传入 cg 即成立,sensitivity 为假值(None/空串)时所用 Ingestor 回落 SESSION_SENSITIVITY,否则用传入值。
833
+ def __init__(self, cg, sensitivity=None):
834
+ self.cg = cg
835
+ # 会话链默认 sensitivity=private;调用方可显式覆盖(测试/受限环境)
836
+ self.ingestor = Ingestor(cg, sensitivity=sensitivity or SESSION_SENSITIVITY)
837
+
838
+ # ---- stat:看水位与支持面 ----
839
+
840
+ # 生效条件:from . import refindex 与 refindex.Ledger(self.cg.root).summary() 均不抛异常时返回该 summary 结果,抛任何异常时返回 {}。
841
+ def _ledger_stat(self):
842
+ # `Ledger` 只有 `summary()`、**没有** `stat()`:此前写 `.stat()` 撞
843
+ # AttributeError 被下面的 except 吞成 `{}`,于是 `ingest stat` 的水位面
844
+ # 永远是空且无人知晓(唯一实现面是 `refindex.Ledger.summary`)。
845
+ try:
846
+ from . import refindex
847
+ return refindex.Ledger(self.cg.root).summary()
848
+ except Exception: # noqa: BLE001
849
+ return {}
850
+
851
+ # 生效条件:调用即返回由 INGEST_REGISTRY 按 kind 分组并排序后的 extensions、list(INGEST_ACTIONS)、self.ingestor.watermarks()、self._ledger_stat() 与固定 note。
852
+ def stat(self):
853
+ kinds = {}
854
+ for ext, kind in INGEST_REGISTRY.items():
855
+ kinds.setdefault(kind, []).append(ext)
856
+ return {"ok": True, "kind": "stat", "actions": list(INGEST_ACTIONS),
857
+ "extensions": {k: sorted(v) for k, v in sorted(kinds.items())},
858
+ "watermarks": self.ingestor.watermarks(),
859
+ "ledger": self._ledger_stat(),
860
+ "note": ("注册表是唯一真源:新增后缀只改 INGEST_REGISTRY。"
861
+ "watermarks=会话流水位;ledger=文档/代码文件水位。")}
862
+
863
+ # ---- 单文件 ----
864
+
865
+ # 生效条件:dispatch_of(path) 为 None 时返回 ok=False 的「不支持的后缀」结果;path 不是文件时返回 ok=False 的「文件不存在」结果;kind=="session" 时转 ingest_jsonl(path, dry_run=dry_run)(layer/sensitivity 不参与);其它 kind 转 _ingest_doc_or_code(path, kind, layer=layer, sensitivity=sensitivity, dry_run=dry_run)。
866
+ def ingest_file(self, path, layer=None, sensitivity=None, dry_run=False):
867
+ kind = dispatch_of(path)
868
+ if kind is None:
869
+ ext = os.path.splitext(path or "")[1] or "(无后缀)"
870
+ return {"ok": False, "path": path, "error": f"不支持的后缀:{ext}",
871
+ "supported": sorted(INGEST_REGISTRY)}
872
+ if not os.path.isfile(path):
873
+ return {"ok": False, "path": path, "error": "文件不存在"}
874
+ # issue #425:库内节点文件 / trash/<id>.md / _protected_history 快照
875
+ # **不是外部文档**——单文件链同样过闸(否则一条 file 摄取即可把
876
+ # restricted 正文抄成 internal,或把已删除节点复活)。
877
+ _g = library_ingest_guard(self.cg, path, "ingest file")
878
+ if _g is not None:
879
+ return _g
880
+ if kind == "session":
881
+ return self.ingest_jsonl(path, dry_run=dry_run)
882
+ return self._ingest_doc_or_code(path, kind, layer=layer,
883
+ sensitivity=sensitivity, dry_run=dry_run)
884
+
885
+ # 生效条件:kind=="code" 用 codeindex 否则用 docindex;mod.extract 抛 ValueError 时返回 ok=False 的「抽取失败」;dry_run 为真时只返回 items 计数与前 20 个 node_id 不写盘;否则经 refindex.add_items(kind 为 code_ref/doc_ref)写入并返回 indexed、ids[:20] 与 sensitivity。
886
+ def _ingest_doc_or_code(self, path, kind, layer=None, sensitivity=None,
887
+ dry_run=False):
888
+ from . import codeindex, docindex, refindex
889
+ mod = codeindex if kind == "code" else docindex
890
+ rel = os.path.basename(path)
891
+ with open(path, encoding="utf-8", errors="replace") as f:
892
+ src = f.read()
893
+ try:
894
+ items = mod.extract(src, path=rel,
895
+ suffix=os.path.splitext(path)[1].lower())
896
+ except ValueError as exc:
897
+ return {"ok": False, "path": path, "error": f"抽取失败:{exc}"}
898
+ items = items or []
899
+ if dry_run:
900
+ return {"ok": True, "dry_run": True, "kind": kind, "path": path,
901
+ "items": len(items),
902
+ "ids": [mod.node_id(i) for i in items[:20]],
903
+ "note": "预演:只抽取计数,未写盘"}
904
+ ref_kind = "code_ref" if kind == "code" else "doc_ref"
905
+ _cf = [] # #352-E:同 id 已被另一个源根占用 ⇒ 不覆写
906
+ ids, sens = add_items_with_source_sensitivity(
907
+ self.cg, items, kind=ref_kind,
908
+ root=os.path.dirname(path) or ".", layer=layer,
909
+ sensitivity=sensitivity, conflicts=_cf)
910
+ _out = {"ok": True, "kind": kind, "path": path, "items": len(items),
911
+ "indexed": len(ids), "ids": ids[:20], "sensitivity": sens}
912
+ if _cf: # 冲突不静默(单文件链同样点名)
913
+ _out["conflict_count"] = len(_cf)
914
+ _out["conflicts"] = _cf[:20]
915
+ return _out
916
+
917
+ # ---- 目录 ----
918
+
919
+ # 生效条件:调用即 os.walk(root) 统计每个文件名经 dispatch_of 得到的 kind(无匹配记 "unsupported")并返回 dry_run 预演计数结果。
920
+ def _dry_dir(self, root):
921
+ counts = {}
922
+ _lib_rel = library_skip_rel(self.cg, root) # issue #425 C7:库根整枝剪掉
923
+ for dp, dns, fns in os.walk(root):
924
+ rel_dir = os.path.relpath(dp, root).replace("\\", "/")
925
+ if rel_dir == ".":
926
+ rel_dir = ""
927
+ if _lib_rel is not None:
928
+ dns[:] = [d for d in dns
929
+ if ("%s/%s" % (rel_dir, d) if rel_dir else d) != _lib_rel
930
+ and not ("%s/%s" % (rel_dir, d) if rel_dir else d)
931
+ .startswith(_lib_rel + "/")]
932
+ for name in fns:
933
+ k = dispatch_of(name) or "unsupported"
934
+ counts[k] = counts.get(k, 0) + 1
935
+ return {"ok": True, "dry_run": True, "kind": "dir", "root": root,
936
+ "counts": counts,
937
+ "note": "预演:仅统计各链文件数,未做任何写入"}
938
+
939
+ # 生效条件:root 非目录时返回 ok=False 的「目录不存在」;dry_run 为真时返回 _dry_dir(root);否则对 doc_ref/code_ref 两链各以 patterns/max_files/max_items/incremental/ledger 调 refindex.index_dir(commit=False) 与 add_items 后 ledger.save(),并把 root 下 **/*.jsonl 前 max_files 个逐个 ingest_jsonl 后返回 out。
940
+ def ingest_dir(self, root, layer=None, sensitivity=None, patterns=None,
941
+ max_files=500, max_items=2000, incremental=False,
942
+ dry_run=False):
943
+ from . import refindex
944
+ if not os.path.isdir(root):
945
+ return {"ok": False, "error": f"目录不存在:{root}"}
946
+ # issue #425:库根本身 / 层目录 / trash / _protected_history / hippocampus
947
+ # **不是外部文档**——直接拒绝(fail-closed),不猜扫描根。
948
+ _g = library_ingest_guard(self.cg, root, "ingest dir")
949
+ if _g is not None:
950
+ return _g
951
+ if dry_run:
952
+ return self._dry_dir(root)
953
+ # issue #425 C7:root 是库根**祖先**时整枝剪掉库根(与 SKIP_DIRS 同处生效)。
954
+ _lib_rel = library_skip_rel(self.cg, root)
955
+ _skip = [_lib_rel] if _lib_rel else None
956
+ ledger = refindex.Ledger(self.cg.root)
957
+ out = {"ok": True, "kind": "dir", "root": root, "chains": {}}
958
+ if _lib_rel:
959
+ out["library_pruned"] = [_lib_rel]
960
+ for ref_kind, key in (("doc_ref", "doc"), ("code_ref", "code")):
961
+ items, errors, stats = refindex.index_dir(
962
+ root, kind=ref_kind, patterns=patterns, max_files=max_files,
963
+ max_items=max_items, incremental=incremental, ledger=ledger,
964
+ skip_dirs=_skip, commit=False)
965
+ _cf = [] # #352-E:同 id 已被另一个源根占用 ⇒ 不覆写
966
+ # issue #452:按来源文件继承密级(判据面补上绝对路径的段)。
967
+ ids, sens = add_items_with_source_sensitivity(
968
+ self.cg, items, kind=ref_kind,
969
+ root=root, layer=layer,
970
+ sensitivity=sensitivity,
971
+ conflicts=_cf)
972
+ for _c in _cf: # 冲突源文件的水位一并丢掉(否则下次增量跳过)
973
+ ledger.drop(_c.get("src")
974
+ or os.path.join(root, _c.get("path") or ""))
975
+ # 写序(同 op=index_code/index_doc):节点先落盘,水位随后——
976
+ # 中途被杀只会留下「节点新 + 水位旧」,下次增量重切,不会静默漏漂移。
977
+ ledger.save()
978
+ out["chains"][key] = {
979
+ "indexed": len(ids), "errors": len(errors),
980
+ "files": stats.get("files"), "truncated": stats.get("truncated"),
981
+ "empty_scan": stats.get("empty_scan"),
982
+ "skipped_unchanged": stats.get("skipped_unchanged", 0),
983
+ "skipped_suffixes": stats.get("skipped_suffixes", []),
984
+ "sensitivity": sens}
985
+ if _cf: # #352-E:冲突不静默
986
+ out["chains"][key]["conflict_count"] = len(_cf)
987
+ out["chains"][key]["conflicts"] = _cf[:20]
988
+ # 会话流(.jsonl)逐个增量摄取——库内台账/回收站下的 jsonl 同样不是
989
+ # 外部会话(issue #425:`hippocampus/*.jsonl` 等库内账本不得被当会话源)。
990
+ jsons = [p for p in sorted(glob.glob(os.path.join(root, "**", "*.jsonl"),
991
+ recursive=True))[:max_files]
992
+ if not library_reserved(self.cg, library_rel(self.cg, p))]
993
+ ses = []
994
+ for p in jsons:
995
+ r = self.ingest_jsonl(p)
996
+ ses.append({"path": p, "written": r.get("written", 0),
997
+ "new_events": r.get("new_events", 0)})
998
+ out["chains"]["session"] = {"files": len(jsons), "results": ses}
999
+ return out
1000
+
1001
+ # ---- 会话流 ----
1002
+
1003
+ @staticmethod
1004
+ # 生效条件:required 形参 path 能被 open(...,encoding="utf-8",errors="replace") 打开时读前 50000 字符,head 含 '"user/message"'、'"assistant/message"'、'"tool/call"' 任一标记则返回 DSHSessionSource(path),否则返回 JsonlSource(path);打开抛 OSError 时直接返回 JsonlSource(path)。
1005
+ def _auto_source(path):
1006
+ """通用 JSONL vs DSH 会话:按内容探测,避免调用方选错源类型。"""
1007
+ try:
1008
+ with open(path, encoding="utf-8", errors="replace") as f:
1009
+ head = f.read(50000)
1010
+ except OSError:
1011
+ return JsonlSource(path)
1012
+ for marker in ('"user/message"', '"assistant/message"', '"tool/call"'):
1013
+ if marker in head:
1014
+ return DSHSessionSource(path)
1015
+ return JsonlSource(path)
1016
+
1017
+ # 生效条件:path 不是文件时返回 ok=False 的「文件不存在」;path 落在记忆库保留子树(层目录/trash/_protected_history/hippocampus 等)内时返回 ok=False 的库内路径拒绝(issue #425);否则经 _auto_source(path) 选源后调 self.ingestor.ingest(src, dry_run=dry_run, max_events=max_events),并补上 ok=True/kind/path/source_class 后返回。
1018
+ def ingest_jsonl(self, path, dry_run=False, max_events=None):
1019
+ if not os.path.isfile(path):
1020
+ return {"ok": False, "error": f"文件不存在:{path}"}
1021
+ _g = library_ingest_guard(self.cg, path, "ingest jsonl")
1022
+ if _g is not None:
1023
+ return _g
1024
+ src = self._auto_source(path)
1025
+ res = self.ingestor.ingest(src, dry_run=dry_run, max_events=max_events)
1026
+ res.update({"ok": True, "kind": "session", "path": path,
1027
+ "source_class": type(src).__name__})
1028
+ return res
1029
+
1030
+
1031
+ # 生效条件:cg 必填;action 为 None/空串时 (action or "stat") 归为 stat;file/jsonl 缺 path 返回 ok=False;dir 的 max_files/max_items 走 int(x or 500)/int(x or 2000),传 0 也变 500/2000;未知 action 抛 ValueError。
1032
+ def run(cg, action: str = "stat", **kw):
1033
+ """ingest op 唯一入口。"""
1034
+ act = (action or "stat").strip().lower()
1035
+ d = FileDispatcher(cg, sensitivity=kw.get("sensitivity"))
1036
+ if act == "stat":
1037
+ return d.stat()
1038
+ if act == "file":
1039
+ p = kw.get("path")
1040
+ if not p:
1041
+ return {"ok": False, "error": "file 动作需要 path"}
1042
+ return d.ingest_file(p, layer=kw.get("layer"),
1043
+ sensitivity=kw.get("sensitivity"),
1044
+ dry_run=bool(kw.get("dry_run")))
1045
+ if act == "dir":
1046
+ # issue #351(2026-10-16 DSH 端):空 path **不得**回落记忆库根。库根下是
1047
+ # 各层节点文件、trash/、_protected_history/,把它们当外部文档切成 doc_ref
1048
+ # 节点会 (a) 把 restricted 正文抄进 internal 节点(无令牌访客可检索)、
1049
+ # (b) 把已 forget 并 accept 删除的节点复活。缺省根改由调用方显式给 path
1050
+ # 或设 env MDCG_INGEST_ROOT(分派层已在白名单检查之前解析)——这里
1051
+ # fail-closed,不猜扫描根。
1052
+ p = kw.get("path")
1053
+ if not p:
1054
+ return {"ok": False, "error": (
1055
+ "dir 动作需要 path(或设 env MDCG_INGEST_ROOT)——不猜测扫描根"
1056
+ "(issue #351:缺省回落记忆库根会把 restricted/已删除节点"
1057
+ "重新索引成 internal 并可被访客检索)")}
1058
+ return d.ingest_dir(p, layer=kw.get("layer"),
1059
+ sensitivity=kw.get("sensitivity"),
1060
+ patterns=kw.get("patterns"),
1061
+ max_files=int(kw.get("max_files") or 500),
1062
+ max_items=int(kw.get("max_items") or 2000),
1063
+ incremental=bool(kw.get("incremental")),
1064
+ dry_run=bool(kw.get("dry_run")))
1065
+ if act == "jsonl":
1066
+ p = kw.get("path")
1067
+ if not p:
1068
+ return {"ok": False, "error": "jsonl 动作需要 path"}
1069
+ return d.ingest_jsonl(p, dry_run=bool(kw.get("dry_run")),
1070
+ max_events=kw.get("max_events"))
1071
+ if act == "hive":
1072
+ # M6:蜂巢任务事件源(hive/jobs → contextual)。root 解析链:显式 path >
1073
+ # env MDCG_HIVE_JOBS > 报错指引(fail-closed,不猜仓库布局)。
1074
+ # §5.5 硬纪律:mine_fix_pairs=False(先落账后挖矿——自动挖掘产物直写
1075
+ # knowledge 层违反双轨制,实测 0→2 污染);默认密级 internal,
1076
+ # error 事件由源层 per-event 覆写 private(失败细节可能含路径/配置)。
1077
+ root = kw.get("path") or os.environ.get("MDCG_HIVE_JOBS")
1078
+ if not root:
1079
+ return {"ok": False, "error": (
1080
+ "hive 动作需要 path(hive/jobs 目录),或设 env MDCG_HIVE_JOBS——"
1081
+ "不猜测仓库布局(fail-closed)")}
1082
+ src = HiveJobsSource(root, error_sensitivity=kw.get("error_sensitivity")
1083
+ if kw.get("error_sensitivity") is not None else "private")
1084
+ rep = Ingestor(cg, layer=kw.get("layer") or "contextual",
1085
+ sensitivity=kw.get("sensitivity") or "internal"
1086
+ ).ingest(src, mine_fix_pairs=False,
1087
+ max_events=kw.get("max_events"),
1088
+ dry_run=bool(kw.get("dry_run")))
1089
+ rep["source"] = src.key()
1090
+ rep["skipped"] = src.skipped
1091
+ return rep
834
1092
  raise ValueError(f"未知 ingest action:{action!r}(允许 {INGEST_ACTIONS})")