@furongjun1999/dsh-memory 0.8.1 → 0.9.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +662 -659
- package/codebuddy/README.md +1 -1
- package/data/policy.json +38 -27
- package/docs/README.md +18 -2
- package/docs/discipline/harnesses.yaml +1 -1
- package/docs/discipline/templates/spec.txt.tmpl +23 -0
- package/docs//350/247/204/350/214/203/AI/347/274/226/347/250/213/351/200/232/347/224/250/350/247/204/350/214/203.md +179 -0
- package/docs//350/247/204/350/214/203/AI/347/274/226/347/250/213/351/200/232/347/224/250/350/247/204/350/214/203_/347/201/265/346/236/242/351/200/202/351/205/215.md +45 -0
- package/docs//350/247/204/350/214/203/AI/347/274/226/347/250/213/351/200/232/347/224/250/350/247/204/350/214/203_/350/207/252/346/243/200/346/270/205/345/215/225.md +69 -0
- package/dsh/README.md +1 -1
- package/dsh/cordis.yml.example +139 -139
- package/hive/README.md +705 -0
- package/hive/id_charset_blocks.txt +147 -0
- package/lib/bridge.d.ts +3 -0
- package/lib/bridge.js +32 -11
- package/lib/cli.js +14 -17
- package/lib/hooks.d.ts +37 -3
- package/lib/hooks.js +176 -24
- package/lib/index.d.ts +2 -2
- package/lib/index.js +29 -6
- package/lib/init.js +21 -21
- package/lib/lib/datapath.d.ts +56 -0
- package/lib/lib/datapath.js +102 -9
- package/lib/lib/hook_audit.d.ts +0 -1
- package/lib/lib/hook_audit.js +51 -3
- package/lib/lib/mdcg_client.d.ts +4 -1
- package/lib/lib/mdcg_client.js +10 -2
- package/lib/lib/mutual.js +61 -15
- package/lib/lib/prompt_safety.js +28 -2
- package/lib/lib/roleplay_web.js +30 -2
- package/lib/lib/token_store.d.ts +25 -6
- package/lib/lib/token_store.js +224 -33
- package/lib/tools.d.ts +15 -2
- package/lib/tools.js +44 -4
- package/md_cg/_guard_kit.py +121 -0
- package/md_cg/admission.py +718 -718
- package/md_cg/anchor.py +114 -0
- package/md_cg/audit.py +37 -7
- package/md_cg/autonomy.py +433 -288
- package/md_cg/autonomy_modes.py +781 -642
- package/md_cg/bench_beir_bm25.py +154 -0
- package/md_cg/bench_beir_scifact.py +170 -0
- package/md_cg/bench_bm25_blockmax.py +134 -0
- package/md_cg/bench_bm25_incremental.py +246 -0
- package/md_cg/bench_ccg_heading.py +118 -0
- package/md_cg/bench_csre_correctness.py +145 -0
- package/md_cg/bench_excerpt_budget.py +94 -0
- package/md_cg/bench_fuzzy_substrings.py +299 -0
- package/md_cg/bench_memory_rsi.py +358 -0
- package/md_cg/bench_memory_rsi_latency.py +82 -0
- package/md_cg/bench_memory_rsi_public.py +116 -0
- package/md_cg/bench_memory_rsi_tools.py +229 -0
- package/md_cg/bench_rrf_candidates.py +212 -0
- package/md_cg/bench_rrf_public.py +100 -0
- package/md_cg/bench_text_projections.py +221 -0
- package/md_cg/bench_timeanchor.py +357 -0
- package/md_cg/bm25.py +601 -0
- package/md_cg/branches.py +404 -327
- package/md_cg/build_postings.py +74 -73
- package/md_cg/ccgc.py +4 -2
- package/md_cg/chain.py +573 -379
- package/md_cg/coldverify.py +531 -292
- package/md_cg/condaccept.py +263 -0
- package/md_cg/condgate.py +308 -0
- package/md_cg/config_registry.py +329 -0
- package/md_cg/config_registry_bulk.py +482 -0
- package/md_cg/config_registry_guard.py +487 -0
- package/md_cg/config_render.py +354 -0
- package/md_cg/config_validate.py +455 -0
- package/md_cg/conformance.py +747 -726
- package/md_cg/conformance_core.py +753 -0
- package/md_cg/consistency.py +1278 -955
- package/md_cg/consolidate.py +1793 -1633
- package/md_cg/crypto.py +346 -24
- package/md_cg/datapath.py +137 -12
- package/md_cg/eval_common.py +522 -575
- package/md_cg/evidence.py +946 -585
- package/md_cg/evidence_delivery.py +101 -0
- package/md_cg/experience_consolidation.py +317 -0
- package/md_cg/export.py +329 -222
- package/md_cg/forgetting.py +1050 -922
- package/md_cg/freshness.py +601 -548
- package/md_cg/fsutil.py +1120 -824
- package/md_cg/goal_gen.py +458 -0
- package/md_cg/hotcache.py +107 -9
- package/md_cg/hyperedge.py +262 -251
- package/md_cg/identity.py +573 -391
- package/md_cg/imgskill.py +1712 -0
- package/md_cg/insight.py +540 -521
- package/md_cg/interface_schema.py +146 -0
- package/md_cg/interop.py +20 -0
- package/md_cg/judgment_manifest.py +24 -1
- package/md_cg/lifecycle.py +426 -345
- package/md_cg/links.py +27 -6
- package/md_cg/logref.py +332 -327
- package/md_cg/mcp_server.py +5227 -4449
- package/md_cg/mdcg.py +1301 -203
- package/md_cg/mdcos.py +7108 -5440
- package/md_cg/memory_rsi.py +308 -0
- package/md_cg/metacognition.py +726 -591
- package/md_cg/nodefile.py +151 -3
- package/md_cg/op_table.py +1128 -0
- package/md_cg/postings.py +524 -300
- package/md_cg/predict.py +1103 -1100
- package/md_cg/probe_memory_rsi.py +95 -0
- package/md_cg/probe_memory_rsi_mcp.py +66 -0
- package/md_cg/protect.py +654 -494
- package/md_cg/protocol.py +492 -411
- package/md_cg/provenance.py +606 -600
- package/md_cg/qmd.py +477 -0
- package/md_cg/qmd_llm.py +334 -0
- package/md_cg/query_excerpt.py +171 -0
- package/md_cg/queue_health.py +130 -0
- package/md_cg/reach.py +470 -453
- package/md_cg/readcache.py +115 -3
- package/md_cg/reconcile.py +239 -228
- package/md_cg/refindex.py +1191 -1042
- package/md_cg/refine.py +613 -604
- package/md_cg/retrieve.py +128 -0
- package/md_cg/review_cli.py +204 -26
- package/md_cg/revise.py +140 -0
- package/md_cg/rollback.py +657 -420
- package/md_cg/rollback_cli.py +124 -104
- package/md_cg/rotate.py +347 -0
- package/md_cg/routing.py +490 -429
- package/md_cg/rrf_candidates.py +204 -0
- package/md_cg/run_tests.py +19 -1
- package/md_cg/scrub.py +177 -23
- package/md_cg/security.py +269 -17
- package/md_cg/self_state.py +1053 -1029
- package/md_cg/self_surface.py +50 -0
- package/md_cg/semantic/canonical.py +144 -144
- package/md_cg/semantic/export_en_zh_map.py +2 -2
- package/md_cg/semantic/zh_en_atoms.py +141 -139
- package/md_cg/session5.py +106 -0
- package/md_cg/signer.py +76 -2
- package/md_cg/sleep.py +1779 -1583
- package/md_cg/sources.py +1091 -833
- package/md_cg/state_events.py +75 -10
- package/md_cg/state_slots.py +312 -221
- package/md_cg/stg.py +85 -18
- package/md_cg/stgidx.py +537 -281
- package/md_cg/subgraph.py +851 -729
- package/md_cg/sustain.py +458 -19
- package/md_cg/task_memory.py +367 -0
- package/md_cg/test_action_derive.py +210 -203
- package/md_cg/test_audit_heartbeat_20261015.py +417 -0
- package/md_cg/test_audit_protection_20261015.py +483 -0
- package/md_cg/test_audit_rotate_parity.py +280 -0
- package/md_cg/test_audit_state_20261015.py +253 -0
- package/md_cg/test_audit_write_20261015.py +161 -0
- package/md_cg/test_auto_defaults.py +43 -4
- package/md_cg/test_autonomy_emotion.py +328 -0
- package/md_cg/test_autonomy_modes.py +2013 -1973
- package/md_cg/test_b1_foreign_read.py +245 -0
- package/md_cg/test_bm25.py +438 -0
- package/md_cg/test_bm25_hydration_refill.py +265 -0
- package/md_cg/test_bm25_incremental.py +241 -0
- package/md_cg/test_bm25_incremental_mutations.py +160 -0
- package/md_cg/test_bm25_mutations.py +205 -0
- package/md_cg/test_bm25_pool_view.py +196 -0
- package/md_cg/test_boot_openblas_guard.py +218 -197
- package/md_cg/test_boundary_hit.py +1 -1
- package/md_cg/test_branches_sensitivity.py +375 -0
- package/md_cg/test_c8_search_rrf_gates.py +1 -1
- package/md_cg/test_canonical_timezones.py +211 -0
- package/md_cg/test_ccg_heading_parser.py +73 -0
- package/md_cg/test_chain_best_first.py +403 -0
- package/md_cg/test_chain_candidate_scope.py +227 -0
- package/md_cg/test_cn_recall_grams.py +206 -0
- package/md_cg/test_code_kind_forbidden.py +101 -0
- package/md_cg/test_coldverify_drain.py +127 -0
- package/md_cg/test_condaccept.py +156 -0
- package/md_cg/test_condgate.py +141 -0
- package/md_cg/test_config_registry.py +55 -0
- package/md_cg/test_crypto_rotate.py +257 -0
- package/md_cg/test_csre_correctness.py +297 -0
- package/md_cg/test_csre_index_import_guard.py +161 -0
- package/md_cg/test_csre_mutations.py +34 -0
- package/md_cg/test_csre_session.py +69 -0
- package/md_cg/test_curiosity_budget.py +468 -0
- package/md_cg/test_d17_autostart_toolface.py +453 -0
- package/md_cg/test_datapath_tilde_home.py +271 -0
- package/md_cg/test_derived_cache_generations.py +256 -0
- package/md_cg/test_evidence_delivery_compact.py +54 -0
- package/md_cg/test_exact_text_cache.py +191 -0
- package/md_cg/test_excerpt_budget.py +188 -0
- package/md_cg/test_experience_consolidation.py +112 -0
- package/md_cg/test_export_default_collision.py +138 -0
- package/md_cg/test_export_snapshots.py +322 -0
- package/md_cg/test_fuzzy_substring_equivalence.py +127 -0
- package/md_cg/test_gain_gate.py +16 -5
- package/md_cg/test_goal_gen.py +355 -0
- package/md_cg/test_h2_session_view_norm.py +251 -233
- package/md_cg/test_h4_sustain_snapshot.py +1490 -1387
- package/md_cg/test_heartbeat_ledger.py +557 -0
- package/md_cg/test_hierarchy_audit_graph.py +237 -0
- package/md_cg/test_i50c_meta_passthrough.py +628 -608
- package/md_cg/test_i50e_readside_protection.py +658 -656
- package/md_cg/test_identity_attribution.py +96 -2
- package/md_cg/test_identity_contract.py +329 -0
- package/md_cg/test_imgskill.py +1444 -0
- package/md_cg/test_index_snapshot_shape.py +240 -0
- package/md_cg/test_interop_judgment.py +258 -228
- package/md_cg/test_issue111_gate_order.py +77 -0
- package/md_cg/test_issue111_scoring_contract.py +87 -0
- package/md_cg/test_issue121_calibration_bounds.py +293 -0
- package/md_cg/test_issue128_session_env_authority.py +246 -0
- package/md_cg/test_issue130_rrf_rank_credibility.py +217 -0
- package/md_cg/test_issue131_postings_content_staleness.py +424 -0
- package/md_cg/test_issue145_sleep_no_silent_rollback.py +168 -0
- package/md_cg/test_issue146_frontmatter_quarantine.py +428 -0
- package/md_cg/test_issue147_pack_id_escape.py +378 -0
- package/md_cg/test_issue148_merge_downgrade_leak.py +192 -0
- package/md_cg/test_issue149_accel_default_face.py +249 -0
- package/md_cg/test_issue150_empty_query_terms.py +328 -0
- package/md_cg/test_issue151_153_157_write_cache_index.py +152 -0
- package/md_cg/test_issue151b_base_add_importance_guard.py +166 -0
- package/md_cg/test_issue152_sealed_postings.py +309 -0
- package/md_cg/test_issue152b_decrypt_cache.py +218 -0
- package/md_cg/test_issue153b_cache_alias_and_canon.py +236 -0
- package/md_cg/test_issue153b_index_generation_invalidation.py +143 -0
- package/md_cg/test_issue157b_index_remainder.py +344 -0
- package/md_cg/test_issue189_conclusion_bidirectional.py +223 -0
- package/md_cg/test_issue190_tidy_idempotent.py +806 -0
- package/md_cg/test_issue191_import_trust.py +382 -0
- package/md_cg/test_issue192_move_layer_indices_guard.py +258 -0
- package/md_cg/test_issue193_consistency_knowledge_guard.py +524 -0
- package/md_cg/test_issue193_write_scope.py +378 -0
- package/md_cg/test_issue197_rank_credibility_property.py +201 -0
- package/md_cg/test_issue211_export_target.py +325 -0
- package/md_cg/test_issue212_writeface_gaps.py +349 -0
- package/md_cg/test_issue214_reject_sensitivity.py +267 -0
- package/md_cg/test_issue215_condition_source_gap.py +272 -0
- package/md_cg/test_issue216_clash_routes.py +214 -0
- package/md_cg/test_issue217_slot_routes.py +248 -0
- package/md_cg/test_issue218_postcondition_value_blind.py +222 -0
- package/md_cg/test_issue225_consistency_torn_log.py +433 -0
- package/md_cg/test_issue245_swarm_traversal.py +366 -0
- package/md_cg/test_issue258_pack_sensitivity.py +468 -0
- package/md_cg/test_issue265_coldverify_multiproc.py +438 -0
- package/md_cg/test_issue274_read_node_id_none.py +341 -0
- package/md_cg/test_issue275_layer_param.py +250 -0
- package/md_cg/test_issue277_node_id_traversal.py +384 -0
- package/md_cg/test_issue278_neg_separated_tension.py +271 -0
- package/md_cg/test_issue278_neg_window.py +294 -0
- package/md_cg/test_issue278_negative_layering.py +83 -0
- package/md_cg/test_issue279_signers_path_guard.py +652 -0
- package/md_cg/test_issue284_skip_not_pass.py +291 -0
- package/md_cg/test_issue294_flywheel_sensitivity.py +573 -0
- package/md_cg/test_issue295_sleep_root_binding.py +589 -0
- package/md_cg/test_issue297_rewrite_sensitivity.py +454 -0
- package/md_cg/test_issue298_consistency_ledger_sensitivity.py +421 -0
- package/md_cg/test_issue301_baseline_corpus_gate.py +245 -0
- package/md_cg/test_issue303_design_boundaries.py +262 -0
- package/md_cg/test_issue313_writelimit_sensitivity.py +404 -0
- package/md_cg/test_issue315_merge_branch_sensitivity.py +384 -0
- package/md_cg/test_issue315_sensitivity_passthrough_guard.py +514 -0
- package/md_cg/test_issue319_tenant_unregistered_bypass.py +257 -0
- package/md_cg/test_issue320_vstate_requires_verify.py +279 -0
- package/md_cg/test_issue321_mcp_deep_json.py +305 -0
- package/md_cg/test_issue322_node_write_lock.py +727 -0
- package/md_cg/test_issue323_aggregate_segment_boundary.py +365 -0
- package/md_cg/test_issue324_write_size_fields.py +215 -0
- package/md_cg/test_issue325_audit_poison.py +230 -0
- package/md_cg/test_issue326_append_jsonl_oserror.py +464 -0
- package/md_cg/test_issue326_ingest_auto_whitelist.py +331 -0
- package/md_cg/test_issue326_key_perms.py +355 -0
- package/md_cg/test_issue326_twin_parser_parity.py +323 -0
- package/md_cg/test_issue326_vm_call_frame.py +569 -0
- package/md_cg/test_issue345_writelimit_effective_sens.py +234 -0
- package/md_cg/test_issue346_overwrite_sensitivity_gate.py +424 -0
- package/md_cg/test_issue346_review_overwrite_guard.py +509 -0
- package/md_cg/test_issue348_insight_verify_op.py +200 -0
- package/md_cg/test_issue348_merge_rmw_lock.py +345 -0
- package/md_cg/test_issue350_casefold_collision.py +236 -0
- package/md_cg/test_issue351_ingest_root_fallback.py +202 -0
- package/md_cg/test_issue352_imgaudit.py +418 -0
- package/md_cg/test_issue352_pathsjson.py +537 -0
- package/md_cg/test_issue352_refindex.py +412 -0
- package/md_cg/test_issue352_rollback.py +604 -0
- package/md_cg/test_issue352_vstateinherit.py +196 -0
- package/md_cg/test_issue354_p1_writelimit_type.py +522 -0
- package/md_cg/test_issue354_p2_datapath_clock.py +381 -0
- package/md_cg/test_issue354_p2_writelimit_toctou.py +327 -0
- package/md_cg/test_issue354_rust_index_containment.py +571 -0
- package/md_cg/test_issue354_rust_wal_containment.py +606 -0
- package/md_cg/test_issue358_truncation_observable.py +247 -0
- package/md_cg/test_issue360_tidy_private_excerpt.py +196 -0
- package/md_cg/test_issue361_blindspot_not_silent_drop.py +494 -0
- package/md_cg/test_issue362_neg_condition.py +468 -0
- package/md_cg/test_issue363_lexical_idf.py +275 -0
- package/md_cg/test_issue370_readcache_assembly.py +488 -0
- package/md_cg/test_issue375_state_slot_retire.py +325 -0
- package/md_cg/test_issue376_heartbeat_failed.py +293 -0
- package/md_cg/test_issue377_layer_protect_stamp.py +324 -0
- package/md_cg/test_issue378_protected_floor.py +310 -0
- package/md_cg/test_issue383_state_event_write_gate.py +462 -0
- package/md_cg/test_issue397_sens_eff_wiring.py +435 -0
- package/md_cg/test_issue398_toolface_op_parity.py +511 -0
- package/md_cg/test_issue399_hive_json_shape.py +317 -0
- package/md_cg/test_issue400_hive_dirty_args.py +450 -0
- package/md_cg/test_issue401_wisdom_codegen.py +295 -0
- package/md_cg/test_issue404_heal_receipt.py +390 -0
- package/md_cg/test_issue405_git_pathspec.py +333 -0
- package/md_cg/test_issue408_eface_registry.py +291 -0
- package/md_cg/test_issue409_compact_index_completeness.py +509 -0
- package/md_cg/test_issue410_verify_demote_write.py +366 -0
- package/md_cg/test_issue411_scrub_idempotence.py +575 -0
- package/md_cg/test_issue412_discipline_producer_parity.py +312 -0
- package/md_cg/test_issue413_preimage_failclosed.py +239 -0
- package/md_cg/test_issue413_preimage_twin.py +340 -0
- package/md_cg/test_issue414_write_args_parity.py +613 -0
- package/md_cg/test_issue415_deps_gate_probe.py +358 -0
- package/md_cg/test_issue416_entry_key_subset.py +525 -0
- package/md_cg/test_issue417_twophase_merge_landed.py +423 -0
- package/md_cg/test_issue418_identity_role_type_gate.py +188 -0
- package/md_cg/test_issue418_role_type_gate.py +265 -0
- package/md_cg/test_issue419_pii_redaction.py +642 -0
- package/md_cg/test_issue424_merge_confirm_sensitivity.py +381 -0
- package/md_cg/test_issue425_ingest_library_path.py +331 -0
- package/md_cg/test_issue426a_dedup_receipt.py +242 -0
- package/md_cg/test_issue426b_root_clash_invisible.py +270 -0
- package/md_cg/test_issue426c_enqueue_not_silent.py +346 -0
- package/md_cg/test_issue426e_restore_overwrite_live.py +310 -0
- package/md_cg/test_issue426f_paths_gbk_failclosed.py +301 -0
- package/md_cg/test_issue427_serve_status_shape.py +191 -0
- package/md_cg/test_issue431_432_468_protect_single_point.py +322 -0
- package/md_cg/test_issue433_outlook_threshold_single_point.py +278 -0
- package/md_cg/test_issue434_435_float_gate.py +90 -0
- package/md_cg/test_issue439_sibling_sensitivity_inherit.py +448 -0
- package/md_cg/test_issue43_default_policy.py +874 -865
- package/md_cg/test_issue443a_state_event_length_gate.py +94 -0
- package/md_cg/test_issue443b_audit_record_length_gate.py +83 -0
- package/md_cg/test_issue443c_self_state_imgskill_length_gate.py +77 -0
- package/md_cg/test_issue444_scrub_evidence_plaintext.py +281 -0
- package/md_cg/test_issue445_metacognition_ledger_gate.py +454 -0
- package/md_cg/test_issue446_453_403_verified_evidence.py +382 -0
- package/md_cg/test_issue446_is_encrypted_envelope.py +166 -0
- package/md_cg/test_issue447_recent_sens_failclosed.py +270 -0
- package/md_cg/test_issue450_sensitivity_fallback_reason.py +51 -0
- package/md_cg/test_issue450_writegate_unknown_sens.py +203 -0
- package/md_cg/test_issue451_audit_actor.py +313 -0
- package/md_cg/test_issue452_ingest_root_hint.py +238 -0
- package/md_cg/test_issue452_ingest_root_sensitivity.py +299 -0
- package/md_cg/test_issue463_ghost_prune_guard.py +259 -0
- package/md_cg/test_issue466_basis_registry.py +469 -0
- package/md_cg/test_issue467_identity_sensitivity.py +204 -0
- package/md_cg/test_issue467_identity_sensitivity_parity.py +361 -0
- package/md_cg/test_issue469_scrub_override_passthrough.py +126 -0
- package/md_cg/test_issue470_wal_event_durable.py +427 -0
- package/md_cg/test_issue471_identity_readable.py +137 -0
- package/md_cg/test_issue472_audit_chain.py +91 -0
- package/md_cg/test_issue472_prune_ledger.py +106 -0
- package/md_cg/test_issue472_rollback_degraded.py +82 -0
- package/md_cg/test_issue491_compose_sandbox.py +460 -0
- package/md_cg/test_issue492_dup_readout_opt_in.py +292 -0
- package/md_cg/test_issue498_merge_trace_stale_index.py +280 -0
- package/md_cg/test_issue511_scoreboard_bad_files.py +256 -0
- package/md_cg/test_issue521_rotate_lock_suffix.py +110 -0
- package/md_cg/test_issue522_review_record_sensitivity.py +267 -0
- package/md_cg/test_issue524_orphan_noise_no_demote.py +88 -0
- package/md_cg/test_issue525_read_null_semantics.py +52 -0
- package/md_cg/test_issue538_hook_condition_space.py +97 -0
- package/md_cg/test_issue540_path_guard_whitespace.py +126 -0
- package/md_cg/test_issue541_tidy_protect_gate.py +141 -0
- package/md_cg/test_issue542_tenant_root_same.py +153 -0
- package/md_cg/test_issue543_issue_authorization.py +147 -0
- package/md_cg/test_issue544_sandbox_reflection_gate.py +175 -0
- package/md_cg/test_issue545_charset_vs_size_diagnostic.py +77 -0
- package/md_cg/test_issue546_ledger_concurrency.py +81 -0
- package/md_cg/test_issue547_append_jsonl_whitelist.py +115 -0
- package/md_cg/test_issue547_audit_durable.py +96 -0
- package/md_cg/test_issue549_durable_append.py +143 -0
- package/md_cg/test_issue557_r1_credential_value_forms.py +76 -0
- package/md_cg/test_issue557_r2_write_entries_size_gate.py +84 -0
- package/md_cg/test_issue557_r3_pem_block_redaction.py +75 -0
- package/md_cg/test_issue559_refine_prefix.py +70 -0
- package/md_cg/test_issue561_pending_review.py +109 -0
- package/md_cg/test_issue562_retired_faces.py +150 -0
- package/md_cg/test_issue66_scrub_semantics.py +107 -0
- package/md_cg/test_issue67_scrub_control_pool.py +233 -0
- package/md_cg/test_issue69_audit_boundary.py +129 -0
- package/md_cg/test_issue71_queue_backlog.py +377 -0
- package/md_cg/test_issue72_coldverify_controlled_drain.py +420 -0
- package/md_cg/test_issue73_doc_chapter_refs.py +326 -0
- package/md_cg/test_issue75_neural_failure.py +582 -0
- package/md_cg/test_issue78_export_concurrency.py +334 -0
- package/md_cg/test_issue80_residual_flush.py +169 -0
- package/md_cg/test_issue82_credential_rules.py +271 -0
- package/md_cg/test_issue82_layer_and_size.py +120 -0
- package/md_cg/test_issue84_portability.py +95 -0
- package/md_cg/test_issue85_export_overwrite.py +113 -0
- package/md_cg/test_issue86_runtime_revoke.py +110 -0
- package/md_cg/test_issue87_default_sensitivity_single.py +201 -0
- package/md_cg/test_issue88_conclusion_face.py +280 -0
- package/md_cg/test_issue88_execution_conflicts.py +87 -0
- package/md_cg/test_issue89_zero_score.py +169 -0
- package/md_cg/test_issue89a_score_cap_tiebreak.py +279 -0
- package/md_cg/test_issue91_verifier_sandbox.py +314 -0
- package/md_cg/test_issue91_verifier_timeout.py +528 -0
- package/md_cg/test_issue93_remember_redaction.py +282 -0
- package/md_cg/test_issue96_key_fallback.py +102 -0
- package/md_cg/test_issue98_tenant_root.py +104 -0
- package/md_cg/test_jsonl_commit_watermark.py +221 -0
- package/md_cg/test_live_validity_cache.py +256 -0
- package/md_cg/test_logref.py +1141 -1109
- package/md_cg/test_mcp_call_argument_types.py +99 -0
- package/md_cg/test_md_access_parity.py +330 -255
- package/md_cg/test_md_cg_no_hive_dep.py +140 -0
- package/md_cg/test_memory_rsi.py +215 -0
- package/md_cg/test_memory_rsi_adversarial.py +91 -0
- package/md_cg/test_mode_parity.py +1413 -1393
- package/md_cg/test_mutation_residue.py +120 -0
- package/md_cg/test_mutation_rollback.py +835 -805
- package/md_cg/test_n123_derive_expiry_chain.py +210 -205
- package/md_cg/test_n130_verify_falsified_protect.py +185 -185
- package/md_cg/test_n209_verify_write_face_gates.py +466 -461
- package/md_cg/test_n230_dirty_replay.py +52 -9
- package/md_cg/test_n62_tenant_bind_failclosed.py +200 -200
- package/md_cg/test_neg_coverage_budget.py +500 -0
- package/md_cg/test_neg_tail_honesty.py +733 -663
- package/md_cg/test_p0.py +16 -2
- package/md_cg/test_p14_root_fallback.py +373 -0
- package/md_cg/test_p1x_ref_root.py +47 -10
- package/md_cg/test_p21_tokens.py +294 -293
- package/md_cg/test_p24_evidence.py +232 -227
- package/md_cg/test_p26_refindex.py +20 -3
- package/md_cg/test_p27_docindex.py +792 -777
- package/md_cg/test_p28_refcheck.py +1053 -1032
- package/md_cg/test_p29_session_ingest_export.py +6 -0
- package/md_cg/test_p2_mcp.py +474 -453
- package/md_cg/test_p2_six_elements.py +476 -463
- package/md_cg/test_p3.py +370 -261
- package/md_cg/test_p33_ccg_wiring.py +300 -293
- package/md_cg/test_p3_legacy_closure.py +438 -433
- package/md_cg/test_p40_refine_worklist.py +244 -241
- package/md_cg/test_p47_session_view.py +25 -9
- package/md_cg/test_p4_freshness.py +781 -774
- package/md_cg/test_p83_jsonrpc_parse_error.py +124 -0
- package/md_cg/test_p9c_dedup_hints.py +288 -283
- package/md_cg/test_policy_required_ccg.py +444 -432
- package/md_cg/test_postings_digest_memo.py +139 -0
- package/md_cg/test_predict_beta.py +138 -135
- package/md_cg/test_qmd.py +500 -0
- package/md_cg/test_query_excerpt.py +209 -0
- package/md_cg/test_reach.py +456 -378
- package/md_cg/test_reach_snapshot_admission.py +218 -0
- package/md_cg/test_readcache_prodpath.py +4 -0
- package/md_cg/test_recall_face_guards.py +854 -812
- package/md_cg/test_recall_metadata.py +422 -0
- package/md_cg/test_retr_s5.py +312 -213
- package/md_cg/test_retr_score_once.py +14 -2
- package/md_cg/test_review_cli_attribution.py +11 -7
- package/md_cg/test_review_queue_batch.py +123 -0
- package/md_cg/test_review_queue_index.py +130 -0
- package/md_cg/test_rrf_candidates.py +292 -0
- package/md_cg/test_rrf_qualified_topk.py +214 -0
- package/md_cg/test_rrf_unique_votes.py +218 -0
- package/md_cg/test_s1_ast_judge.py +110 -0
- package/md_cg/test_scrub_ledger.py +413 -0
- package/md_cg/test_security_audit_b26.py +34 -3
- package/md_cg/test_security_audit_v21.py +1 -2
- package/md_cg/test_sensitivity_predicate_single_point.py +184 -0
- package/md_cg/test_session5_identity.py +93 -0
- package/md_cg/test_session_attribution_consistency.py +813 -0
- package/md_cg/test_session_identity_tristate.py +486 -0
- package/md_cg/test_sleep_p1.py +35 -3
- package/md_cg/test_state_event_op.py +521 -514
- package/md_cg/test_state_projection_streaming.py +136 -0
- package/md_cg/test_stg_interval_candidates.py +280 -0
- package/md_cg/test_stg_time_blocks.py +239 -0
- package/md_cg/test_stgidx_index_parity.py +1024 -1020
- package/md_cg/test_subgraph_bounded_bfs.py +227 -0
- package/md_cg/test_sustain_bounded.py +652 -624
- package/md_cg/test_task_memory.py +394 -0
- package/md_cg/test_text_projection_mutations.py +101 -0
- package/md_cg/test_time_anchor.py +421 -0
- package/md_cg/test_time_core_lint.py +1024 -975
- package/md_cg/test_tool_face.py +200 -189
- package/md_cg/test_trust_bounded_frontier.py +248 -0
- package/md_cg/test_trust_edge_reachability.py +405 -0
- package/md_cg/test_w5_path_fingerprint.py +381 -0
- package/md_cg/test_w7_redlines.py +659 -0
- package/md_cg/test_write_return_shape.py +919 -0
- package/md_cg/test_writepipe.py +228 -226
- package/md_cg/timeanchor.py +534 -0
- package/md_cg/tokens.py +68 -5
- package/md_cg/tool_face.py +273 -260
- package/md_cg/trust.py +1114 -1030
- package/md_cg/twophase.py +303 -231
- package/md_cg/weights.py +642 -638
- package/md_cg/whitebox.py +536 -527
- package/md_cg/whitebox_kb/aeis_core/core.py +5026 -5026
- package/md_cg/whitebox_kb/wisdom/chat_engine.py +1504 -1506
- package/md_cg/whitebox_kb/wisdom/code_compose.py +876 -837
- package/md_cg/whitebox_kb/wisdom/compiler_code_units.py +3040 -3033
- package/md_cg/whitebox_kb/wisdom/compose_engine.py +1634 -1633
- package/md_cg/whitebox_kb/wisdom/core/condition_vm.py +318 -298
- package/md_cg/whitebox_kb/wisdom/core/parser.py +1082 -1065
- package/md_cg/whitebox_kb/wisdom/csre.py +412 -264
- package/md_cg/whitebox_kb/wisdom/csre_session.py +97 -0
- package/md_cg/whitebox_kb/wisdom/md_access.py +490 -470
- package/md_cg/whitebox_kb/wisdom/migrate_wisdom.py +497 -476
- package/md_cg/whitebox_kb/wisdom/mini_python.py +995 -958
- package/md_cg/whitebox_kb/wisdom/navigate.py +249 -248
- package/md_cg/whitebox_kb/wisdom/net_units.py +2900 -2895
- package/md_cg/whitebox_kb/wisdom/neural_retrieve.py +100 -10
- package/md_cg/whitebox_kb/wisdom/semantic_translate.py +4060 -4054
- package/md_cg/whitebox_kb/wisdom/test_ecosystem_assembly.py +158 -158
- package/md_cg/whitebox_kb/wisdom/test_safe_module_proxy.py +147 -0
- package/md_cg/whitebox_kb/wisdom/verifier.py +1918 -1297
- package/md_cg/writelimit.py +584 -390
- package/md_cg/writepipe.py +1337 -826
- package/package.json +14 -9
- package/scripts/workspace_index.py +328 -0
- package/src/bridge.ts +34 -11
- package/src/cli.ts +75 -65
- package/src/hooks.ts +932 -776
- package/src/index.ts +30 -7
- package/src/init.ts +362 -361
- package/src/lib/datapath.ts +419 -326
- package/src/lib/hook_audit.ts +42 -3
- package/src/lib/mdcg_client.ts +469 -461
- package/src/lib/mutual.ts +481 -432
- package/src/lib/prompt_safety.ts +178 -149
- package/src/lib/roleplay_web.ts +30 -2
- package/src/lib/token_store.ts +274 -52
- package/src/tools.ts +271 -223
- package/zcode/README.md +1 -1
- package/docs/GBrain/345/217/257/345/200/237/351/211/264/347/202/271_/347/201/265/346/236/242/350/220/275/347/202/271/344/272/244/346/216/245_20260919.md +0 -169
- package/docs/Pi/345/217/257/345/255/246/344/271/240/344/274/230/347/202/271_/347/201/265/346/236/242/345/244/247/350/204/221/346/224/271/350/277/233/344/272/244/346/216/245_20260915.md +0 -144
- package/docs/eval/AGI/344/270/203/347/273/264/350/257/204/345/210/206/346/212/245/345/221/212_md_cg_v1.0.md +0 -299
- package/docs/eval/AGI/344/270/203/347/273/264/350/257/204/345/210/206/346/212/245/345/221/212_md_cg_v2.0.md +0 -239
- package/docs/eval/AGI/344/270/203/347/273/264/350/257/204/345/210/206/346/212/245/345/221/212_/345/205/255/345/256/266/350/256/260/345/277/206/347/263/273/347/273/237/345/220/214/345/260/272/345/256/236/346/265/213_v3.1.md +0 -340
- package/docs/eval/DSH/346/227/245/345/277/227/347/264/242/345/274/225v2_/345/217/202/350/200/203dsh-TUI_v1.0.md +0 -248
- package/docs/eval/DSH/346/227/245/345/277/227/347/264/242/345/274/225/346/225/210/346/236/234/351/252/214/350/257/201_v1.0.md +0 -209
- package/docs/eval/DSH/347/253/257/347/274/272/351/231/267/344/270/223/351/241/271_v1.0.md +0 -254
- package/docs/eval/N225_/347/264/242/345/274/225/346/227/245/345/277/227/351/235/236/345/257/271/350/261/241/350/243/205/350/275/275/351/235/242/347/261/273/345/236/213/351/227/270_v1.0.md +0 -418
- package/docs/eval/P1b2_/350/257/273/351/235/242/344/273/243/351/231/205/344/277/256/345/244/215_v1.0.md +0 -124
- package/docs/eval/bench_lingshu_self/bench_self.py +0 -140
- package/docs/eval/bench_lingshu_self/self_bench_result.json +0 -404
- package/docs/eval/bench_lingshu_self//347/201/265/346/236/242/350/207/252/345/272/223/347/253/257/345/210/260/347/253/257/346/243/200/347/264/242/345/256/236/346/265/213_v1.0.md +0 -38
- package/docs/eval/cons200_/345/206/262/347/252/201/346/243/200/346/265/213/351/200/211/351/235/242_/345/256/236/346/226/275/350/256/260/345/275/225_v1.0.md +0 -223
- package/docs/eval/cons200_/345/206/262/347/252/201/346/243/200/346/265/213/351/200/211/351/235/242_/345/256/236/346/226/275/350/256/260/345/275/225_v1.1.md +0 -340
- package/docs/eval/issue43_/351/273/230/350/256/244/347/255/226/347/225/245/344/270/216/351/224/256/347/261/273/345/236/213/351/227/270_v1.1.md +0 -441
- package/docs/eval/issue43_/351/273/230/350/256/244/347/255/226/347/225/245/345/212/240/350/275/275/344/270/216/345/207/255/346/215/256/346/230/216/346/226/207/351/230/237/345/210/227_v1.0.md +0 -398
- package/docs/eval/issue50_/345/205/203/346/225/260/346/215/256/351/200/217/344/274/240/344/270/216/345/205/234/345/272/225_/344/277/256/345/244/215/350/256/260/345/275/225_v1.0.md +0 -437
- package/docs/eval/issue50_/345/215/212/351/207/215/345/244/215/345/276/205/345/256/232/345/244/215/346/240/270_/344/277/256/345/244/215/350/256/260/345/275/225_v1.0.md +0 -488
- package/docs/eval/issue50_/345/276/205/345/256/232/345/244/215/346/240/270/345/205/245/351/230/237_/344/277/256/345/244/215/350/256/260/345/275/225_v1.0.md +0 -428
- package/docs/eval/issue50_/350/257/273/351/235/242/344/277/235/346/212/244/345/217/252/350/256/244/346/230/276/345/274/217/346/235/245/346/272/220_/344/277/256/345/244/215/350/256/260/345/275/225_v1.0.md +0 -271
- package/docs/eval/issue50_/351/207/215/350/246/201/345/272/246/345/220/214/346/272/220/344/270/216/344/277/235/346/212/244/350/257/255/344/271/211_/344/277/256/345/244/215/350/256/260/345/275/225_v1.0.md +0 -306
- package/docs/eval/issue51_/344/270/200/351/224/256/345/256/211/350/243/205/345/244/261/350/264/245_/345/275/222/345/261/236/345/210/244/345/256/232_v1.0.md +0 -58
- package/docs/eval/issue52_/346/235/241/344/273/266/345/205/210/350/241/214/344/270/216/346/210/252/346/226/255/345/217/257/350/247/202/346/265/213_/344/277/256/345/244/215/350/256/260/345/275/225_v1.0.md +0 -360
- package/docs/eval/issue64_replay_check_/346/240/270/351/252/214/344/270/216/344/277/256/345/244/215_v1.0.md +0 -284
- package/docs/eval/issue65_session_recall_/344/274/232/350/257/235/351/232/224/347/246/273_/346/240/270/351/252/214/344/270/216/344/277/256/345/244/215_v1.0.md +0 -329
- package/docs/eval//344/270/211/346/241/243/350/207/252/346/262/273_/346/255/245/351/252/244/342/221/241/346/241/243/344/275/215/345/215/225/344/270/200/345/205/245/345/217/243_/350/220/275/347/240/201/350/256/260/345/275/225_v1.0.md +0 -651
- package/docs/eval//344/270/211/346/241/243/350/207/252/346/262/273_/346/255/245/351/252/244/342/221/242/345/217/230/346/233/264/345/215/225/344/270/216/345/233/236/346/273/232/345/216/237/350/257/255_/350/220/275/347/240/201/350/256/260/345/275/225_v1.0.md +0 -439
- package/docs/eval//344/270/211/346/241/243/350/207/252/346/262/273_/346/255/245/351/252/244/342/221/243/345/207/206/345/205/245/350/257/273/346/225/260_/350/220/275/347/240/201/350/256/260/345/275/225_v1.0.md +0 -1337
- package/docs/eval//344/270/211/346/241/243/350/207/252/346/262/273_/346/255/245/351/252/244/342/221/244/346/224/266/345/256/230/344/270/216/345/205/250/351/223/276/351/252/214/346/224/266_/350/220/275/347/240/201/350/256/260/345/275/225_v1.0.md +0 -1817
- package/docs/eval//344/270/215/345/217/257/351/235/240/346/200/247/350/220/275/345/234/260_P0_v1.0.md +0 -185
- package/docs/eval//344/274/230/345/214/226/347/254/254/344/270/200/346/211/271_/346/216/245/347/272/277/344/270/216/347/255/211/344/273/267/345/217/230/346/215/242_v1.0.md +0 -498
- package/docs/eval//344/274/230/345/214/226/347/254/254/344/270/211/346/211/271_/351/227/250/347/246/201/350/275/254/346/255/243/344/270/216/350/260/203/345/272/246/346/255/242/350/241/200/344/270/216/351/227/250/346/216/247/346/224/266/345/217/243_v1.0.md +0 -669
- package/docs/eval//344/274/230/345/214/226/347/254/254/344/272/214/346/211/271_/344/276/235/350/265/226/351/200/217/344/274/240/344/270/216/345/257/271/346/213/215/345/217/243/345/276/204/344/270/216/350/264/237/347/274/223/345/255/230_v1.0.md +0 -683
- package/docs/eval//345/207/272/350/264/247/351/235/242/345/206/222/347/203/237_/350/277/233/350/264/247/351/227/250/347/246/201_v1.0.md +0 -460
- package/docs/eval//345/217/221/345/270/20306_/344/270/200/351/224/256/351/205/215/347/275/256/344/270/216DSH0172_v1.0.md +0 -171
- package/docs/eval//345/217/221/345/270/20307_/345/244/226/351/203/250/346/212/245/345/221/212/345/233/233/346/211/271/344/277/256/345/244/215/344/270/216/346/217/222/344/273/266/351/235/242/345/212/240/345/233/272_v1.0.md +0 -209
- package/docs/eval//345/217/221/345/270/20308_/350/207/252/350/277/255/344/273/243/344/270/216/347/235/241/347/234/240_/345/233/276/346/243/200/347/264/242/350/267/257/344/270/216/346/235/203/351/207/215_v1.0.md +0 -97
- package/docs/eval//345/217/221/345/270/20309_/346/243/200/347/264/242/351/235/242/344/270/211/346/211/271/346/224/266/345/217/243_v1.0.md +0 -66
- package/docs/eval//345/217/221/345/270/20310_/350/257/273/351/235/242/346/215/237/345/235/217UTF8/345/256/266/346/227/217/346/224/266/345/217/243_v1.0.md +0 -71
- package/docs/eval//345/217/221/345/270/20311_/345/244/226/346/212/245/345/205/255/350/277/236/344/277/256/344/270/216/344/270/226/347/225/214/346/250/241/345/236/213/345/272/225/345/272/247_v1.0.md +0 -91
- package/docs/eval//345/217/221/345/270/20312_/344/270/226/347/225/214/346/250/241/345/236/213/345/212/237/350/203/275/347/253/257_v1.0.md +0 -108
- package/docs/eval//345/217/221/345/270/20313_/344/270/212/344/270/213/346/226/207/350/207/252/347/256/241/347/220/206/346/234/272/345/210/266_v1.0.md +0 -114
- package/docs/eval//345/217/221/345/270/20314_/350/272/253/344/275/223/303/227/350/204/221/347/273/204/345/220/210_v1.0.md +0 -160
- package/docs/eval//345/217/221/345/270/20315_/344/274/232/350/257/235/351/232/224/347/246/273/344/277/256/345/244/215/344/270/216/350/272/253/344/275/223/345/272/223/345/261/225/347/244/272_v1.0.md +0 -88
- package/docs/eval//345/256/236/351/252/214/346/226/271/346/241/210_/345/255/246/344/271/240/351/227/255/347/216/257AB/344/270/216/346/250/252/350/257/204_v1.0.md +0 -174
- package/docs/eval//345/275/222/344/270/200/345/261/202/347/274/272/347/234/201/347/277/273/345/205/263_/344/277/256/345/244/215/350/256/260/345/275/225_v1.0.md +0 -458
- package/docs/eval//346/200/247/350/203/275/344/270/223/351/241/271_/345/206/267/346/237/245/350/257/242/344/270/216/345/206/205/345/255/230_v1.0.md +0 -218
- package/docs/eval//346/225/205/351/232/234/346/263/250/345/205/245/345/256/236/346/265/213_v1.0.md +0 -432
- package/docs/eval//346/250/252/350/257/204_/345/205/255/345/256/266100/351/242/230/344/270/255/350/213/261/345/217/214/346/237/245_v1.0.md +0 -223
- package/docs/eval//347/224/250/346/210/267/350/243/201/345/206/263/345/244/204/347/275/256_/347/254/254/344/270/200/346/211/271_v1.0.md +0 -195
- package/docs/eval//347/224/250/346/210/267/350/243/201/345/206/263/345/244/204/347/275/256_/347/254/254/344/272/214/346/211/271B6_/346/234/254/346/234/272/350/267/257/345/276/204/347/233/270/345/257/271/345/214/226_v1.0.md +0 -78
- package/docs/eval//347/253/257/345/210/260/347/253/257LoCoMoQA/345/220/214/345/217/243/345/276/204/345/257/271/347/205/247_v1.0.md +0 -100
- package/docs/eval//347/253/257/345/210/260/347/253/257/345/271/262/346/211/260/346/261/240/350/257/204/346/265/213_/347/241/256/345/256/232/346/200/247/350/243/201/345/206/263vsLLM_judge_v1.1.md +0 -197
- package/docs/eval//347/254/2543/345/261/202stg/347/273/223/346/236/204/347/264/242/345/274/225_/345/256/236/346/226/275/350/256/260/345/275/225_v1.0.md +0 -250
- package/docs/eval//347/254/254/344/270/211/346/226/271/351/252/214/350/257/201/346/212/245/345/221/212_LoCoMo_/347/201/265/346/236/242_.png +0 -0
- package/docs/eval//347/254/254/344/270/211/346/226/271/351/252/214/350/257/201/346/212/245/345/221/212_/347/201/265/346/236/242_vs_dejavu_/347/273/237/344/270/200/350/257/204/345/210/206_v7.md +0 -202
- package/docs/eval//347/274/226/347/240/201/351/235/242/345/211/215/347/275/256_/345/205/245/345/217/243/350/207/252/344/277/235/350/257/201UTF8/344/270/216/346/226/207/346/234/254open/345/256/210/345/215/253_v1.0.md +0 -646
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v1.md +0 -156
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v10.md +0 -210
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v11.md +0 -227
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v12.md +0 -203
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v13.md +0 -233
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v14.md +0 -191
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v15.md +0 -213
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v16.md +0 -214
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v17.md +0 -199
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v18.md +0 -183
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v19.md +0 -207
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v2.md +0 -156
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v20.md +0 -283
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v21.md +0 -224
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v22.md +0 -223
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v23.md +0 -293
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v24.md +0 -303
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v25.md +0 -230
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v26.md +0 -196
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v27.md +0 -259
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v28.md +0 -217
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v29.md +0 -270
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v3.md +0 -152
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v30.md +0 -231
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v31.md +0 -281
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v4.md +0 -128
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v5.md +0 -114
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v6.md +0 -192
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v7.md +0 -187
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v8.md +0 -207
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v9.md +0 -203
- package/docs/eval//350/257/255/344/271/211/346/227/266/347/251/272/345/233/276/350/241/245/345/205/250_P1_/346/212/275/345/217/226/345/231/250v4_/344/270/211/347/216/207/350/257/273/346/225/260_v1.0.md +0 -231
- package/docs/eval//351/262/270/345/250/230/344/270/203/347/261/273/347/212/266/346/200/201/350/277/275/350/270/252_v1.0.md +0 -176
- package/docs/hive//344/270/273/344/273/243/347/220/206/345/255/220/344/273/243/347/220/206/350/256/260/345/277/206/346/236/266/346/236/204/350/256/276/350/256/241.md +0 -385
- package/docs/hive//344/273/244/347/211/214/344/270/216/350/247/222/350/211/262/346/235/203/350/201/214/345/210/206/347/246/273_v0.1.md +0 -165
- package/docs/hive//344/273/244/347/211/214/350/257/255/344/271/211/344/277/256/346/255/243_/344/273/262/350/243/201/344/275/215_v0.1.md +0 -76
- package/docs/hive//345/244/232/347/253/257harness/351/200/232/344/277/241/345/245/221/347/272/246_v0.1.md +0 -42
- package/docs/hive//345/255/220/344/273/243/347/220/206/351/205/215/347/275/256/346/240/207/345/207/206_v0.5.md +0 -236
- package/docs/hive//345/256/211/345/205/250/345/256/241/350/256/241/345/256/236/351/224/232_v0.1.md +0 -202
- package/docs/hive//346/243/200/347/264/242/346/224/266/346/225/233/345/256/236/346/265/213/344/270/216S1b/350/256/276/350/256/241_v0.1.md +0 -43
- package/docs/hive//346/243/200/347/264/242/347/256/227/346/263/225/345/217/243/345/276/204/345/257/271/347/205/247_v0.1.md +0 -210
- package/docs/hive//346/243/200/347/264/242/350/267/257/345/276/204/344/270/216/350/256/244/347/237/245/347/273/223/346/236/204/345/245/221/347/272/246_v0.1.md +0 -152
- package/docs/hive//350/234/202/345/267/242M6_ingest/345/256/236/346/226/275/350/256/241/345/210/222_v0.1.md +0 -47
- package/docs/hive//350/234/202/345/267/242/345/217/214/345/256/236/344/276/213/344/272/222/351/252/214_/350/256/276/350/256/241/345/256/232/347/250/277.md +0 -519
- package/docs/hive//350/234/202/345/267/242/345/267/245/344/275/234/350/256/260/345/277/206_/351/241/271/347/233/256/350/256/241/345/210/222.md +0 -85
- package/docs/hive//350/234/202/345/267/242/350/256/276/350/256/241_/347/220/206/350/256/272/345/257/271/351/275/220_v0.1.md +0 -191
- package/docs/hive//350/234/202/345/267/242/350/277/255/344/273/243_/345/256/217/350/247/202/344/270/216/347/276/244/344/275/223/350/260/203/345/272/246_v0.1.md +0 -90
- package/docs/images/lingshu-moonlight-covenant-poster-preview.jpg +0 -0
- package/docs/images/lingshu-moonlight-covenant-poster.png +0 -0
- package/docs/mdcg/D_meta_/345/267/245/347/250/213/345/214/226/346/226/271/346/241/210_v0.2.md +0 -216
- package/docs/mdcg/README/350/257/246/347/273/206/347/211/210_v0.4.10.md +0 -746
- package/docs/mdcg/guardrail-charter.md +0 -164
- package/docs/mdcg/lingshu_tutorial.html +0 -14956
- package/docs/mdcg/memory-assessment.html +0 -923
- package/docs/mdcg/memory_score.html +0 -213
- package/docs/mdcg/memory_score.png +0 -0
- package/docs/mdcg/release_v0.3.0.md +0 -36
- package/docs/mdcg/release_v0.4.11.md +0 -49
- package/docs/mdcg/release_v0.4.5.md +0 -55
- package/docs/mdcg/tool_table_v0.3.0.md +0 -117
- package/docs/mdcg//344/270/226/347/225/214/346/250/241/345/236/213/345/212/237/350/203/275/347/253/257_/344/275/277/347/224/250/344/270/216/350/277/220/347/273/264_v1.0.md +0 -384
- package/docs/mdcg//344/273/243/347/240/201/350/257/204/345/256/241/344/270/216/346/235/241/344/273/266/345/214/226/346/263/250/351/207/212_/345/245/221/347/272/246_v0.1.md +0 -82
- package/docs/mdcg//345/205/250/345/272/223/344/273/243/347/240/201/350/257/204/345/256/241/344/270/216/346/235/241/344/273/266/345/214/226/346/263/250/351/207/212_/350/256/241/345/210/222_v0.1.md +0 -600
- package/docs/mdcg//345/212/237/350/203/275/350/260/203/347/224/250/346/230/240/345/260/204/350/241/250_v0.1.md +0 -266
- package/docs/mdcg//345/215/225/345/205/203/350/207/252/346/210/221/351/224/232/347/202/271_/347/263/273/347/273/237/346/217/220/347/244/272/350/257/215/346/240/207/345/207/206_v0.3.md +0 -275
- package/docs/mdcg//345/217/221/345/270/203/351/227/250/347/246/201/351/223/276_v0.1.md +0 -134
- package/docs/mdcg//346/272/220/347/240/201/347/272/247/346/236/266/346/236/204/345/256/241/350/256/241_GPT/346/211/271/350/257/204/345/257/271/347/205/247_v1.0.md +0 -130
- package/docs/mdcg//347/201/265/346/236/24282/345/267/245/345/205/267_/345/212/237/350/203/275/346/225/264/347/220/206/344/270/216/350/277/201/347/247/273/346/230/240/345/260/204_v0.1.md +0 -278
- package/docs/mdcg//347/201/265/346/236/242MCP/345/267/245/345/205/267/346/200/273/350/241/250_v3.4.md +0 -164
- package/docs/mdcg//347/201/265/346/236/242_/350/207/252/346/210/221/345/261/202/345/256/232/344/271/211.md +0 -43
- package/docs/mdcg//347/201/265/346/236/242/344/270/211/345/261/202/346/213/206/345/210/206/350/247/204/345/210/222_v0.1.md +0 -181
- package/docs/mdcg//347/201/265/346/236/242/350/256/260/345/277/206/345/212/250/350/257/215/345/215/217/350/256/256_v1.0-draft.md +0 -177
- package/docs/mdcg//347/216/257/344/272/214_/347/231/275/347/256/261/345/241/253/345/205/205/346/265/201/346/260/264/347/272/277_v0.1.md +0 -31
- package/docs/mdcg//347/235/241/347/234/240/345/221/250/346/234/237_/350/277/220/347/273/264/345/211/215/346/217/220/344/270/216/347/273/264/346/212/244/346/214/207/345/215/227_v1.0.md +0 -337
- package/docs/mdcg//347/274/272/345/217/243/345/215/225_P0/346/224/266/345/217/243_v0.1.md +0 -270
- package/docs/mdcg//350/256/244/347/237/245/345/233/276_G4-G8/347/274/272/345/217/243/350/243/201/345/256/232/345/215/225_v0.1.md +0 -491
- package/docs/mdcg//350/256/244/347/237/245/345/233/276_MD/347/233/256/345/275/225/346/226/271/346/241/210_v0.1.md +0 -552
- package/docs/mdcg//350/256/244/347/237/245/345/233/276_/346/235/241/344/273/266/347/251/272/351/227/264/345/220/210/346/210/220/344/270/216/347/224/237/346/225/210/346/235/241/344/273/266/345/217/243/345/276/204_v0.1.md +0 -340
- package/docs/mdcg//350/256/244/347/237/245/345/233/276_/347/264/242/345/274/225/344/270/216/345/267/245/347/250/213/350/247/204/350/214/203/345/214/226_/350/256/241/345/210/222_v0.1.md +0 -624
- package/docs/mdcg//350/256/244/347/237/245/345/233/276/344/275/234/344/270/272/350/256/260/345/277/206/346/223/215/344/275/234/347/263/273/347/273/237_/350/257/204/344/274/260/344/270/216/350/267/257/347/272/277_v0.1.md +0 -94
- package/docs/mdcg//350/256/260/345/277/206/346/223/215/344/275/234/347/263/273/347/273/237_MdCGOS/344/270/216MCP/346/216/245/345/205/245_v0.1.md +0 -281
- package/docs/mdcg//350/267/250/347/253/257/351/252/214/350/257/201/344/270/216/345/220/214/346/255/245/345/215/217/350/256/256_v0.1.md +0 -93
- package/docs/plans/GridWorld/346/234/200/345/260/217/351/227/255/347/216/257/350/247/204/346/240/274_v0.1.md +0 -176
- package/docs/plans/stg/346/235/241/344/273/266/345/214/226/344/270/216/347/273/223/346/236/204/347/264/242/345/274/225_/350/256/276/350/256/241_v0.1.md +0 -124
- package/docs/plans//345/205/250/344/270/255/346/226/207/347/274/226/347/240/201/344/270/216/350/234/202/345/267/242/344/273/273/345/212/241/346/240/207/350/257/206/345/245/221/347/272/246_v2.0.md +0 -220
- package/docs/plans//345/221/275/345/220/215/346/262/273/347/220/206_/351/241/271/347/233/256/350/256/241/345/210/222.md +0 -81
- package/docs/plans//345/244/232/344/270/273/344/275/223/344/270/226/347/225/214/346/250/241/345/236/213_/345/257/271/351/275/220/350/257/204/344/274/260/344/270/216/350/220/275/345/234/260/350/256/276/350/256/241_v0.1.md +0 -141
- package/docs/plans//347/201/265/346/236/242/350/272/253/344/275/223/303/227/350/204/221_/344/270/226/347/225/214/346/250/241/345/236/213/345/257/271/346/216/245/350/256/276/350/256/241_v0.1.md +0 -119
- package/docs/plans//347/234/237/346/272/220/347/264/242/345/274/225_/351/200/232/347/224/250/346/234/272/345/210/266_v0.3.md +0 -227
- package/docs/plans//347/235/241/347/234/240/344/270/216/350/207/252/350/277/255/344/273/243_/345/212/237/350/203/275/344/274/230/345/214/226/350/256/276/350/256/241_v0.4.md +0 -547
- package/docs/plans//347/237/245/350/257/206/345/233/276/350/260/261/351/241/271/347/233/256_/344/272/244/346/216/245/346/226/207/346/241/243_20260914.md +0 -308
- package/docs/plans//350/234/202/345/267/242/346/250/241/345/236/213/345/257/206/351/222/245/351/205/215/347/275/256/351/235/242_v1.0.md +0 -385
- package/docs/plans//350/256/260/345/277/206/350/207/252/345/244/204/347/220/206/344/270/211/346/241/243/350/207/252/346/262/273_/350/256/276/350/256/241_v0.2.md +0 -207
- package/docs/plans//350/257/255/344/271/211/346/227/266/347/251/272/345/233/276/350/241/245/345/205/250_/344/270/226/347/225/214/346/250/241/345/236/213/345/212/237/350/203/275/347/253/257_/350/256/276/350/256/241_v0.1.md +0 -107
- package/docs/promo/bilibili-cover.jpg +0 -0
- package/docs/promo/promo-visual-1.png +0 -0
- package/docs/promo/promo-visual-2.png +0 -0
- package/docs/promo/promo-visual-3.png +0 -0
- package/docs/promo/promo-visual-4.png +0 -0
- package/docs/promo/promo-visual-5.png +0 -0
- package/docs/swarm//350/234/202/347/276/244/344/272/222/350/201/224_v0.1.md +0 -704
- package/docs/swarm//350/234/202/347/276/244/345/220/214/351/224/231/346/243/200/346/265/213/345/256/236/351/252/214/345/215/217/350/256/256_v0.1.md +0 -242
- package/docs/swarm//350/234/202/347/276/244/345/244/232/346/231/272/350/203/275/344/275/223_/344/275/277/347/224/250/346/212/245/345/221/212_20260913.md +0 -68
- package/docs/swarm//350/234/202/347/276/244/345/244/232/346/231/272/350/203/275/344/275/223_/345/212/237/350/203/275/350/257/264/346/230/216_v0.6.md +0 -130
- package/docs/swarm//350/234/202/347/276/244/347/233/262/345/214/272/346/240/207/350/256/260_v1.0.md +0 -89
- package/docs/theory//344/270/215/345/217/257/351/235/240/345/256/232/347/220/206/344/270/216/345/244/261/346/225/210/344/274/230/345/205/210/346/241/206/346/236/266_v0.3.md +0 -365
- package/docs/theory//344/277/241/346/201/257/345/267/256/344/270/272/344/273/200/344/271/210/345/277/205/347/204/266/345/255/230/345/234/250/344/270/224/350/207/252/347/204/266/346/211/251/345/244/247.md +0 -239
- package/docs/theory//345/215/225/347/272/277/347/250/213/344/270/216/346/263/250/346/204/217/345/212/233/351/233/206/344/270/255_/346/227/240/344/272/211/350/256/256/347/220/206/350/256/272/346/226/207/346/241/243_v1.0.md +0 -129
- package/docs/theory//345/271/266/345/217/221/345/277/205/347/204/266/346/200/247/347/220/206/350/256/272_v0.2.md +0 -134
- package/docs/theory//346/231/272/350/203/275/347/232/204/345/205/254/347/220/206/345/214/226/345/237/272/347/237/263.md +0 -789
- package/docs/theory//346/231/272/350/203/275/347/232/204/350/256/244/347/237/245/350/277/207/347/250/213.md +0 -379
- package/docs/theory//346/231/272/350/203/275/350/256/2723.4.md +0 -5260
- package/docs/theory//346/246/202/345/277/265/345/210/206/345/261/202/345/257/271/351/275/220/350/241/250_v0.1.md +0 -145
- package/docs/theory//347/220/206/350/256/272_/346/234/272/345/210/266_/344/273/243/347/240/201_/345/256/236/351/252/214_/347/274/272/345/217/243/347/237/251/351/230/265_v0.1.md +0 -144
- package/docs/theory//347/220/206/350/256/272/344/273/223/346/213/206/345/210/206/344/270/216/347/231/275/347/256/261/347/237/245/350/257/206/345/272/223/345/206/205/350/277/201_v0.1.md +0 -198
- package/docs/theory//347/231/275/347/256/261/346/231/272/350/203/275/346/230/257/344/273/200/344/271/210/357/274/237.md +0 -812
- package/docs/theory//347/231/275/347/256/261/346/231/272/350/203/275/347/263/273/345/210/227/302/267/347/254/254/344/272/224/347/257/207/357/274/232/350/256/251AI/347/234/237/346/255/243/350/243/205/344/270/212/350/256/260/345/277/206.md +0 -421
- package/docs/theory//350/256/244/347/237/245/344/273/243/347/220/206/344/270/216/346/224/266/346/225/233/347/273/223/346/236/204_/346/235/241/344/273/266/350/256/272/351/207/215/346/236/204_v0.1.md +0 -221
- package/docs/world_model//344/270/226/347/225/214/346/250/241/345/236/213_/347/245/236/347/273/217/347/275/221/347/273/234/345/272/225/345/261/202/346/236/266/346/236/204_v1.0.md +0 -237
- package/docs//344/270/215/345/217/257/351/235/240/346/200/247/347/220/206/350/256/272_v0.1.md +0 -343
- package/docs//345/217/221/345/270/203/344/273/266/345/233/236/346/272/257/350/257/264/346/230/216_v0.1.md +0 -82
- package/docs//345/255/230/347/256/227/344/270/200/344/275/223/346/236/266/346/236/204/345/255/246/344/271/240_/345/244/226/351/203/250/347/220/206/350/256/272/345/257/271/347/205/247/344/270/216/350/267/257/347/272/277/344/272/244/346/216/245_20260916.md +0 -98
- package/docs//345/255/230/347/256/227/344/270/200/344/275/223/347/245/236/347/273/217/347/275/221/347/273/234/346/236/266/346/236/204_/346/200/273/347/272/262/344/270/216/344/272/244/346/216/245_20260916.md +0 -109
- package/docs//347/201/265/346/236/242/350/207/252/346/210/221/346/224/271/350/277/233/345/267/245/344/275/234/350/256/241/345/210/222_/345/244/226/351/203/250/347/240/224/347/251/266/347/263/273/345/210/227/345/220/270/346/224/266_v1_20260919.md +0 -286
- package/docs//347/231/275/347/256/261/345/214/226/347/272/262/351/242/206_/344/270/211/351/241/271/347/233/256/345/255/246/344/271/240/346/200/273/346/236/266/346/236/204/344/270/216Ghidra/344/272/244/346/216/245_20260916.md +0 -104
- package/docs//347/247/244_/350/256/260/345/277/206/347/263/273/347/273/237/350/257/204/346/265/213/350/247/204/350/214/203_v2.1.md +0 -225
- package/docs//350/256/260/345/277/206/347/263/273/347/273/237/345/215/263/347/231/275/347/256/261/345/214/226/346/231/272/350/203/275_/346/236/266/346/236/204/345/257/271/347/205/247_v0.2.md +0 -121
- package/docs//350/256/260/345/277/206/347/263/273/347/273/237/345/215/263/347/231/275/347/256/261/345/214/226/346/231/272/350/203/275_/346/236/266/346/236/204/345/257/271/347/205/247_v0.3.md +0 -117
- package/docs//350/256/260/345/277/206/350/257/204/345/256/241/347/263/273/347/273/237_/347/253/213/351/241/271/350/256/276/350/256/241/344/270/216/346/226/275/345/267/245/344/272/244/346/216/245_20260915.md +0 -183
package/md_cg/sources.py
CHANGED
|
@@ -1,834 +1,1092 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
|
-
"""md_cg · 设备驱动(记忆 OS #3):把外部会话流接进认知图
|
|
3
|
-
|
|
4
|
-
设计:
|
|
5
|
-
Source(事件源)—— 把某种外部存储解析成统一事件流:
|
|
6
|
-
{"t": epoch 秒, "seq": 序号, "role": ..., "text": ..., "session": ..., "cwd": ...}
|
|
7
|
-
|
|
8
|
-
⚠ **单位纪律(issue #23)**:`t` 一律 **epoch 秒**,与图内 `created_at` /
|
|
9
|
-
`condition_space.time_window` / `trust.parse_time` 同口径。外部源里的 13 位毫秒
|
|
10
|
-
(DSH 的 `time` 字段、通用 JSONL 的毫秒戳)在**源适配层**经 `trust.epoch_seconds`
|
|
11
|
-
归一后即不外溢——否则 `condition_space.time_window` 会落毫秒,把
|
|
12
|
-
`stg(op=timeline)` 的倒序头部整片占满并顶掉 auto-recall。
|
|
13
|
-
|
|
14
|
-
Ingestor(摄取器)—— 增量 watermark + 去重 + 写节点 + 自动 fix-pair 挖掘。
|
|
15
|
-
|
|
16
|
-
内置源:
|
|
17
|
-
· JsonlSource —— 通用 JSONL(每行一个事件对象)
|
|
18
|
-
· DSHSessionSource —— DeepSeek Harness 会话(~/.dsh/sessions/**/session.jsonl[.zstd])
|
|
19
|
-
zstd 为**可选**依赖:缺失时优雅降级(跳过 .zstd 文件并告警)
|
|
20
|
-
|
|
21
|
-
默认敏感度:会话内容是私有记忆 → sensitivity="private"(避免落入公开根)。
|
|
22
|
-
|
|
23
|
-
零第三方依赖(zstd 为可选增强)。
|
|
24
|
-
"""
|
|
25
|
-
from __future__ import annotations
|
|
26
|
-
|
|
27
|
-
import glob
|
|
28
|
-
import json
|
|
29
|
-
import os
|
|
30
|
-
import time
|
|
31
|
-
|
|
32
|
-
from . import trust
|
|
33
|
-
from .fsutil import publish
|
|
34
|
-
from .security import DEFAULT_SENSITIVITY
|
|
35
|
-
|
|
36
|
-
# 会话事件的默认落层与敏感度
|
|
37
|
-
SESSION_LAYER = "contextual"
|
|
38
|
-
SESSION_SENSITIVITY = "private"
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
# --------------------------------------------------------------------------
|
|
42
|
-
# 事件源
|
|
43
|
-
# --------------------------------------------------------------------------
|
|
44
|
-
|
|
45
|
-
# 生效条件:无必填构造形参,类常量 name="source" 即实例默认;仅当子类覆写 events() 时才产出事件,基类 events() 恒抛 NotImplementedError。
|
|
46
|
-
class Source:
|
|
47
|
-
"""事件源基类。"""
|
|
48
|
-
|
|
49
|
-
name = "source"
|
|
50
|
-
|
|
51
|
-
#
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
#
|
|
84
|
-
def
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
"""
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
""
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
""
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
# 生效条件:
|
|
198
|
-
def
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
ev["
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
def
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
def
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
seq
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
#
|
|
594
|
-
#
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
#
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
#
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
#
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
#
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
#
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
return
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
#
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
if
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""md_cg · 设备驱动(记忆 OS #3):把外部会话流接进认知图
|
|
3
|
+
|
|
4
|
+
设计:
|
|
5
|
+
Source(事件源)—— 把某种外部存储解析成统一事件流:
|
|
6
|
+
{"t": epoch 秒, "seq": 序号, "role": ..., "text": ..., "session": ..., "cwd": ...}
|
|
7
|
+
|
|
8
|
+
⚠ **单位纪律(issue #23)**:`t` 一律 **epoch 秒**,与图内 `created_at` /
|
|
9
|
+
`condition_space.time_window` / `trust.parse_time` 同口径。外部源里的 13 位毫秒
|
|
10
|
+
(DSH 的 `time` 字段、通用 JSONL 的毫秒戳)在**源适配层**经 `trust.epoch_seconds`
|
|
11
|
+
归一后即不外溢——否则 `condition_space.time_window` 会落毫秒,把
|
|
12
|
+
`stg(op=timeline)` 的倒序头部整片占满并顶掉 auto-recall。
|
|
13
|
+
|
|
14
|
+
Ingestor(摄取器)—— 增量 watermark + 去重 + 写节点 + 自动 fix-pair 挖掘。
|
|
15
|
+
|
|
16
|
+
内置源:
|
|
17
|
+
· JsonlSource —— 通用 JSONL(每行一个事件对象)
|
|
18
|
+
· DSHSessionSource —— DeepSeek Harness 会话(~/.dsh/sessions/**/session.jsonl[.zstd])
|
|
19
|
+
zstd 为**可选**依赖:缺失时优雅降级(跳过 .zstd 文件并告警)
|
|
20
|
+
|
|
21
|
+
默认敏感度:会话内容是私有记忆 → sensitivity="private"(避免落入公开根)。
|
|
22
|
+
|
|
23
|
+
零第三方依赖(zstd 为可选增强)。
|
|
24
|
+
"""
|
|
25
|
+
from __future__ import annotations
|
|
26
|
+
|
|
27
|
+
import glob
|
|
28
|
+
import json
|
|
29
|
+
import os
|
|
30
|
+
import time
|
|
31
|
+
|
|
32
|
+
from . import trust
|
|
33
|
+
from .fsutil import publish
|
|
34
|
+
from .security import DEFAULT_SENSITIVITY
|
|
35
|
+
|
|
36
|
+
# 会话事件的默认落层与敏感度
|
|
37
|
+
SESSION_LAYER = "contextual"
|
|
38
|
+
SESSION_SENSITIVITY = "private"
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
# --------------------------------------------------------------------------
|
|
42
|
+
# 事件源
|
|
43
|
+
# --------------------------------------------------------------------------
|
|
44
|
+
|
|
45
|
+
# 生效条件:无必填构造形参,类常量 name="source" 即实例默认;仅当子类覆写 events() 时才产出事件,基类 events() 恒抛 NotImplementedError。
|
|
46
|
+
class Source:
|
|
47
|
+
"""事件源基类。"""
|
|
48
|
+
|
|
49
|
+
name = "source"
|
|
50
|
+
|
|
51
|
+
#: #466:**来源声明面**(本件新增)。外部内容经 ingest 落库时,基底/密级取自来源声明,
|
|
52
|
+
#: 不再由写入点硬编码——「同一份外部文本走不同链得到不同基底」的根因就是硬编码。
|
|
53
|
+
#: 基底缺省 `untrusted`:在 `nodefile.VERIFICATION_BASIS`(**可接受集**)内,
|
|
54
|
+
#: 但**不在** `forgetting.VERIFIED_BASES`(**已验**集)里,也不在理科/文科两档内——
|
|
55
|
+
#: 外部内容本就不应作为可信基底进入权威记忆。
|
|
56
|
+
#: 密级缺省 `internal`(**本件不降级**:降 public 会放宽外带面,与本件方向相反)。
|
|
57
|
+
default_basis = "untrusted"
|
|
58
|
+
default_sensitivity = "internal"
|
|
59
|
+
|
|
60
|
+
# 生效条件:任何调用都直接 raise NotImplementedError(基类占位,无其它分支)。
|
|
61
|
+
def events(self):
|
|
62
|
+
raise NotImplementedError
|
|
63
|
+
|
|
64
|
+
# 生效条件:无前置;返回类常量 self.name(基类为 "source"),作为 watermark 的稳定标识,不含路径与运行期状态;
|
|
65
|
+
def key(self):
|
|
66
|
+
"""源的稳定标识(用于 watermark)。"""
|
|
67
|
+
return self.name
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
# 生效条件:required 形参 path 总被存入 self.path;可选形参 name 为假值(None/空串)时 self.name 回落到 "jsonl:"+os.path.basename(path),为真值时用 name 本身,t_key/role_key/text_key/default_role 原样存入属性。
|
|
71
|
+
class JsonlSource(Source):
|
|
72
|
+
"""通用 JSONL 会话源。
|
|
73
|
+
|
|
74
|
+
每行是事件对象;字段映射可配置:
|
|
75
|
+
t_key —— 时间戳字段(默认 "time",也接受 ISO 字符串)
|
|
76
|
+
role_key —— 角色字段(默认 "role")
|
|
77
|
+
text_key —— 文本字段(默认 "text")
|
|
78
|
+
"""
|
|
79
|
+
|
|
80
|
+
default_basis = "untrusted" # #466:外部 JSONL 会话 ⇒ 未验基底
|
|
81
|
+
default_sensitivity = "internal"
|
|
82
|
+
|
|
83
|
+
# 生效条件:传入 path;name 为假值(None/空串)时回落为 "jsonl:"+os.path.basename(path),t_key/role_key/text_key/default_role 原样存为属性(默认值 "time"/"role"/"text"/"user")。
|
|
84
|
+
def __init__(self, path: str, name: str = None, t_key="time", role_key="role",
|
|
85
|
+
text_key="text", default_role="user"):
|
|
86
|
+
self.path = path
|
|
87
|
+
self.name = name or ("jsonl:" + os.path.basename(path))
|
|
88
|
+
self.t_key, self.role_key, self.text_key = t_key, role_key, text_key
|
|
89
|
+
self.default_role = default_role
|
|
90
|
+
|
|
91
|
+
# 生效条件:无前置;返回 self.name(构造时已回落为 "jsonl:"+basename(path)),只随构造参数变化、不随文件内容变化;
|
|
92
|
+
def key(self):
|
|
93
|
+
return self.name
|
|
94
|
+
|
|
95
|
+
# 生效条件:o.get(self.t_key) 经 float 可转数值时返回 trust.epoch_seconds(该值)(秒值原样、13 位毫秒 /1000 归一到**秒**);float 抛 TypeError/ValueError 且 v 为字符串时按 v[:19] 与 "%Y-%m-%dT%H:%M:%S" 解析,成功返回 mktime(**秒**)、抛 ValueError 返回 0.0;键缺失/None/其它不可转类型返回 0.0。
|
|
96
|
+
def _ts(self, o):
|
|
97
|
+
v = o.get(self.t_key)
|
|
98
|
+
try:
|
|
99
|
+
return trust.epoch_seconds(float(v)) or 0.0
|
|
100
|
+
except (TypeError, ValueError):
|
|
101
|
+
pass
|
|
102
|
+
if isinstance(v, str):
|
|
103
|
+
try:
|
|
104
|
+
return time.mktime(time.strptime(v[:19], "%Y-%m-%dT%H:%M:%S"))
|
|
105
|
+
except ValueError:
|
|
106
|
+
return 0.0
|
|
107
|
+
return 0.0
|
|
108
|
+
|
|
109
|
+
# 生效条件:os.path.exists(self.path) 为真时逐行产出,空行、json.loads 抛 ValueError、o.get(self.text_key) 为假的行被跳过,产出项为 t=self._ts(o)、seq=o.get("seq", i)、role=o.get(self.role_key) or self.default_role、text=str(text)、session/cwd 取 o 同名键;path 不存在时直接 return 不产出。
|
|
110
|
+
def events(self):
|
|
111
|
+
if not os.path.exists(self.path):
|
|
112
|
+
return
|
|
113
|
+
with open(self.path, encoding="utf-8", errors="replace") as f:
|
|
114
|
+
for i, line in enumerate(f):
|
|
115
|
+
line = line.strip()
|
|
116
|
+
if not line:
|
|
117
|
+
continue
|
|
118
|
+
try:
|
|
119
|
+
o = json.loads(line)
|
|
120
|
+
except ValueError:
|
|
121
|
+
continue
|
|
122
|
+
text = o.get(self.text_key)
|
|
123
|
+
if not text:
|
|
124
|
+
continue
|
|
125
|
+
yield {"t": self._ts(o), "seq": o.get("seq", i),
|
|
126
|
+
"role": o.get(self.role_key) or self.default_role,
|
|
127
|
+
"text": str(text), "session": o.get("session"),
|
|
128
|
+
"cwd": o.get("cwd")}
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
# 生效条件:path 指向的内容可读且 zstandard 可导入时返回 StringIO(raw.decode("utf-8", errors="replace"));ImportError 时返回 None。
|
|
132
|
+
def _zstd_reader(path):
|
|
133
|
+
"""返回可读的文本迭代器;zstd 不可用返回 None。"""
|
|
134
|
+
import io
|
|
135
|
+
try:
|
|
136
|
+
import zstandard as zstd
|
|
137
|
+
except ImportError:
|
|
138
|
+
return None
|
|
139
|
+
with open(path, "rb") as f:
|
|
140
|
+
raw = zstd.ZstdDecompressor().stream_reader(f).read()
|
|
141
|
+
return io.StringIO(raw.decode("utf-8", errors="replace"))
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
# 生效条件:required 形参 path 总被存入 self.path,可选形参 include_reasoning 原样存入,self.name 恒为 "dsh:"+os.path.basename(os.path.dirname(path))(与 include_reasoning 取值无关)。
|
|
145
|
+
class DSHSessionSource(Source):
|
|
146
|
+
"""DeepSeek Harness 会话源。
|
|
147
|
+
|
|
148
|
+
事件映射:
|
|
149
|
+
user/message → role=user
|
|
150
|
+
assistant/message → role=assistant(只取 content[].text,reasoning 默认丢弃)
|
|
151
|
+
tool/call → role=command("name(args)" 形式)
|
|
152
|
+
tool/result → role=tool-output
|
|
153
|
+
"""
|
|
154
|
+
|
|
155
|
+
default_basis = "untrusted" # #466:外部会话转录 ⇒ 未验基底
|
|
156
|
+
default_sensitivity = "internal"
|
|
157
|
+
|
|
158
|
+
# 生效条件:传入 path 即成立,include_reasoning 原样存为属性(默认 False),name 固定为 "dsh:"+os.path.basename(os.path.dirname(path))。
|
|
159
|
+
def __init__(self, path: str, include_reasoning: bool = False):
|
|
160
|
+
self.path = path
|
|
161
|
+
self.include_reasoning = include_reasoning
|
|
162
|
+
self.name = "dsh:" + os.path.basename(os.path.dirname(path))
|
|
163
|
+
|
|
164
|
+
# 生效条件:无前置;返回 self.name(构造时固定为 "dsh:"+basename(dirname(path))),与 include_reasoning 取值无关;
|
|
165
|
+
def key(self):
|
|
166
|
+
return self.name
|
|
167
|
+
|
|
168
|
+
@staticmethod
|
|
169
|
+
# 生效条件:可选形参 root 为假值(None/空串)时改用默认目录 os.path.join(expanduser("~"),".dsh","sessions"),否则用传入 root;对 root 下递归 glob 到的 session.jsonl 与 session.jsonl.zstd 逐条 os.stat(抛 OSError 的条目跳过),按 (-st_size, path) 升序键排序(两处 glob 均无命中时为空列表),可选形参 limit 为假值(None/0)时返回全部 rows,否则返回 rows[:limit];每行为 (path, size, mtime) 三元组。
|
|
170
|
+
def discover_detailed(root: str = None, limit: int = None):
|
|
171
|
+
"""候选会话 → `[(path, size, mtime), ...]`(**排序唯一真源**,`discover` 复用)。
|
|
172
|
+
|
|
173
|
+
排序:会话**文件体积**降序 → `path` 升序(确定性终键)。
|
|
174
|
+
**刻意不按最近活动排序**——这正是 `source='auto'` 可能选中很久以前会话的
|
|
175
|
+
原因;把 size/mtime 一并透出,让「为什么选它」在返回体里可辨
|
|
176
|
+
(issue #23 附带建议:先让依据可见,是否换策略另议)。
|
|
177
|
+
"""
|
|
178
|
+
root = root or os.path.join(os.path.expanduser("~"), ".dsh", "sessions")
|
|
179
|
+
files = glob.glob(os.path.join(root, "**", "session.jsonl"), recursive=True)
|
|
180
|
+
files += glob.glob(os.path.join(root, "**", "session.jsonl.zstd"), recursive=True)
|
|
181
|
+
rows = []
|
|
182
|
+
for p in files:
|
|
183
|
+
try:
|
|
184
|
+
st = os.stat(p)
|
|
185
|
+
except OSError:
|
|
186
|
+
continue # 竞态删除:跳过而非整体失败
|
|
187
|
+
rows.append((p, st.st_size, st.st_mtime))
|
|
188
|
+
rows.sort(key=lambda r: (-r[1], r[0]))
|
|
189
|
+
return rows[:limit] if limit else rows
|
|
190
|
+
|
|
191
|
+
@staticmethod
|
|
192
|
+
# 生效条件:可选形参 root/limit 原样转交 discover_detailed,返回其结果的 path 列(同为体积降序、同确定性终键),limit 为假值时返回全部。
|
|
193
|
+
def discover(root: str = None, limit: int = None):
|
|
194
|
+
"""候选会话路径(体积降序)——`discover_detailed` 的路径投影。"""
|
|
195
|
+
return [p for p, _s, _m in DSHSessionSource.discover_detailed(root, limit)]
|
|
196
|
+
|
|
197
|
+
# 生效条件:self.path 以 ".zstd" 结尾时经 _zstd_reader 逐行产出(其返回 None 时 raise RuntimeError),否则以 utf-8/errors=replace 打开 self.path 逐行产出。
|
|
198
|
+
def _lines(self):
|
|
199
|
+
if self.path.endswith(".zstd"):
|
|
200
|
+
fh = _zstd_reader(self.path)
|
|
201
|
+
if fh is None:
|
|
202
|
+
raise RuntimeError("zstd 不可用(pip install zstandard),跳过该会话")
|
|
203
|
+
try:
|
|
204
|
+
yield from fh
|
|
205
|
+
finally:
|
|
206
|
+
fh.close()
|
|
207
|
+
else:
|
|
208
|
+
with open(self.path, encoding="utf-8", errors="replace") as f:
|
|
209
|
+
yield from f
|
|
210
|
+
|
|
211
|
+
@staticmethod
|
|
212
|
+
# 生效条件:required 形参 content 为 str 时原样返回 content;为 list 时收集其中 str 元素及 type 属于 ("text","input-text") 且 text 为真值的 dict 元素(取 str(c["text"])),以 "\n" 连接返回(无可收集元素时为空串 "");既非 str 也非 list 时返回 ""。
|
|
213
|
+
def _text_of(content):
|
|
214
|
+
"""content 可能是 [{type,text}] 或字符串。"""
|
|
215
|
+
if isinstance(content, str):
|
|
216
|
+
return content
|
|
217
|
+
if isinstance(content, list):
|
|
218
|
+
parts = []
|
|
219
|
+
for c in content:
|
|
220
|
+
if isinstance(c, dict):
|
|
221
|
+
if c.get("type") in ("text", "input-text") and c.get("text"):
|
|
222
|
+
parts.append(str(c["text"]))
|
|
223
|
+
elif isinstance(c, str):
|
|
224
|
+
parts.append(c)
|
|
225
|
+
return "\n".join(parts)
|
|
226
|
+
return ""
|
|
227
|
+
|
|
228
|
+
# 生效条件:逐行解析后按 o.get("type") 分派——"session" 只更新 sess/cwd 不产出;"user/message" 产出 role=user 与 _text_of(data.get("content"));"assistant/message" 产出 role=assistant 与 _text_of(msg.get("content")),include_reasoning 为真时再把 content 中 type=="reasoning" 且有 text 的项追加 "\n[reasoning] "+str(c["text"]);"tool/call" 产出 role=command 与 f"{name}({arguments or ''})";"tool/result" 产出 role=tool-output,仅当 inner[0] 为 dict 时 text=_text_of(inner[0].get("content"));其它 type 或 ev 中 text 为空的事件不产出。
|
|
229
|
+
def events(self):
|
|
230
|
+
sess = None
|
|
231
|
+
cwd = None
|
|
232
|
+
for line in self._lines():
|
|
233
|
+
line = line.strip()
|
|
234
|
+
if not line:
|
|
235
|
+
continue
|
|
236
|
+
try:
|
|
237
|
+
o = json.loads(line)
|
|
238
|
+
except ValueError:
|
|
239
|
+
continue
|
|
240
|
+
t = o.get("type")
|
|
241
|
+
data = o.get("data") or {}
|
|
242
|
+
if t == "session":
|
|
243
|
+
sess, cwd = o.get("id"), o.get("cwd")
|
|
244
|
+
continue
|
|
245
|
+
ev = {"t": trust.epoch_seconds(float(o.get("time") or 0)) or 0.0,
|
|
246
|
+
"seq": o.get("seq"),
|
|
247
|
+
"session": sess, "cwd": cwd}
|
|
248
|
+
if t == "user/message":
|
|
249
|
+
ev["role"], ev["text"] = "user", self._text_of(data.get("content"))
|
|
250
|
+
elif t == "assistant/message":
|
|
251
|
+
msg = data.get("message") or {}
|
|
252
|
+
ev["role"] = "assistant"
|
|
253
|
+
ev["text"] = self._text_of(msg.get("content"))
|
|
254
|
+
if self.include_reasoning:
|
|
255
|
+
for c in (msg.get("content") or []):
|
|
256
|
+
if isinstance(c, dict) and c.get("type") == "reasoning" \
|
|
257
|
+
and c.get("text"):
|
|
258
|
+
ev["text"] = (ev["text"] or "") + "\n[reasoning] " + str(c["text"])
|
|
259
|
+
elif t == "tool/call":
|
|
260
|
+
ev["role"] = "command"
|
|
261
|
+
ev["text"] = f"{data.get('name')}({data.get('arguments') or ''})"
|
|
262
|
+
elif t == "tool/result":
|
|
263
|
+
msg = data.get("message") or {}
|
|
264
|
+
inner = msg.get("content") or []
|
|
265
|
+
txt = ""
|
|
266
|
+
if inner and isinstance(inner[0], dict):
|
|
267
|
+
txt = self._text_of(inner[0].get("content"))
|
|
268
|
+
ev["role"], ev["text"] = "tool-output", txt
|
|
269
|
+
else:
|
|
270
|
+
continue
|
|
271
|
+
if ev.get("text"):
|
|
272
|
+
yield ev
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
# --------------------------------------------------------------------------
|
|
276
|
+
# 蜂巢任务事件源(M6:hive/jobs → contextual,外部设计稿 蜂巢记忆架构设计.md §5)
|
|
277
|
+
# --------------------------------------------------------------------------
|
|
278
|
+
|
|
279
|
+
class HiveJobsSource(Source):
|
|
280
|
+
"""蜂巢任务事件源。
|
|
281
|
+
|
|
282
|
+
事件映射(设计稿 §5.3):
|
|
283
|
+
start → role=user,任务开始(task 摘要)
|
|
284
|
+
tool → **跳过**(防流水账爆炸;tool_trace 已在 result 有摘要)
|
|
285
|
+
handoff → role=assistant,续跑卡
|
|
286
|
+
error → role=assistant,「任务失败(job=…):…」(fix-pair 前件)
|
|
287
|
+
final → role=assistant,content 头
|
|
288
|
+
result.json 终态 → **权威确认事件**(progress 可能因强杀缺失 final):
|
|
289
|
+
done → 「任务完成(job=…)」;error/timeout/killed → 「任务失败(job=…)」
|
|
290
|
+
|
|
291
|
+
排序:**job_id 名升序**(`sorted(listdir)`,确定性),**不是**时间序——
|
|
292
|
+
契约 v2(`docs/plans/全中文编码与蜂巢任务标识契约_v2.0.md` §四.5)起 id 改为
|
|
293
|
+
语义四槽 `h_<身份>_<任务>_<单元>_<编号>`,名序不再等于提交时序:旧形态
|
|
294
|
+
`h<unix_ms>_<pid>` 才是「名序 = 时间序」的**巧合代理**。据实订正(C2:本仓
|
|
295
|
+
禁止把代理当结构保证)——本源的时序由来源行自带的 ts 承载,排序只求
|
|
296
|
+
**跨 job 全序且确定**(seq 由本源按枚举顺序递增,ingest 去重键 = (session, seq),
|
|
297
|
+
名序的确定性保证该键跨轮稳定)。
|
|
298
|
+
诚实边界(**已发现、未在本批动码**):本源的 `_jobs()` 若需要**真时间序**,
|
|
299
|
+
应改走与 Rust 侧同判据的 created_ts 真值(`hive/src/job.rs::list_jobs_by_created`
|
|
300
|
+
的语义,Python 侧对应 `mcp_server._list_jobs_by_created`)——契约 §四.5 的消费者
|
|
301
|
+
清单未列本源,且改动牵动 md_cg 摄入链语义与其守卫,故留待单独裁决,不静默改。
|
|
302
|
+
解析失败的行/条目计入 self.skipped,不终杀批次。
|
|
303
|
+
"""
|
|
304
|
+
|
|
305
|
+
name = "hive_jobs"
|
|
306
|
+
|
|
307
|
+
#: #466:hive 任务面同为**外部**内容 ⇒ 基底缺省 untrusted;
|
|
308
|
+
#: 密级保持 internal **不变**(本件不带宽外带面),只是改为可声明。
|
|
309
|
+
default_basis = "untrusted"
|
|
310
|
+
default_sensitivity = "internal"
|
|
311
|
+
|
|
312
|
+
def __init__(self, root: str, error_sensitivity: str = "private"):
|
|
313
|
+
self.root = os.path.abspath(root)
|
|
314
|
+
self.skipped = 0
|
|
315
|
+
# §5.5「error:true 建议 private」——默认 private;调用方 clearance 不足时
|
|
316
|
+
# 可显式降级 error_sensitivity="internal"(显式权衡:接受失败细节入 internal,
|
|
317
|
+
# 换取误差归因原料不丢)。能否落盘由写入者 clearance 裁决(写隔离)。
|
|
318
|
+
self.error_sensitivity = error_sensitivity
|
|
319
|
+
|
|
320
|
+
def key(self):
|
|
321
|
+
"""多仓/多池隔离:root 参与键(watermark 按源路径各自推进)。"""
|
|
322
|
+
return f"hive_jobs:{self.root}"
|
|
323
|
+
|
|
324
|
+
def _jobs(self):
|
|
325
|
+
try:
|
|
326
|
+
return sorted(d for d in os.listdir(self.root)
|
|
327
|
+
if d.startswith("h")
|
|
328
|
+
and os.path.isdir(os.path.join(self.root, d)))
|
|
329
|
+
except OSError:
|
|
330
|
+
return []
|
|
331
|
+
|
|
332
|
+
def _read_json(self, path):
|
|
333
|
+
try:
|
|
334
|
+
with open(path, encoding="utf-8") as f:
|
|
335
|
+
return json.load(f)
|
|
336
|
+
except (OSError, ValueError):
|
|
337
|
+
self.skipped += 1
|
|
338
|
+
return None
|
|
339
|
+
|
|
340
|
+
def events(self):
|
|
341
|
+
seq = 0
|
|
342
|
+
for job_id in self._jobs():
|
|
343
|
+
jdir = os.path.join(self.root, job_id)
|
|
344
|
+
sess = f"hive:{job_id}"
|
|
345
|
+
spec = self._read_json(os.path.join(jdir, "spec.json")) or {}
|
|
346
|
+
task = str(spec.get("user_prompt") or "")[:200]
|
|
347
|
+
|
|
348
|
+
has_final = False
|
|
349
|
+
prog = os.path.join(jdir, "progress.jsonl")
|
|
350
|
+
if os.path.isfile(prog):
|
|
351
|
+
try:
|
|
352
|
+
fh = open(prog, encoding="utf-8", errors="replace")
|
|
353
|
+
except OSError:
|
|
354
|
+
fh = None
|
|
355
|
+
if fh is not None:
|
|
356
|
+
with fh:
|
|
357
|
+
for line in fh:
|
|
358
|
+
line = line.strip()
|
|
359
|
+
if not line:
|
|
360
|
+
continue
|
|
361
|
+
try:
|
|
362
|
+
o = json.loads(line)
|
|
363
|
+
except ValueError:
|
|
364
|
+
self.skipped += 1
|
|
365
|
+
continue
|
|
366
|
+
kind = o.get("kind")
|
|
367
|
+
t = trust.epoch_seconds(float(o.get("ts") or 0)) or 0.0
|
|
368
|
+
if kind == "start":
|
|
369
|
+
seq += 1
|
|
370
|
+
yield {"t": t, "seq": seq, "session": sess,
|
|
371
|
+
"role": "user",
|
|
372
|
+
"text": f"任务开始(job={job_id}):{task}"}
|
|
373
|
+
elif kind == "handoff":
|
|
374
|
+
seq += 1
|
|
375
|
+
has_final = True
|
|
376
|
+
yield {"t": t, "seq": seq, "session": sess,
|
|
377
|
+
"role": "assistant",
|
|
378
|
+
"text": f"续跑卡(job={job_id}):"
|
|
379
|
+
f"{str(o.get('summary') or '')[:400]}"}
|
|
380
|
+
elif kind == "error":
|
|
381
|
+
seq += 1
|
|
382
|
+
ev = {"t": t, "seq": seq, "session": sess,
|
|
383
|
+
"role": "assistant",
|
|
384
|
+
"text": f"任务失败(job={job_id}):"
|
|
385
|
+
f"{str(o.get('error') or '')[:300]}"}
|
|
386
|
+
if self.error_sensitivity:
|
|
387
|
+
ev["sensitivity"] = self.error_sensitivity
|
|
388
|
+
yield ev
|
|
389
|
+
elif kind == "final":
|
|
390
|
+
seq += 1
|
|
391
|
+
has_final = True
|
|
392
|
+
yield {"t": t, "seq": seq, "session": sess,
|
|
393
|
+
"role": "assistant",
|
|
394
|
+
"text": str(o.get("content_head") or "")[:400]}
|
|
395
|
+
# tool / budget_stop / force_final / 其它 → 纯跳过
|
|
396
|
+
# (§5.3 原设计为聚合计数,实现从简——终态由 result.json
|
|
397
|
+
# 权威确认承载,无需逐条累计;防流水账爆炸目标不变。
|
|
398
|
+
# 注释曾照抄设计稿口径称「仅聚合计数」,与实现漂移,
|
|
399
|
+
# zcode 外评抓出,2026-09-23 修正)
|
|
400
|
+
|
|
401
|
+
# result.json 终态权威确认(progress 可能因强杀缺失 final)
|
|
402
|
+
res = self._read_json(os.path.join(jdir, "result.json")) or {}
|
|
403
|
+
state = "done" if res.get("ok") is True else \
|
|
404
|
+
("error" if res.get("ok") is False else None)
|
|
405
|
+
if state is None:
|
|
406
|
+
continue
|
|
407
|
+
t = trust.epoch_seconds(float(res.get("finished_ts") or 0))
|
|
408
|
+
if not t:
|
|
409
|
+
# 兜底(批次8 实测缺陷):exec_cmd 旧版 result 无 finished_ts,
|
|
410
|
+
# t=0 会被水位整片过滤(确定性任务事件永远摄不进来)——
|
|
411
|
+
# 回落 result.json 的 mtime(产物诞生时间,同「产物说了算」族)
|
|
412
|
+
rpath = os.path.join(jdir, "result.json")
|
|
413
|
+
t = trust.epoch_seconds(os.path.getmtime(rpath)) if \
|
|
414
|
+
os.path.isfile(rpath) else 0.0
|
|
415
|
+
t = t or 0.0
|
|
416
|
+
head = str(res.get("content") or "")[:300]
|
|
417
|
+
err = str(res.get("error") or "")[:300]
|
|
418
|
+
seq += 1
|
|
419
|
+
if state == "done":
|
|
420
|
+
# 前缀独立成行:content 的行首结构(命令行等)不被破坏,
|
|
421
|
+
# 保证 mine_fix_pairs 的 _FIX_RE 行首启发式仍能命中修复证据
|
|
422
|
+
text = f"任务完成(job={job_id}):\n{head}"
|
|
423
|
+
# 重放命令行(批次8):cmd 任务(exec_cmd)的 result.steps 携带
|
|
424
|
+
# 原始 argv——附到事件正文,事件自身携带「做了什么」的可复放
|
|
425
|
+
# 信息,同时命令形态可被 _FIX_RE 行首启发式命中(fix-pair 原料)
|
|
426
|
+
steps = res.get("steps")
|
|
427
|
+
if isinstance(steps, list) and steps:
|
|
428
|
+
argv = (steps[-1] or {}).get("command") or []
|
|
429
|
+
if argv:
|
|
430
|
+
text += "\n" + " ".join(str(a) for a in argv)
|
|
431
|
+
yield {"t": t, "seq": seq, "session": sess, "role": "assistant",
|
|
432
|
+
"text": text}
|
|
433
|
+
else:
|
|
434
|
+
tag = "超时强杀" if st_err_is_timeout(err) else "失败"
|
|
435
|
+
ev = {"t": t, "seq": seq, "session": sess, "role": "assistant",
|
|
436
|
+
"text": f"任务{tag}(job={job_id}):{err or head}"
|
|
437
|
+
+ ("" if has_final else "(progress 缺 final,"
|
|
438
|
+
"本条为 result 终态补位)")}
|
|
439
|
+
if self.error_sensitivity:
|
|
440
|
+
ev["sensitivity"] = self.error_sensitivity
|
|
441
|
+
yield ev
|
|
442
|
+
|
|
443
|
+
|
|
444
|
+
def st_err_is_timeout(err: str) -> bool:
|
|
445
|
+
return "超时" in (err or "")
|
|
446
|
+
|
|
447
|
+
|
|
448
|
+
# --------------------------------------------------------------------------
|
|
449
|
+
# 摄取器
|
|
450
|
+
# --------------------------------------------------------------------------
|
|
451
|
+
|
|
452
|
+
# 生效条件:required 形参 cg 总被存入并以其 cg.root 拼出 self.path=os.path.join(cg.root,"_sources.json");layer/sensitivity 原样存入,未传时取模块级常量 SESSION_LAYER、SESSION_SENSITIVITY 作为默认值。
|
|
453
|
+
class Ingestor:
|
|
454
|
+
"""增量摄取:watermark + 去重 + 写节点 + 自动 fix-pair 挖掘。
|
|
455
|
+
|
|
456
|
+
watermark 文件:<root>/_sources.json
|
|
457
|
+
{source_key: {"t": 最后时间戳(epoch 秒), "seq": 最后序号, "count": 已摄取条数}}
|
|
458
|
+
|
|
459
|
+
存量水位兼容:**旧版写入的是毫秒**,读回时经 `trust.epoch_seconds` 归一到秒,
|
|
460
|
+
故升级后无需清 `_sources.json` 重建(否则 `ev["t"] < last_t` 恒真 → 新事件全被跳过)。
|
|
461
|
+
"""
|
|
462
|
+
|
|
463
|
+
# 生效条件:传入带 root 的 cg 即成立,layer/sensitivity 默认 SESSION_LAYER/SESSION_SENSITIVITY 并原样存为属性,路径为 os.path.join(cg.root, "_sources.json")。
|
|
464
|
+
def __init__(self, cg, layer: str = SESSION_LAYER,
|
|
465
|
+
sensitivity: str = SESSION_SENSITIVITY):
|
|
466
|
+
self.cg = cg
|
|
467
|
+
self.layer = layer
|
|
468
|
+
self.sensitivity = sensitivity
|
|
469
|
+
self.path = os.path.join(cg.root, "_sources.json")
|
|
470
|
+
|
|
471
|
+
# ---- watermark ----
|
|
472
|
+
|
|
473
|
+
# 生效条件:self.path 存在、json.load 成功且结果为 dict 时返回该 dict;path 不存在、抛 ValueError/OSError 或结果非 dict 时返回 {"schema": 1, "sources": {}}。
|
|
474
|
+
def _load(self):
|
|
475
|
+
if os.path.exists(self.path):
|
|
476
|
+
try:
|
|
477
|
+
with open(self.path, encoding="utf-8") as f:
|
|
478
|
+
d = json.load(f)
|
|
479
|
+
if isinstance(d, dict):
|
|
480
|
+
return d
|
|
481
|
+
except (ValueError, OSError):
|
|
482
|
+
pass
|
|
483
|
+
return {"schema": 1, "sources": {}}
|
|
484
|
+
|
|
485
|
+
# 生效条件:传入 d 时以 ensure_ascii=False/indent=1 写入 self.path+".tmp",再 publish(带 Windows 短重试的 os.replace)覆盖 self.path。
|
|
486
|
+
def _save(self, d):
|
|
487
|
+
tmp = self.path + ".tmp"
|
|
488
|
+
with open(tmp, "w", encoding="utf-8") as f:
|
|
489
|
+
json.dump(d, f, ensure_ascii=False, indent=1)
|
|
490
|
+
publish(tmp, self.path)
|
|
491
|
+
|
|
492
|
+
# 生效条件:key 命中 self._load()["sources"] 时返回其值,缺 key 时返回 {}(_load 结果缺 "sources" 键则抛 KeyError)。
|
|
493
|
+
def watermark(self, key: str):
|
|
494
|
+
return self._load()["sources"].get(key, {})
|
|
495
|
+
|
|
496
|
+
# 生效条件:调用即返回 dict(self._load()["sources"]) 的浅拷贝(_load 结果缺 "sources" 键则抛 KeyError)。
|
|
497
|
+
def watermarks(self):
|
|
498
|
+
return dict(self._load()["sources"])
|
|
499
|
+
|
|
500
|
+
# ---- 摄取 ----
|
|
501
|
+
|
|
502
|
+
# 生效条件:对必需形参 source,先按 watermark(source.key()) 跳过 t<last_t(wm 的 "t" 为假值时视作 0)及 t==last_t 且 last_seq 非 None 且 ev["seq"] 为 int 且 <= last_seq 的事件、并按 (session, seq) 去重,max_events 为真值(非 0/None)时取满即停;dry_run 为假时逐条 cg.add(nid 已在 cg.index["nodes"] 中则跳过,cg.add 抛异常则 denied+=1、记 last_error 并继续),denied 与 last_error 同时成立时补 last_error/hint 与 denied_events 明细,mine_fix_pairs 为真且 new_events 非空且非 dry_run 时以 as_proposals=True 调 mine_fix_pairs(自动产物走审核队列)并结果附 fix_pairs,new_events 非空且非 dry_run 时以末事件写回水位(count 累加 written),最后返回 result。
|
|
503
|
+
def ingest(self, source, mine_fix_pairs: bool = False, max_events: int = None,
|
|
504
|
+
dry_run: bool = False):
|
|
505
|
+
"""摄取一个源的新事件。返回统计。
|
|
506
|
+
|
|
507
|
+
去重键:(session, seq) —— 同一事件不重复入库。
|
|
508
|
+
增量:只处理 (t, seq) 大于 watermark 的事件。
|
|
509
|
+
mine_fix_pairs 默认 False(先落账后挖矿——§5.5 系统纪律化,
|
|
510
|
+
zcode 外评 break#2:默认 True 曾使 file/jsonl/mdcg_ingest 三入口
|
|
511
|
+
绕过审核队列直写 knowledge 层);显式开启时产物走 propose 队列。
|
|
512
|
+
"""
|
|
513
|
+
key = source.key()
|
|
514
|
+
wm = self.watermark(key)
|
|
515
|
+
# 水位单位归一:**存量水位是毫秒**(旧版 `_ts` 产出),不归一会让
|
|
516
|
+
# `ev["t"] < last_t` 恒真 → 升级后新事件被整片误判为「已处理」而跳过。
|
|
517
|
+
last_t, last_seq = trust.epoch_seconds(float(wm.get("t") or 0)) or 0.0, \
|
|
518
|
+
wm.get("seq")
|
|
519
|
+
new_events, seen = [], set()
|
|
520
|
+
denied_events = [] # 被拒事件明细(session/seq/原因)——可观测可重放
|
|
521
|
+
for ev in source.events():
|
|
522
|
+
if ev.get("t", 0) < last_t:
|
|
523
|
+
continue
|
|
524
|
+
if ev.get("t", 0) == last_t and last_seq is not None \
|
|
525
|
+
and isinstance(ev.get("seq"), int) and ev["seq"] <= last_seq:
|
|
526
|
+
continue
|
|
527
|
+
dedup = (ev.get("session"), ev.get("seq"))
|
|
528
|
+
if dedup in seen:
|
|
529
|
+
continue
|
|
530
|
+
seen.add(dedup)
|
|
531
|
+
new_events.append(ev)
|
|
532
|
+
if max_events and len(new_events) >= max_events:
|
|
533
|
+
break
|
|
534
|
+
|
|
535
|
+
written, ids, denied = 0, [], 0
|
|
536
|
+
if not dry_run:
|
|
537
|
+
for ev in new_events:
|
|
538
|
+
nid = "src_%s_%s" % (_sig(key)[:6], _sig(
|
|
539
|
+
f"{ev.get('session')}:{ev.get('seq')}")[:10])
|
|
540
|
+
if nid in self.cg.index["nodes"]:
|
|
541
|
+
continue # 幂等
|
|
542
|
+
# #466(Lead 裁定):基底取自**来源声明**(事件级 basis 优先,否则源级
|
|
543
|
+
# default_basis)。取值主体是形参 `source`(= Source 实例)、**不是 `self`**:
|
|
544
|
+
# 本方法是 `Ingestor.ingest(self, source, …)`,Ingestor 只是**搬运者**、不该
|
|
545
|
+
# 自己声明基底。写 `self.default_basis` 会让 Ingestor 无此属性 ⇒ AttributeError,
|
|
546
|
+
# 且被下方 except Exception 吞成 denied 计数 ⇒ **外部内容静默不落库**
|
|
547
|
+
# (2026-10-15 回归事故根因)。
|
|
548
|
+
# ★两条通道**必须同源**:CCG 正文行的「验证方式:」与 fm 的
|
|
549
|
+
# verification_basis 共用**同一个 basis 变量**——正文声明可验、枚举说未验
|
|
550
|
+
# 会让同一节点自相矛盾(守卫 H01 钉这一点)。
|
|
551
|
+
basis = ev.get("basis") or source.default_basis
|
|
552
|
+
body = ("# 功能名:会话事件\n"
|
|
553
|
+
f"# 生效条件:检索「{str(ev.get('text'))[:20]}」\n"
|
|
554
|
+
"# 子功能:记录会话事件\n"
|
|
555
|
+
f"# 执行:{str(ev.get('text'))[:80]}\n"
|
|
556
|
+
f"# 验证方式:{basis}(会话原始记录)\n"
|
|
557
|
+
"# 不适用条件:其它会话\n\n"
|
|
558
|
+
f"{ev.get('text')}\n")
|
|
559
|
+
|
|
560
|
+
try:
|
|
561
|
+
self.cg.add(nid, body, layer=self.layer, role=ev.get("role"),
|
|
562
|
+
tags=["session", key],
|
|
563
|
+
# 事件级密级覆盖(M6 §5.5):error 事件建议 private
|
|
564
|
+
# (失败细节可能含路径/配置),缺省回落源级默认
|
|
565
|
+
sensitivity=ev.get("sensitivity") or self.sensitivity,
|
|
566
|
+
# #466:不再硬编码 "data"(外部内容不得自带权威基底)。
|
|
567
|
+
verification_basis=basis,
|
|
568
|
+
condition_space={"observation_position": key,
|
|
569
|
+
"observation_tool": "会话流",
|
|
570
|
+
"time_window": [ev.get("t") or 0,
|
|
571
|
+
ev.get("t") or 0]})
|
|
572
|
+
except Exception as exc: # noqa: BLE001 —— 权限/层错误不中断整批
|
|
573
|
+
denied += 1
|
|
574
|
+
# 诚实化(M6):denied 事件虽被水位跳过,但明细必须可见——
|
|
575
|
+
# 静默丢失会吞掉误差归因闭环的原料(error 事件恰是原料)
|
|
576
|
+
denied_events.append({
|
|
577
|
+
"session": ev.get("session"), "seq": ev.get("seq"),
|
|
578
|
+
"error": str(exc)[:150]})
|
|
579
|
+
self.last_error = f"{type(exc).__name__}: {exc}"
|
|
580
|
+
continue
|
|
581
|
+
ids.append(nid)
|
|
582
|
+
written += 1
|
|
583
|
+
|
|
584
|
+
result = {"source": key, "new_events": len(new_events), "written": written,
|
|
585
|
+
"denied": denied, "ids": ids, "dry_run": dry_run,
|
|
586
|
+
"sensitivity": self.sensitivity}
|
|
587
|
+
if denied_events:
|
|
588
|
+
result["denied_events"] = denied_events
|
|
589
|
+
if denied and getattr(self, "last_error", None):
|
|
590
|
+
result["last_error"] = self.last_error
|
|
591
|
+
result["hint"] = ("会话内容默认 sensitivity=private;"
|
|
592
|
+
"调用方需 MDCG_CLEARANCE=private 才能写入")
|
|
593
|
+
# 自动 fix-pair 挖掘(对标 deja-vu:错误→修复)——as_proposals=True:
|
|
594
|
+
# 自动管线产物走审核队列,绝不直写 knowledge 层(§5.5 系统纪律)
|
|
595
|
+
if mine_fix_pairs and new_events and not dry_run:
|
|
596
|
+
result["fix_pairs"] = self.cg.mine_fix_pairs(
|
|
597
|
+
[{"role": e.get("role"), "text": e.get("text")}
|
|
598
|
+
for e in new_events], as_proposals=True)
|
|
599
|
+
|
|
600
|
+
if new_events and not dry_run:
|
|
601
|
+
d = self._load()
|
|
602
|
+
last = new_events[-1]
|
|
603
|
+
d["sources"][key] = {
|
|
604
|
+
"t": last.get("t") or last_t,
|
|
605
|
+
"seq": last.get("seq"),
|
|
606
|
+
"count": (wm.get("count") or 0) + written,
|
|
607
|
+
"updated_at": time.time(),
|
|
608
|
+
"path": getattr(source, "path", None),
|
|
609
|
+
}
|
|
610
|
+
self._save(d)
|
|
611
|
+
return result
|
|
612
|
+
|
|
613
|
+
|
|
614
|
+
# 生效条件:text 为 None 或假值时按 "" 参与 sha1;n 默认 12,返回 hexdigest 前 n 位(n=0 得空串)。
|
|
615
|
+
def _sig(text: str, n: int = 12) -> str:
|
|
616
|
+
import hashlib
|
|
617
|
+
return hashlib.sha1((text or "").encode("utf-8")).hexdigest()[:n]
|
|
618
|
+
|
|
619
|
+
|
|
620
|
+
# --------------------------------------------------------------------------
|
|
621
|
+
# 摄取分派(P0 · ingest op):按扩展名选摄取方式,单一入口吃多种文件
|
|
622
|
+
# --------------------------------------------------------------------------
|
|
623
|
+
#
|
|
624
|
+
# 三条摄取链:
|
|
625
|
+
# session —— 会话流(.jsonl):走 Ingestor(watermark + 去重 + fix-pair)
|
|
626
|
+
# doc —— 文档(.md/.txt/...):走 docindex.extract + refindex.add_items
|
|
627
|
+
# code —— 代码(.py/.ts/...):走 codeindex.extract + refindex.add_items
|
|
628
|
+
#
|
|
629
|
+
# 设计要点:
|
|
630
|
+
# - 注册表 INGEST_REGISTRY 是唯一真源:新增后缀只改这里。
|
|
631
|
+
# - dir 动作分链处理:目录里 doc / code / jsonl 混放时各链互不干扰。
|
|
632
|
+
# - 幂等:沿用 refindex.Ledger(size+mtime 水位)与 Ingestor watermark。
|
|
633
|
+
# - 预演:dry_run=True 只统计、不写入(对应计划「可预演」要求)。
|
|
634
|
+
|
|
635
|
+
INGEST_ACTIONS = ("file", "dir", "jsonl", "stat", "hive")
|
|
636
|
+
|
|
637
|
+
INGEST_REGISTRY = {
|
|
638
|
+
# 会话流
|
|
639
|
+
".jsonl": "session", ".ndjson": "session",
|
|
640
|
+
# 文档
|
|
641
|
+
".md": "doc", ".markdown": "doc", ".txt": "doc", ".rst": "doc",
|
|
642
|
+
".html": "doc", ".htm": "doc",
|
|
643
|
+
# 代码
|
|
644
|
+
".py": "code", ".js": "code", ".mjs": "code", ".ts": "code",
|
|
645
|
+
".tsx": "code", ".jsx": "code", ".go": "code", ".rs": "code",
|
|
646
|
+
".java": "code", ".kt": "code", ".swift": "code", ".rb": "code",
|
|
647
|
+
".php": "code", ".cs": "code", ".cpp": "code", ".cc": "code",
|
|
648
|
+
".c": "code", ".h": "code", ".hpp": "code",
|
|
649
|
+
}
|
|
650
|
+
|
|
651
|
+
|
|
652
|
+
# 生效条件:path 为 None 或空串时 splitext 得 "" 且未登记 → 返回 None;扩展名(小写)存在于 INGEST_REGISTRY 时返回其 kind。
|
|
653
|
+
def dispatch_of(path: str):
|
|
654
|
+
"""按扩展名返回摄取方式(session / doc / code);未登记返回 None。"""
|
|
655
|
+
return INGEST_REGISTRY.get(os.path.splitext(path or "")[1].lower())
|
|
656
|
+
|
|
657
|
+
|
|
658
|
+
# --------------------------------------------------------------------------
|
|
659
|
+
# 库内路径闸(issue #425):库内节点文件 / 回收站 / 保护历史**不是外部文档**
|
|
660
|
+
# --------------------------------------------------------------------------
|
|
661
|
+
# 病象:`ingest dir path=<库根>` 或 `ingest file path=trash/<id>.md` 时,遍历把
|
|
662
|
+
# 各层节点文件、回收站、保护历史当外部 md 重新切成 doc_* 节点——
|
|
663
|
+
# · restricted 节点正文被抄进 internal 节点(无令牌访客可检索);
|
|
664
|
+
# · 已 forget+accept 删除的节点从 trash/ 复活成活节点。
|
|
665
|
+
# 根因(条件层):摄取链只判「path 非空」,不判「它是不是记忆库自己」;而 doc 节点
|
|
666
|
+
# 的密级判据(`docindex.sensitivity_for`)只看路径段,源节点 frontmatter 里的
|
|
667
|
+
# `sensitivity: restricted` 根本不参与 ⇒ 重索引必然降级。库内节点文件的密级真源
|
|
668
|
+
# 是节点 frontmatter(`cg.get` 面),**不是**路径——本链不读它,故这里 **fail-closed**:
|
|
669
|
+
# 库根本身、各层目录、trash/、_protected_history/、hippocampus/ 一律拒绝/剪枝,
|
|
670
|
+
# 不猜、不降级。库根下**普通子目录**仍可作为外部资料根(保留既有用法)。
|
|
671
|
+
# 姊妹判据:export 侧已有 `security.check_export_target`(#211),本闸是 ingest 侧对应的一道。
|
|
672
|
+
|
|
673
|
+
#: 库根下不得被 ingest 当外部文档的子树(相对库根首段)。
|
|
674
|
+
#: 层目录从 `mdcg.LAYERS` 取(唯一真源),不在此复制一份。
|
|
675
|
+
_LIBRARY_EXTRA_RESERVED = ("trash", "_protected_history", "hippocampus",
|
|
676
|
+
"_index_log", "export")
|
|
677
|
+
|
|
678
|
+
|
|
679
|
+
def library_reserved_dirs():
|
|
680
|
+
"""库根下保留子树的首段名(层目录 + 回收站/保护历史/海马体等)。
|
|
681
|
+
|
|
682
|
+
层目录**只从 `mdcg.LAYERS` 取**(唯一真源,不在此复制一份);取不到即
|
|
683
|
+
fail-closed 抛错——宁可本次摄取失败,也不静默放宽成「层目录可被当外部文档」。
|
|
684
|
+
"""
|
|
685
|
+
from .mdcg import LAYERS as _LAYERS
|
|
686
|
+
return tuple(_LAYERS) + _LIBRARY_EXTRA_RESERVED
|
|
687
|
+
|
|
688
|
+
|
|
689
|
+
def library_rel(cg, path):
|
|
690
|
+
"""path 相对记忆库根的路径(正斜杠);不在库内返回 None,等于库根返回 ""。
|
|
691
|
+
|
|
692
|
+
realpath + normcase 后比较:符号链接/大小写不改变「是否库内」的结论。
|
|
693
|
+
"""
|
|
694
|
+
root = getattr(cg, "root", None)
|
|
695
|
+
if not root or not path or not str(path).strip():
|
|
696
|
+
return None
|
|
697
|
+
try:
|
|
698
|
+
lib = os.path.normcase(os.path.realpath(root))
|
|
699
|
+
real = os.path.normcase(os.path.realpath(os.path.expanduser(str(path))))
|
|
700
|
+
except (OSError, ValueError):
|
|
701
|
+
return None
|
|
702
|
+
if real == lib:
|
|
703
|
+
return ""
|
|
704
|
+
if real.startswith(lib.rstrip(os.sep) + os.sep):
|
|
705
|
+
return os.path.relpath(real, lib).replace("\\", "/")
|
|
706
|
+
return None
|
|
707
|
+
|
|
708
|
+
|
|
709
|
+
def library_reserved(cg, rel):
|
|
710
|
+
"""相对库根的路径是否落在保留子树(含库根本身)——这些不是外部文档。"""
|
|
711
|
+
if rel is None:
|
|
712
|
+
return False
|
|
713
|
+
if rel == "":
|
|
714
|
+
return True
|
|
715
|
+
return rel.replace("\\", "/").split("/")[0] in library_reserved_dirs()
|
|
716
|
+
|
|
717
|
+
|
|
718
|
+
def library_skip_rel(cg, root):
|
|
719
|
+
"""root 是库根**祖先**时,返回库根相对 root 的路径(遍历时整枝剪掉);否则 None。
|
|
720
|
+
|
|
721
|
+
issue #425 C7:`MDCG_INGEST_ROOT` 设成库根上级目录时,`dir` 空 path 会从上级
|
|
722
|
+
开始 walk,把整个记忆库扫进来。剪枝比「逐文件判」便宜且不漏(层目录、trash、
|
|
723
|
+
_protected_history 都在库根之下)。
|
|
724
|
+
"""
|
|
725
|
+
try:
|
|
726
|
+
lib = os.path.normcase(os.path.realpath(cg.root))
|
|
727
|
+
r = os.path.normcase(os.path.realpath(root))
|
|
728
|
+
except (OSError, ValueError):
|
|
729
|
+
return None
|
|
730
|
+
if lib == r:
|
|
731
|
+
return None
|
|
732
|
+
rel = os.path.relpath(lib, r).replace("\\", "/")
|
|
733
|
+
if rel == ".." or rel.startswith("../"):
|
|
734
|
+
return None
|
|
735
|
+
return rel
|
|
736
|
+
|
|
737
|
+
|
|
738
|
+
# 生效条件:cg 无 root 或 path 为空时返回 None(无路径参数不判);否则按 realpath 判——path 等于库根或落在库根保留子树(层目录/trash/_protected_history/hippocampus 等)内时返回 ok=False 的拒绝回执(error 点名 issue #425 与原因);库外或库根下普通子目录返回 None(放行)。
|
|
739
|
+
def library_ingest_guard(cg, path, what="ingest"):
|
|
740
|
+
"""库内路径闸的**单点**判定(issue #425):返回拒绝回执或 None。"""
|
|
741
|
+
rel = library_rel(cg, path)
|
|
742
|
+
if not library_reserved(cg, rel):
|
|
743
|
+
return None
|
|
744
|
+
where = "记忆库根" if rel == "" else f"记忆库保留子树「{rel.split('/')[0]}」"
|
|
745
|
+
return {"ok": False, "path": path, "error": (
|
|
746
|
+
f"{what} 拒绝:{path} 落在{where}内——库内节点文件/回收站/保护历史"
|
|
747
|
+
f"**不是外部文档**(issue #425:重新索引会把 restricted 正文抄成 internal、"
|
|
748
|
+
f"把已删除节点复活)。外部资料请放在**库根之外**的根再摄取"
|
|
749
|
+
f"(设 MDCG_INGEST_ROOT);库内节点的维护/重建不走 ingest 面。")}
|
|
750
|
+
|
|
751
|
+
|
|
752
|
+
# --------------------------------------------------------------------------
|
|
753
|
+
# 来源密级继承(issue #452):判据面必须同时看**绝对路径**与相对路径的段
|
|
754
|
+
# --------------------------------------------------------------------------
|
|
755
|
+
# 病象:`docindex.index_dir` 产出的 item["path"] 是**相对被摄取根**的;而
|
|
756
|
+
# `refindex.add_items` 用 `docindex.sensitivity_for(it["path"])` 判密级。
|
|
757
|
+
# 当**摄取根本身就是敏感目录**(`ingest(path=<…>/private)`)时,`private` 这一段
|
|
758
|
+
# 正是 root 自身,被 relpath 抹掉 ⇒ 提示永远命中不了,敏感文档以 internal 落库,
|
|
759
|
+
# internal 档身份读得回其正文摘要。同一内容换成父目录摄取则正确判 private 并
|
|
760
|
+
# fail-closed——同一份文档、同一个写者,只因 path 参数不同而一紧一松。
|
|
761
|
+
#
|
|
762
|
+
# 根因(条件层):判据的**输入面**取错了路径(相对路径丢了根自身的段),
|
|
763
|
+
# 不是 sensitivity_for 的实现有问题。故修法:判据仍只有 `docindex.sensitivity_for`
|
|
764
|
+
# 一处,但喂给它的是**源文件的绝对路径**(绝对路径的段集合 ⊇ 相对路径的段集合,
|
|
765
|
+
# 只会更严、不会更松)。item["path"] 保持相对 root 不变——节点 id / doc_ref 的
|
|
766
|
+
# 稳定标识与回读口径不受影响(换 id 会动存量幂等,不在本条范围)。
|
|
767
|
+
def source_sensitivity(path, explicit=None):
|
|
768
|
+
"""源文件密级判据的**唯一取值点**(issue #452):路径面取绝对路径。
|
|
769
|
+
|
|
770
|
+
判据本体仍只有 `docindex.sensitivity_for` 一处;本函数只补两件事:
|
|
771
|
+
① **路径面**喂绝对路径(绝对路径的段集合 ⊇ 相对路径,只会更严);
|
|
772
|
+
② **来源声明面**(issue #425 修复建议 3):源文件 frontmatter 里若带
|
|
773
|
+
`sensitivity`,取「路径判据」与「来源声明」中**更严**的一档——
|
|
774
|
+
否则把一份 restricted 节点文件复制到库外再摄取,密级就凭空降成
|
|
775
|
+
internal(「删掉的比留下的更泄露」同族)。缺键/值域非法时不猜,
|
|
776
|
+
按路径判据(既有语义);库内节点文件由 `library_ingest_guard` 直接拒,
|
|
777
|
+
走不到这里。
|
|
778
|
+
"""
|
|
779
|
+
from . import docindex, nodefile
|
|
780
|
+
from .security import SENSITIVITY_ORDER
|
|
781
|
+
s, _basis = docindex.sensitivity_for(path, explicit)
|
|
782
|
+
cands = [s]
|
|
783
|
+
try:
|
|
784
|
+
with open(path, encoding="utf-8", errors="replace") as f:
|
|
785
|
+
fm, _content = nodefile.loads(f.read())
|
|
786
|
+
except (OSError, ValueError):
|
|
787
|
+
fm = {}
|
|
788
|
+
fm_s = fm.get("sensitivity") if isinstance(fm, dict) else None
|
|
789
|
+
if isinstance(fm_s, str) and fm_s in SENSITIVITY_ORDER:
|
|
790
|
+
cands.append(fm_s)
|
|
791
|
+
return max(cands, key=SENSITIVITY_ORDER.index)
|
|
792
|
+
|
|
793
|
+
|
|
794
|
+
def add_items_with_source_sensitivity(cg, items, *, kind, root, layer=None,
|
|
795
|
+
sensitivity=None, conflicts=None):
|
|
796
|
+
"""按**来源文件**分组调 `refindex.add_items`,每组带该文件继承来的密级。
|
|
797
|
+
|
|
798
|
+
`refindex.add_items` 只接受一个批次级 `sensitivity`,而密级提示可能落在
|
|
799
|
+
**被摄取根自身**的名字上(#452)。故按 item["path"](相对 root 的源文件路径)
|
|
800
|
+
分组,每组以 `source_sensitivity(join(root, rel), sensitivity)` 作为该组的
|
|
801
|
+
密级——判据仍是 `docindex.sensitivity_for` 单点,只是输入面补上了绝对路径。
|
|
802
|
+
doc 之外(code_ref)不涉密级继承,保持原批次一次调用。
|
|
803
|
+
返回 `(ids, sens_counts)`,与 `refindex.add_items` 同形。
|
|
804
|
+
"""
|
|
805
|
+
from . import refindex
|
|
806
|
+
if kind != "doc_ref" or not items:
|
|
807
|
+
return refindex.add_items(cg, items, kind=kind, root=root, layer=layer,
|
|
808
|
+
sensitivity=sensitivity, conflicts=conflicts)
|
|
809
|
+
groups = {}
|
|
810
|
+
for it in items:
|
|
811
|
+
groups.setdefault(it.get("path") or "", []).append(it)
|
|
812
|
+
if len(groups) <= 1:
|
|
813
|
+
rel = next(iter(groups), "")
|
|
814
|
+
s = source_sensitivity(os.path.join(root, rel), sensitivity) if rel else sensitivity
|
|
815
|
+
return refindex.add_items(cg, items, kind=kind, root=root, layer=layer,
|
|
816
|
+
sensitivity=s, conflicts=conflicts)
|
|
817
|
+
ids, sens = [], {}
|
|
818
|
+
for rel, grp in groups.items():
|
|
819
|
+
s = source_sensitivity(os.path.join(root, rel), sensitivity)
|
|
820
|
+
_ids, _sens = refindex.add_items(cg, grp, kind=kind, root=root, layer=layer,
|
|
821
|
+
sensitivity=s, conflicts=conflicts)
|
|
822
|
+
ids.extend(_ids)
|
|
823
|
+
for k, v in (_sens or {}).items():
|
|
824
|
+
sens[k] = sens.get(k, 0) + v
|
|
825
|
+
return ids, sens
|
|
826
|
+
|
|
827
|
+
|
|
828
|
+
# 生效条件:required 形参 cg 总被存入;可选形参 sensitivity 为假值(None/空串)时内部 Ingestor 的 sensitivity 回落模块级常量 SESSION_SENSITIVITY,为真值时用传入的 sensitivity。
|
|
829
|
+
class FileDispatcher:
|
|
830
|
+
"""单一入口吃多种文件:按扩展名分派到会话流 / 文档 / 代码三条摄取链。"""
|
|
831
|
+
|
|
832
|
+
# 生效条件:传入 cg 即成立,sensitivity 为假值(None/空串)时所用 Ingestor 回落 SESSION_SENSITIVITY,否则用传入值。
|
|
833
|
+
def __init__(self, cg, sensitivity=None):
|
|
834
|
+
self.cg = cg
|
|
835
|
+
# 会话链默认 sensitivity=private;调用方可显式覆盖(测试/受限环境)
|
|
836
|
+
self.ingestor = Ingestor(cg, sensitivity=sensitivity or SESSION_SENSITIVITY)
|
|
837
|
+
|
|
838
|
+
# ---- stat:看水位与支持面 ----
|
|
839
|
+
|
|
840
|
+
# 生效条件:from . import refindex 与 refindex.Ledger(self.cg.root).summary() 均不抛异常时返回该 summary 结果,抛任何异常时返回 {}。
|
|
841
|
+
def _ledger_stat(self):
|
|
842
|
+
# `Ledger` 只有 `summary()`、**没有** `stat()`:此前写 `.stat()` 撞
|
|
843
|
+
# AttributeError 被下面的 except 吞成 `{}`,于是 `ingest stat` 的水位面
|
|
844
|
+
# 永远是空且无人知晓(唯一实现面是 `refindex.Ledger.summary`)。
|
|
845
|
+
try:
|
|
846
|
+
from . import refindex
|
|
847
|
+
return refindex.Ledger(self.cg.root).summary()
|
|
848
|
+
except Exception: # noqa: BLE001
|
|
849
|
+
return {}
|
|
850
|
+
|
|
851
|
+
# 生效条件:调用即返回由 INGEST_REGISTRY 按 kind 分组并排序后的 extensions、list(INGEST_ACTIONS)、self.ingestor.watermarks()、self._ledger_stat() 与固定 note。
|
|
852
|
+
def stat(self):
|
|
853
|
+
kinds = {}
|
|
854
|
+
for ext, kind in INGEST_REGISTRY.items():
|
|
855
|
+
kinds.setdefault(kind, []).append(ext)
|
|
856
|
+
return {"ok": True, "kind": "stat", "actions": list(INGEST_ACTIONS),
|
|
857
|
+
"extensions": {k: sorted(v) for k, v in sorted(kinds.items())},
|
|
858
|
+
"watermarks": self.ingestor.watermarks(),
|
|
859
|
+
"ledger": self._ledger_stat(),
|
|
860
|
+
"note": ("注册表是唯一真源:新增后缀只改 INGEST_REGISTRY。"
|
|
861
|
+
"watermarks=会话流水位;ledger=文档/代码文件水位。")}
|
|
862
|
+
|
|
863
|
+
# ---- 单文件 ----
|
|
864
|
+
|
|
865
|
+
# 生效条件:dispatch_of(path) 为 None 时返回 ok=False 的「不支持的后缀」结果;path 不是文件时返回 ok=False 的「文件不存在」结果;kind=="session" 时转 ingest_jsonl(path, dry_run=dry_run)(layer/sensitivity 不参与);其它 kind 转 _ingest_doc_or_code(path, kind, layer=layer, sensitivity=sensitivity, dry_run=dry_run)。
|
|
866
|
+
def ingest_file(self, path, layer=None, sensitivity=None, dry_run=False):
|
|
867
|
+
kind = dispatch_of(path)
|
|
868
|
+
if kind is None:
|
|
869
|
+
ext = os.path.splitext(path or "")[1] or "(无后缀)"
|
|
870
|
+
return {"ok": False, "path": path, "error": f"不支持的后缀:{ext}",
|
|
871
|
+
"supported": sorted(INGEST_REGISTRY)}
|
|
872
|
+
if not os.path.isfile(path):
|
|
873
|
+
return {"ok": False, "path": path, "error": "文件不存在"}
|
|
874
|
+
# issue #425:库内节点文件 / trash/<id>.md / _protected_history 快照
|
|
875
|
+
# **不是外部文档**——单文件链同样过闸(否则一条 file 摄取即可把
|
|
876
|
+
# restricted 正文抄成 internal,或把已删除节点复活)。
|
|
877
|
+
_g = library_ingest_guard(self.cg, path, "ingest file")
|
|
878
|
+
if _g is not None:
|
|
879
|
+
return _g
|
|
880
|
+
if kind == "session":
|
|
881
|
+
return self.ingest_jsonl(path, dry_run=dry_run)
|
|
882
|
+
return self._ingest_doc_or_code(path, kind, layer=layer,
|
|
883
|
+
sensitivity=sensitivity, dry_run=dry_run)
|
|
884
|
+
|
|
885
|
+
# 生效条件:kind=="code" 用 codeindex 否则用 docindex;mod.extract 抛 ValueError 时返回 ok=False 的「抽取失败」;dry_run 为真时只返回 items 计数与前 20 个 node_id 不写盘;否则经 refindex.add_items(kind 为 code_ref/doc_ref)写入并返回 indexed、ids[:20] 与 sensitivity。
|
|
886
|
+
def _ingest_doc_or_code(self, path, kind, layer=None, sensitivity=None,
|
|
887
|
+
dry_run=False):
|
|
888
|
+
from . import codeindex, docindex, refindex
|
|
889
|
+
mod = codeindex if kind == "code" else docindex
|
|
890
|
+
rel = os.path.basename(path)
|
|
891
|
+
with open(path, encoding="utf-8", errors="replace") as f:
|
|
892
|
+
src = f.read()
|
|
893
|
+
try:
|
|
894
|
+
items = mod.extract(src, path=rel,
|
|
895
|
+
suffix=os.path.splitext(path)[1].lower())
|
|
896
|
+
except ValueError as exc:
|
|
897
|
+
return {"ok": False, "path": path, "error": f"抽取失败:{exc}"}
|
|
898
|
+
items = items or []
|
|
899
|
+
if dry_run:
|
|
900
|
+
return {"ok": True, "dry_run": True, "kind": kind, "path": path,
|
|
901
|
+
"items": len(items),
|
|
902
|
+
"ids": [mod.node_id(i) for i in items[:20]],
|
|
903
|
+
"note": "预演:只抽取计数,未写盘"}
|
|
904
|
+
ref_kind = "code_ref" if kind == "code" else "doc_ref"
|
|
905
|
+
_cf = [] # #352-E:同 id 已被另一个源根占用 ⇒ 不覆写
|
|
906
|
+
ids, sens = add_items_with_source_sensitivity(
|
|
907
|
+
self.cg, items, kind=ref_kind,
|
|
908
|
+
root=os.path.dirname(path) or ".", layer=layer,
|
|
909
|
+
sensitivity=sensitivity, conflicts=_cf)
|
|
910
|
+
_out = {"ok": True, "kind": kind, "path": path, "items": len(items),
|
|
911
|
+
"indexed": len(ids), "ids": ids[:20], "sensitivity": sens}
|
|
912
|
+
if _cf: # 冲突不静默(单文件链同样点名)
|
|
913
|
+
_out["conflict_count"] = len(_cf)
|
|
914
|
+
_out["conflicts"] = _cf[:20]
|
|
915
|
+
return _out
|
|
916
|
+
|
|
917
|
+
# ---- 目录 ----
|
|
918
|
+
|
|
919
|
+
# 生效条件:调用即 os.walk(root) 统计每个文件名经 dispatch_of 得到的 kind(无匹配记 "unsupported")并返回 dry_run 预演计数结果。
|
|
920
|
+
def _dry_dir(self, root):
|
|
921
|
+
counts = {}
|
|
922
|
+
_lib_rel = library_skip_rel(self.cg, root) # issue #425 C7:库根整枝剪掉
|
|
923
|
+
for dp, dns, fns in os.walk(root):
|
|
924
|
+
rel_dir = os.path.relpath(dp, root).replace("\\", "/")
|
|
925
|
+
if rel_dir == ".":
|
|
926
|
+
rel_dir = ""
|
|
927
|
+
if _lib_rel is not None:
|
|
928
|
+
dns[:] = [d for d in dns
|
|
929
|
+
if ("%s/%s" % (rel_dir, d) if rel_dir else d) != _lib_rel
|
|
930
|
+
and not ("%s/%s" % (rel_dir, d) if rel_dir else d)
|
|
931
|
+
.startswith(_lib_rel + "/")]
|
|
932
|
+
for name in fns:
|
|
933
|
+
k = dispatch_of(name) or "unsupported"
|
|
934
|
+
counts[k] = counts.get(k, 0) + 1
|
|
935
|
+
return {"ok": True, "dry_run": True, "kind": "dir", "root": root,
|
|
936
|
+
"counts": counts,
|
|
937
|
+
"note": "预演:仅统计各链文件数,未做任何写入"}
|
|
938
|
+
|
|
939
|
+
# 生效条件:root 非目录时返回 ok=False 的「目录不存在」;dry_run 为真时返回 _dry_dir(root);否则对 doc_ref/code_ref 两链各以 patterns/max_files/max_items/incremental/ledger 调 refindex.index_dir(commit=False) 与 add_items 后 ledger.save(),并把 root 下 **/*.jsonl 前 max_files 个逐个 ingest_jsonl 后返回 out。
|
|
940
|
+
def ingest_dir(self, root, layer=None, sensitivity=None, patterns=None,
|
|
941
|
+
max_files=500, max_items=2000, incremental=False,
|
|
942
|
+
dry_run=False):
|
|
943
|
+
from . import refindex
|
|
944
|
+
if not os.path.isdir(root):
|
|
945
|
+
return {"ok": False, "error": f"目录不存在:{root}"}
|
|
946
|
+
# issue #425:库根本身 / 层目录 / trash / _protected_history / hippocampus
|
|
947
|
+
# **不是外部文档**——直接拒绝(fail-closed),不猜扫描根。
|
|
948
|
+
_g = library_ingest_guard(self.cg, root, "ingest dir")
|
|
949
|
+
if _g is not None:
|
|
950
|
+
return _g
|
|
951
|
+
if dry_run:
|
|
952
|
+
return self._dry_dir(root)
|
|
953
|
+
# issue #425 C7:root 是库根**祖先**时整枝剪掉库根(与 SKIP_DIRS 同处生效)。
|
|
954
|
+
_lib_rel = library_skip_rel(self.cg, root)
|
|
955
|
+
_skip = [_lib_rel] if _lib_rel else None
|
|
956
|
+
ledger = refindex.Ledger(self.cg.root)
|
|
957
|
+
out = {"ok": True, "kind": "dir", "root": root, "chains": {}}
|
|
958
|
+
if _lib_rel:
|
|
959
|
+
out["library_pruned"] = [_lib_rel]
|
|
960
|
+
for ref_kind, key in (("doc_ref", "doc"), ("code_ref", "code")):
|
|
961
|
+
items, errors, stats = refindex.index_dir(
|
|
962
|
+
root, kind=ref_kind, patterns=patterns, max_files=max_files,
|
|
963
|
+
max_items=max_items, incremental=incremental, ledger=ledger,
|
|
964
|
+
skip_dirs=_skip, commit=False)
|
|
965
|
+
_cf = [] # #352-E:同 id 已被另一个源根占用 ⇒ 不覆写
|
|
966
|
+
# issue #452:按来源文件继承密级(判据面补上绝对路径的段)。
|
|
967
|
+
ids, sens = add_items_with_source_sensitivity(
|
|
968
|
+
self.cg, items, kind=ref_kind,
|
|
969
|
+
root=root, layer=layer,
|
|
970
|
+
sensitivity=sensitivity,
|
|
971
|
+
conflicts=_cf)
|
|
972
|
+
for _c in _cf: # 冲突源文件的水位一并丢掉(否则下次增量跳过)
|
|
973
|
+
ledger.drop(_c.get("src")
|
|
974
|
+
or os.path.join(root, _c.get("path") or ""))
|
|
975
|
+
# 写序(同 op=index_code/index_doc):节点先落盘,水位随后——
|
|
976
|
+
# 中途被杀只会留下「节点新 + 水位旧」,下次增量重切,不会静默漏漂移。
|
|
977
|
+
ledger.save()
|
|
978
|
+
out["chains"][key] = {
|
|
979
|
+
"indexed": len(ids), "errors": len(errors),
|
|
980
|
+
"files": stats.get("files"), "truncated": stats.get("truncated"),
|
|
981
|
+
"empty_scan": stats.get("empty_scan"),
|
|
982
|
+
"skipped_unchanged": stats.get("skipped_unchanged", 0),
|
|
983
|
+
"skipped_suffixes": stats.get("skipped_suffixes", []),
|
|
984
|
+
"sensitivity": sens}
|
|
985
|
+
if _cf: # #352-E:冲突不静默
|
|
986
|
+
out["chains"][key]["conflict_count"] = len(_cf)
|
|
987
|
+
out["chains"][key]["conflicts"] = _cf[:20]
|
|
988
|
+
# 会话流(.jsonl)逐个增量摄取——库内台账/回收站下的 jsonl 同样不是
|
|
989
|
+
# 外部会话(issue #425:`hippocampus/*.jsonl` 等库内账本不得被当会话源)。
|
|
990
|
+
jsons = [p for p in sorted(glob.glob(os.path.join(root, "**", "*.jsonl"),
|
|
991
|
+
recursive=True))[:max_files]
|
|
992
|
+
if not library_reserved(self.cg, library_rel(self.cg, p))]
|
|
993
|
+
ses = []
|
|
994
|
+
for p in jsons:
|
|
995
|
+
r = self.ingest_jsonl(p)
|
|
996
|
+
ses.append({"path": p, "written": r.get("written", 0),
|
|
997
|
+
"new_events": r.get("new_events", 0)})
|
|
998
|
+
out["chains"]["session"] = {"files": len(jsons), "results": ses}
|
|
999
|
+
return out
|
|
1000
|
+
|
|
1001
|
+
# ---- 会话流 ----
|
|
1002
|
+
|
|
1003
|
+
@staticmethod
|
|
1004
|
+
# 生效条件:required 形参 path 能被 open(...,encoding="utf-8",errors="replace") 打开时读前 50000 字符,head 含 '"user/message"'、'"assistant/message"'、'"tool/call"' 任一标记则返回 DSHSessionSource(path),否则返回 JsonlSource(path);打开抛 OSError 时直接返回 JsonlSource(path)。
|
|
1005
|
+
def _auto_source(path):
|
|
1006
|
+
"""通用 JSONL vs DSH 会话:按内容探测,避免调用方选错源类型。"""
|
|
1007
|
+
try:
|
|
1008
|
+
with open(path, encoding="utf-8", errors="replace") as f:
|
|
1009
|
+
head = f.read(50000)
|
|
1010
|
+
except OSError:
|
|
1011
|
+
return JsonlSource(path)
|
|
1012
|
+
for marker in ('"user/message"', '"assistant/message"', '"tool/call"'):
|
|
1013
|
+
if marker in head:
|
|
1014
|
+
return DSHSessionSource(path)
|
|
1015
|
+
return JsonlSource(path)
|
|
1016
|
+
|
|
1017
|
+
# 生效条件:path 不是文件时返回 ok=False 的「文件不存在」;path 落在记忆库保留子树(层目录/trash/_protected_history/hippocampus 等)内时返回 ok=False 的库内路径拒绝(issue #425);否则经 _auto_source(path) 选源后调 self.ingestor.ingest(src, dry_run=dry_run, max_events=max_events),并补上 ok=True/kind/path/source_class 后返回。
|
|
1018
|
+
def ingest_jsonl(self, path, dry_run=False, max_events=None):
|
|
1019
|
+
if not os.path.isfile(path):
|
|
1020
|
+
return {"ok": False, "error": f"文件不存在:{path}"}
|
|
1021
|
+
_g = library_ingest_guard(self.cg, path, "ingest jsonl")
|
|
1022
|
+
if _g is not None:
|
|
1023
|
+
return _g
|
|
1024
|
+
src = self._auto_source(path)
|
|
1025
|
+
res = self.ingestor.ingest(src, dry_run=dry_run, max_events=max_events)
|
|
1026
|
+
res.update({"ok": True, "kind": "session", "path": path,
|
|
1027
|
+
"source_class": type(src).__name__})
|
|
1028
|
+
return res
|
|
1029
|
+
|
|
1030
|
+
|
|
1031
|
+
# 生效条件:cg 必填;action 为 None/空串时 (action or "stat") 归为 stat;file/jsonl 缺 path 返回 ok=False;dir 的 max_files/max_items 走 int(x or 500)/int(x or 2000),传 0 也变 500/2000;未知 action 抛 ValueError。
|
|
1032
|
+
def run(cg, action: str = "stat", **kw):
|
|
1033
|
+
"""ingest op 唯一入口。"""
|
|
1034
|
+
act = (action or "stat").strip().lower()
|
|
1035
|
+
d = FileDispatcher(cg, sensitivity=kw.get("sensitivity"))
|
|
1036
|
+
if act == "stat":
|
|
1037
|
+
return d.stat()
|
|
1038
|
+
if act == "file":
|
|
1039
|
+
p = kw.get("path")
|
|
1040
|
+
if not p:
|
|
1041
|
+
return {"ok": False, "error": "file 动作需要 path"}
|
|
1042
|
+
return d.ingest_file(p, layer=kw.get("layer"),
|
|
1043
|
+
sensitivity=kw.get("sensitivity"),
|
|
1044
|
+
dry_run=bool(kw.get("dry_run")))
|
|
1045
|
+
if act == "dir":
|
|
1046
|
+
# issue #351(2026-10-16 DSH 端):空 path **不得**回落记忆库根。库根下是
|
|
1047
|
+
# 各层节点文件、trash/、_protected_history/,把它们当外部文档切成 doc_ref
|
|
1048
|
+
# 节点会 (a) 把 restricted 正文抄进 internal 节点(无令牌访客可检索)、
|
|
1049
|
+
# (b) 把已 forget 并 accept 删除的节点复活。缺省根改由调用方显式给 path
|
|
1050
|
+
# 或设 env MDCG_INGEST_ROOT(分派层已在白名单检查之前解析)——这里
|
|
1051
|
+
# fail-closed,不猜扫描根。
|
|
1052
|
+
p = kw.get("path")
|
|
1053
|
+
if not p:
|
|
1054
|
+
return {"ok": False, "error": (
|
|
1055
|
+
"dir 动作需要 path(或设 env MDCG_INGEST_ROOT)——不猜测扫描根"
|
|
1056
|
+
"(issue #351:缺省回落记忆库根会把 restricted/已删除节点"
|
|
1057
|
+
"重新索引成 internal 并可被访客检索)")}
|
|
1058
|
+
return d.ingest_dir(p, layer=kw.get("layer"),
|
|
1059
|
+
sensitivity=kw.get("sensitivity"),
|
|
1060
|
+
patterns=kw.get("patterns"),
|
|
1061
|
+
max_files=int(kw.get("max_files") or 500),
|
|
1062
|
+
max_items=int(kw.get("max_items") or 2000),
|
|
1063
|
+
incremental=bool(kw.get("incremental")),
|
|
1064
|
+
dry_run=bool(kw.get("dry_run")))
|
|
1065
|
+
if act == "jsonl":
|
|
1066
|
+
p = kw.get("path")
|
|
1067
|
+
if not p:
|
|
1068
|
+
return {"ok": False, "error": "jsonl 动作需要 path"}
|
|
1069
|
+
return d.ingest_jsonl(p, dry_run=bool(kw.get("dry_run")),
|
|
1070
|
+
max_events=kw.get("max_events"))
|
|
1071
|
+
if act == "hive":
|
|
1072
|
+
# M6:蜂巢任务事件源(hive/jobs → contextual)。root 解析链:显式 path >
|
|
1073
|
+
# env MDCG_HIVE_JOBS > 报错指引(fail-closed,不猜仓库布局)。
|
|
1074
|
+
# §5.5 硬纪律:mine_fix_pairs=False(先落账后挖矿——自动挖掘产物直写
|
|
1075
|
+
# knowledge 层违反双轨制,实测 0→2 污染);默认密级 internal,
|
|
1076
|
+
# error 事件由源层 per-event 覆写 private(失败细节可能含路径/配置)。
|
|
1077
|
+
root = kw.get("path") or os.environ.get("MDCG_HIVE_JOBS")
|
|
1078
|
+
if not root:
|
|
1079
|
+
return {"ok": False, "error": (
|
|
1080
|
+
"hive 动作需要 path(hive/jobs 目录),或设 env MDCG_HIVE_JOBS——"
|
|
1081
|
+
"不猜测仓库布局(fail-closed)")}
|
|
1082
|
+
src = HiveJobsSource(root, error_sensitivity=kw.get("error_sensitivity")
|
|
1083
|
+
if kw.get("error_sensitivity") is not None else "private")
|
|
1084
|
+
rep = Ingestor(cg, layer=kw.get("layer") or "contextual",
|
|
1085
|
+
sensitivity=kw.get("sensitivity") or "internal"
|
|
1086
|
+
).ingest(src, mine_fix_pairs=False,
|
|
1087
|
+
max_events=kw.get("max_events"),
|
|
1088
|
+
dry_run=bool(kw.get("dry_run")))
|
|
1089
|
+
rep["source"] = src.key()
|
|
1090
|
+
rep["skipped"] = src.skipped
|
|
1091
|
+
return rep
|
|
834
1092
|
raise ValueError(f"未知 ingest action:{action!r}(允许 {INGEST_ACTIONS})")
|