@furongjun1999/dsh-memory 0.8.1 → 0.9.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +662 -659
- package/codebuddy/README.md +1 -1
- package/data/policy.json +38 -27
- package/docs/README.md +18 -2
- package/docs/discipline/harnesses.yaml +1 -1
- package/docs/discipline/templates/spec.txt.tmpl +23 -0
- package/docs//350/247/204/350/214/203/AI/347/274/226/347/250/213/351/200/232/347/224/250/350/247/204/350/214/203.md +179 -0
- package/docs//350/247/204/350/214/203/AI/347/274/226/347/250/213/351/200/232/347/224/250/350/247/204/350/214/203_/347/201/265/346/236/242/351/200/202/351/205/215.md +45 -0
- package/docs//350/247/204/350/214/203/AI/347/274/226/347/250/213/351/200/232/347/224/250/350/247/204/350/214/203_/350/207/252/346/243/200/346/270/205/345/215/225.md +69 -0
- package/dsh/README.md +1 -1
- package/dsh/cordis.yml.example +139 -139
- package/hive/README.md +705 -0
- package/hive/id_charset_blocks.txt +147 -0
- package/lib/bridge.d.ts +3 -0
- package/lib/bridge.js +32 -11
- package/lib/cli.js +14 -17
- package/lib/hooks.d.ts +37 -3
- package/lib/hooks.js +176 -24
- package/lib/index.d.ts +2 -2
- package/lib/index.js +29 -6
- package/lib/init.js +21 -21
- package/lib/lib/datapath.d.ts +56 -0
- package/lib/lib/datapath.js +102 -9
- package/lib/lib/hook_audit.d.ts +0 -1
- package/lib/lib/hook_audit.js +51 -3
- package/lib/lib/mdcg_client.d.ts +4 -1
- package/lib/lib/mdcg_client.js +10 -2
- package/lib/lib/mutual.js +61 -15
- package/lib/lib/prompt_safety.js +28 -2
- package/lib/lib/roleplay_web.js +30 -2
- package/lib/lib/token_store.d.ts +25 -6
- package/lib/lib/token_store.js +224 -33
- package/lib/tools.d.ts +15 -2
- package/lib/tools.js +44 -4
- package/md_cg/_guard_kit.py +121 -0
- package/md_cg/admission.py +718 -718
- package/md_cg/anchor.py +114 -0
- package/md_cg/audit.py +37 -7
- package/md_cg/autonomy.py +433 -288
- package/md_cg/autonomy_modes.py +781 -642
- package/md_cg/bench_beir_bm25.py +154 -0
- package/md_cg/bench_beir_scifact.py +170 -0
- package/md_cg/bench_bm25_blockmax.py +134 -0
- package/md_cg/bench_bm25_incremental.py +246 -0
- package/md_cg/bench_ccg_heading.py +118 -0
- package/md_cg/bench_csre_correctness.py +145 -0
- package/md_cg/bench_excerpt_budget.py +94 -0
- package/md_cg/bench_fuzzy_substrings.py +299 -0
- package/md_cg/bench_memory_rsi.py +358 -0
- package/md_cg/bench_memory_rsi_latency.py +82 -0
- package/md_cg/bench_memory_rsi_public.py +116 -0
- package/md_cg/bench_memory_rsi_tools.py +229 -0
- package/md_cg/bench_rrf_candidates.py +212 -0
- package/md_cg/bench_rrf_public.py +100 -0
- package/md_cg/bench_text_projections.py +221 -0
- package/md_cg/bench_timeanchor.py +357 -0
- package/md_cg/bm25.py +601 -0
- package/md_cg/branches.py +404 -327
- package/md_cg/build_postings.py +74 -73
- package/md_cg/ccgc.py +4 -2
- package/md_cg/chain.py +573 -379
- package/md_cg/coldverify.py +531 -292
- package/md_cg/condaccept.py +263 -0
- package/md_cg/condgate.py +308 -0
- package/md_cg/config_registry.py +329 -0
- package/md_cg/config_registry_bulk.py +482 -0
- package/md_cg/config_registry_guard.py +487 -0
- package/md_cg/config_render.py +354 -0
- package/md_cg/config_validate.py +455 -0
- package/md_cg/conformance.py +747 -726
- package/md_cg/conformance_core.py +753 -0
- package/md_cg/consistency.py +1278 -955
- package/md_cg/consolidate.py +1793 -1633
- package/md_cg/crypto.py +346 -24
- package/md_cg/datapath.py +137 -12
- package/md_cg/eval_common.py +522 -575
- package/md_cg/evidence.py +946 -585
- package/md_cg/evidence_delivery.py +101 -0
- package/md_cg/experience_consolidation.py +317 -0
- package/md_cg/export.py +329 -222
- package/md_cg/forgetting.py +1050 -922
- package/md_cg/freshness.py +601 -548
- package/md_cg/fsutil.py +1120 -824
- package/md_cg/goal_gen.py +458 -0
- package/md_cg/hotcache.py +107 -9
- package/md_cg/hyperedge.py +262 -251
- package/md_cg/identity.py +573 -391
- package/md_cg/imgskill.py +1712 -0
- package/md_cg/insight.py +540 -521
- package/md_cg/interface_schema.py +146 -0
- package/md_cg/interop.py +20 -0
- package/md_cg/judgment_manifest.py +24 -1
- package/md_cg/lifecycle.py +426 -345
- package/md_cg/links.py +27 -6
- package/md_cg/logref.py +332 -327
- package/md_cg/mcp_server.py +5227 -4449
- package/md_cg/mdcg.py +1301 -203
- package/md_cg/mdcos.py +7108 -5440
- package/md_cg/memory_rsi.py +308 -0
- package/md_cg/metacognition.py +726 -591
- package/md_cg/nodefile.py +151 -3
- package/md_cg/op_table.py +1128 -0
- package/md_cg/postings.py +524 -300
- package/md_cg/predict.py +1103 -1100
- package/md_cg/probe_memory_rsi.py +95 -0
- package/md_cg/probe_memory_rsi_mcp.py +66 -0
- package/md_cg/protect.py +654 -494
- package/md_cg/protocol.py +492 -411
- package/md_cg/provenance.py +606 -600
- package/md_cg/qmd.py +477 -0
- package/md_cg/qmd_llm.py +334 -0
- package/md_cg/query_excerpt.py +171 -0
- package/md_cg/queue_health.py +130 -0
- package/md_cg/reach.py +470 -453
- package/md_cg/readcache.py +115 -3
- package/md_cg/reconcile.py +239 -228
- package/md_cg/refindex.py +1191 -1042
- package/md_cg/refine.py +613 -604
- package/md_cg/retrieve.py +128 -0
- package/md_cg/review_cli.py +204 -26
- package/md_cg/revise.py +140 -0
- package/md_cg/rollback.py +657 -420
- package/md_cg/rollback_cli.py +124 -104
- package/md_cg/rotate.py +347 -0
- package/md_cg/routing.py +490 -429
- package/md_cg/rrf_candidates.py +204 -0
- package/md_cg/run_tests.py +19 -1
- package/md_cg/scrub.py +177 -23
- package/md_cg/security.py +269 -17
- package/md_cg/self_state.py +1053 -1029
- package/md_cg/self_surface.py +50 -0
- package/md_cg/semantic/canonical.py +144 -144
- package/md_cg/semantic/export_en_zh_map.py +2 -2
- package/md_cg/semantic/zh_en_atoms.py +141 -139
- package/md_cg/session5.py +106 -0
- package/md_cg/signer.py +76 -2
- package/md_cg/sleep.py +1779 -1583
- package/md_cg/sources.py +1091 -833
- package/md_cg/state_events.py +75 -10
- package/md_cg/state_slots.py +312 -221
- package/md_cg/stg.py +85 -18
- package/md_cg/stgidx.py +537 -281
- package/md_cg/subgraph.py +851 -729
- package/md_cg/sustain.py +458 -19
- package/md_cg/task_memory.py +367 -0
- package/md_cg/test_action_derive.py +210 -203
- package/md_cg/test_audit_heartbeat_20261015.py +417 -0
- package/md_cg/test_audit_protection_20261015.py +483 -0
- package/md_cg/test_audit_rotate_parity.py +280 -0
- package/md_cg/test_audit_state_20261015.py +253 -0
- package/md_cg/test_audit_write_20261015.py +161 -0
- package/md_cg/test_auto_defaults.py +43 -4
- package/md_cg/test_autonomy_emotion.py +328 -0
- package/md_cg/test_autonomy_modes.py +2013 -1973
- package/md_cg/test_b1_foreign_read.py +245 -0
- package/md_cg/test_bm25.py +438 -0
- package/md_cg/test_bm25_hydration_refill.py +265 -0
- package/md_cg/test_bm25_incremental.py +241 -0
- package/md_cg/test_bm25_incremental_mutations.py +160 -0
- package/md_cg/test_bm25_mutations.py +205 -0
- package/md_cg/test_bm25_pool_view.py +196 -0
- package/md_cg/test_boot_openblas_guard.py +218 -197
- package/md_cg/test_boundary_hit.py +1 -1
- package/md_cg/test_branches_sensitivity.py +375 -0
- package/md_cg/test_c8_search_rrf_gates.py +1 -1
- package/md_cg/test_canonical_timezones.py +211 -0
- package/md_cg/test_ccg_heading_parser.py +73 -0
- package/md_cg/test_chain_best_first.py +403 -0
- package/md_cg/test_chain_candidate_scope.py +227 -0
- package/md_cg/test_cn_recall_grams.py +206 -0
- package/md_cg/test_code_kind_forbidden.py +101 -0
- package/md_cg/test_coldverify_drain.py +127 -0
- package/md_cg/test_condaccept.py +156 -0
- package/md_cg/test_condgate.py +141 -0
- package/md_cg/test_config_registry.py +55 -0
- package/md_cg/test_crypto_rotate.py +257 -0
- package/md_cg/test_csre_correctness.py +297 -0
- package/md_cg/test_csre_index_import_guard.py +161 -0
- package/md_cg/test_csre_mutations.py +34 -0
- package/md_cg/test_csre_session.py +69 -0
- package/md_cg/test_curiosity_budget.py +468 -0
- package/md_cg/test_d17_autostart_toolface.py +453 -0
- package/md_cg/test_datapath_tilde_home.py +271 -0
- package/md_cg/test_derived_cache_generations.py +256 -0
- package/md_cg/test_evidence_delivery_compact.py +54 -0
- package/md_cg/test_exact_text_cache.py +191 -0
- package/md_cg/test_excerpt_budget.py +188 -0
- package/md_cg/test_experience_consolidation.py +112 -0
- package/md_cg/test_export_default_collision.py +138 -0
- package/md_cg/test_export_snapshots.py +322 -0
- package/md_cg/test_fuzzy_substring_equivalence.py +127 -0
- package/md_cg/test_gain_gate.py +16 -5
- package/md_cg/test_goal_gen.py +355 -0
- package/md_cg/test_h2_session_view_norm.py +251 -233
- package/md_cg/test_h4_sustain_snapshot.py +1490 -1387
- package/md_cg/test_heartbeat_ledger.py +557 -0
- package/md_cg/test_hierarchy_audit_graph.py +237 -0
- package/md_cg/test_i50c_meta_passthrough.py +628 -608
- package/md_cg/test_i50e_readside_protection.py +658 -656
- package/md_cg/test_identity_attribution.py +96 -2
- package/md_cg/test_identity_contract.py +329 -0
- package/md_cg/test_imgskill.py +1444 -0
- package/md_cg/test_index_snapshot_shape.py +240 -0
- package/md_cg/test_interop_judgment.py +258 -228
- package/md_cg/test_issue111_gate_order.py +77 -0
- package/md_cg/test_issue111_scoring_contract.py +87 -0
- package/md_cg/test_issue121_calibration_bounds.py +293 -0
- package/md_cg/test_issue128_session_env_authority.py +246 -0
- package/md_cg/test_issue130_rrf_rank_credibility.py +217 -0
- package/md_cg/test_issue131_postings_content_staleness.py +424 -0
- package/md_cg/test_issue145_sleep_no_silent_rollback.py +168 -0
- package/md_cg/test_issue146_frontmatter_quarantine.py +428 -0
- package/md_cg/test_issue147_pack_id_escape.py +378 -0
- package/md_cg/test_issue148_merge_downgrade_leak.py +192 -0
- package/md_cg/test_issue149_accel_default_face.py +249 -0
- package/md_cg/test_issue150_empty_query_terms.py +328 -0
- package/md_cg/test_issue151_153_157_write_cache_index.py +152 -0
- package/md_cg/test_issue151b_base_add_importance_guard.py +166 -0
- package/md_cg/test_issue152_sealed_postings.py +309 -0
- package/md_cg/test_issue152b_decrypt_cache.py +218 -0
- package/md_cg/test_issue153b_cache_alias_and_canon.py +236 -0
- package/md_cg/test_issue153b_index_generation_invalidation.py +143 -0
- package/md_cg/test_issue157b_index_remainder.py +344 -0
- package/md_cg/test_issue189_conclusion_bidirectional.py +223 -0
- package/md_cg/test_issue190_tidy_idempotent.py +806 -0
- package/md_cg/test_issue191_import_trust.py +382 -0
- package/md_cg/test_issue192_move_layer_indices_guard.py +258 -0
- package/md_cg/test_issue193_consistency_knowledge_guard.py +524 -0
- package/md_cg/test_issue193_write_scope.py +378 -0
- package/md_cg/test_issue197_rank_credibility_property.py +201 -0
- package/md_cg/test_issue211_export_target.py +325 -0
- package/md_cg/test_issue212_writeface_gaps.py +349 -0
- package/md_cg/test_issue214_reject_sensitivity.py +267 -0
- package/md_cg/test_issue215_condition_source_gap.py +272 -0
- package/md_cg/test_issue216_clash_routes.py +214 -0
- package/md_cg/test_issue217_slot_routes.py +248 -0
- package/md_cg/test_issue218_postcondition_value_blind.py +222 -0
- package/md_cg/test_issue225_consistency_torn_log.py +433 -0
- package/md_cg/test_issue245_swarm_traversal.py +366 -0
- package/md_cg/test_issue258_pack_sensitivity.py +468 -0
- package/md_cg/test_issue265_coldverify_multiproc.py +438 -0
- package/md_cg/test_issue274_read_node_id_none.py +341 -0
- package/md_cg/test_issue275_layer_param.py +250 -0
- package/md_cg/test_issue277_node_id_traversal.py +384 -0
- package/md_cg/test_issue278_neg_separated_tension.py +271 -0
- package/md_cg/test_issue278_neg_window.py +294 -0
- package/md_cg/test_issue278_negative_layering.py +83 -0
- package/md_cg/test_issue279_signers_path_guard.py +652 -0
- package/md_cg/test_issue284_skip_not_pass.py +291 -0
- package/md_cg/test_issue294_flywheel_sensitivity.py +573 -0
- package/md_cg/test_issue295_sleep_root_binding.py +589 -0
- package/md_cg/test_issue297_rewrite_sensitivity.py +454 -0
- package/md_cg/test_issue298_consistency_ledger_sensitivity.py +421 -0
- package/md_cg/test_issue301_baseline_corpus_gate.py +245 -0
- package/md_cg/test_issue303_design_boundaries.py +262 -0
- package/md_cg/test_issue313_writelimit_sensitivity.py +404 -0
- package/md_cg/test_issue315_merge_branch_sensitivity.py +384 -0
- package/md_cg/test_issue315_sensitivity_passthrough_guard.py +514 -0
- package/md_cg/test_issue319_tenant_unregistered_bypass.py +257 -0
- package/md_cg/test_issue320_vstate_requires_verify.py +279 -0
- package/md_cg/test_issue321_mcp_deep_json.py +305 -0
- package/md_cg/test_issue322_node_write_lock.py +727 -0
- package/md_cg/test_issue323_aggregate_segment_boundary.py +365 -0
- package/md_cg/test_issue324_write_size_fields.py +215 -0
- package/md_cg/test_issue325_audit_poison.py +230 -0
- package/md_cg/test_issue326_append_jsonl_oserror.py +464 -0
- package/md_cg/test_issue326_ingest_auto_whitelist.py +331 -0
- package/md_cg/test_issue326_key_perms.py +355 -0
- package/md_cg/test_issue326_twin_parser_parity.py +323 -0
- package/md_cg/test_issue326_vm_call_frame.py +569 -0
- package/md_cg/test_issue345_writelimit_effective_sens.py +234 -0
- package/md_cg/test_issue346_overwrite_sensitivity_gate.py +424 -0
- package/md_cg/test_issue346_review_overwrite_guard.py +509 -0
- package/md_cg/test_issue348_insight_verify_op.py +200 -0
- package/md_cg/test_issue348_merge_rmw_lock.py +345 -0
- package/md_cg/test_issue350_casefold_collision.py +236 -0
- package/md_cg/test_issue351_ingest_root_fallback.py +202 -0
- package/md_cg/test_issue352_imgaudit.py +418 -0
- package/md_cg/test_issue352_pathsjson.py +537 -0
- package/md_cg/test_issue352_refindex.py +412 -0
- package/md_cg/test_issue352_rollback.py +604 -0
- package/md_cg/test_issue352_vstateinherit.py +196 -0
- package/md_cg/test_issue354_p1_writelimit_type.py +522 -0
- package/md_cg/test_issue354_p2_datapath_clock.py +381 -0
- package/md_cg/test_issue354_p2_writelimit_toctou.py +327 -0
- package/md_cg/test_issue354_rust_index_containment.py +571 -0
- package/md_cg/test_issue354_rust_wal_containment.py +606 -0
- package/md_cg/test_issue358_truncation_observable.py +247 -0
- package/md_cg/test_issue360_tidy_private_excerpt.py +196 -0
- package/md_cg/test_issue361_blindspot_not_silent_drop.py +494 -0
- package/md_cg/test_issue362_neg_condition.py +468 -0
- package/md_cg/test_issue363_lexical_idf.py +275 -0
- package/md_cg/test_issue370_readcache_assembly.py +488 -0
- package/md_cg/test_issue375_state_slot_retire.py +325 -0
- package/md_cg/test_issue376_heartbeat_failed.py +293 -0
- package/md_cg/test_issue377_layer_protect_stamp.py +324 -0
- package/md_cg/test_issue378_protected_floor.py +310 -0
- package/md_cg/test_issue383_state_event_write_gate.py +462 -0
- package/md_cg/test_issue397_sens_eff_wiring.py +435 -0
- package/md_cg/test_issue398_toolface_op_parity.py +511 -0
- package/md_cg/test_issue399_hive_json_shape.py +317 -0
- package/md_cg/test_issue400_hive_dirty_args.py +450 -0
- package/md_cg/test_issue401_wisdom_codegen.py +295 -0
- package/md_cg/test_issue404_heal_receipt.py +390 -0
- package/md_cg/test_issue405_git_pathspec.py +333 -0
- package/md_cg/test_issue408_eface_registry.py +291 -0
- package/md_cg/test_issue409_compact_index_completeness.py +509 -0
- package/md_cg/test_issue410_verify_demote_write.py +366 -0
- package/md_cg/test_issue411_scrub_idempotence.py +575 -0
- package/md_cg/test_issue412_discipline_producer_parity.py +312 -0
- package/md_cg/test_issue413_preimage_failclosed.py +239 -0
- package/md_cg/test_issue413_preimage_twin.py +340 -0
- package/md_cg/test_issue414_write_args_parity.py +613 -0
- package/md_cg/test_issue415_deps_gate_probe.py +358 -0
- package/md_cg/test_issue416_entry_key_subset.py +525 -0
- package/md_cg/test_issue417_twophase_merge_landed.py +423 -0
- package/md_cg/test_issue418_identity_role_type_gate.py +188 -0
- package/md_cg/test_issue418_role_type_gate.py +265 -0
- package/md_cg/test_issue419_pii_redaction.py +642 -0
- package/md_cg/test_issue424_merge_confirm_sensitivity.py +381 -0
- package/md_cg/test_issue425_ingest_library_path.py +331 -0
- package/md_cg/test_issue426a_dedup_receipt.py +242 -0
- package/md_cg/test_issue426b_root_clash_invisible.py +270 -0
- package/md_cg/test_issue426c_enqueue_not_silent.py +346 -0
- package/md_cg/test_issue426e_restore_overwrite_live.py +310 -0
- package/md_cg/test_issue426f_paths_gbk_failclosed.py +301 -0
- package/md_cg/test_issue427_serve_status_shape.py +191 -0
- package/md_cg/test_issue431_432_468_protect_single_point.py +322 -0
- package/md_cg/test_issue433_outlook_threshold_single_point.py +278 -0
- package/md_cg/test_issue434_435_float_gate.py +90 -0
- package/md_cg/test_issue439_sibling_sensitivity_inherit.py +448 -0
- package/md_cg/test_issue43_default_policy.py +874 -865
- package/md_cg/test_issue443a_state_event_length_gate.py +94 -0
- package/md_cg/test_issue443b_audit_record_length_gate.py +83 -0
- package/md_cg/test_issue443c_self_state_imgskill_length_gate.py +77 -0
- package/md_cg/test_issue444_scrub_evidence_plaintext.py +281 -0
- package/md_cg/test_issue445_metacognition_ledger_gate.py +454 -0
- package/md_cg/test_issue446_453_403_verified_evidence.py +382 -0
- package/md_cg/test_issue446_is_encrypted_envelope.py +166 -0
- package/md_cg/test_issue447_recent_sens_failclosed.py +270 -0
- package/md_cg/test_issue450_sensitivity_fallback_reason.py +51 -0
- package/md_cg/test_issue450_writegate_unknown_sens.py +203 -0
- package/md_cg/test_issue451_audit_actor.py +313 -0
- package/md_cg/test_issue452_ingest_root_hint.py +238 -0
- package/md_cg/test_issue452_ingest_root_sensitivity.py +299 -0
- package/md_cg/test_issue463_ghost_prune_guard.py +259 -0
- package/md_cg/test_issue466_basis_registry.py +469 -0
- package/md_cg/test_issue467_identity_sensitivity.py +204 -0
- package/md_cg/test_issue467_identity_sensitivity_parity.py +361 -0
- package/md_cg/test_issue469_scrub_override_passthrough.py +126 -0
- package/md_cg/test_issue470_wal_event_durable.py +427 -0
- package/md_cg/test_issue471_identity_readable.py +137 -0
- package/md_cg/test_issue472_audit_chain.py +91 -0
- package/md_cg/test_issue472_prune_ledger.py +106 -0
- package/md_cg/test_issue472_rollback_degraded.py +82 -0
- package/md_cg/test_issue491_compose_sandbox.py +460 -0
- package/md_cg/test_issue492_dup_readout_opt_in.py +292 -0
- package/md_cg/test_issue498_merge_trace_stale_index.py +280 -0
- package/md_cg/test_issue511_scoreboard_bad_files.py +256 -0
- package/md_cg/test_issue521_rotate_lock_suffix.py +110 -0
- package/md_cg/test_issue522_review_record_sensitivity.py +267 -0
- package/md_cg/test_issue524_orphan_noise_no_demote.py +88 -0
- package/md_cg/test_issue525_read_null_semantics.py +52 -0
- package/md_cg/test_issue538_hook_condition_space.py +97 -0
- package/md_cg/test_issue540_path_guard_whitespace.py +126 -0
- package/md_cg/test_issue541_tidy_protect_gate.py +141 -0
- package/md_cg/test_issue542_tenant_root_same.py +153 -0
- package/md_cg/test_issue543_issue_authorization.py +147 -0
- package/md_cg/test_issue544_sandbox_reflection_gate.py +175 -0
- package/md_cg/test_issue545_charset_vs_size_diagnostic.py +77 -0
- package/md_cg/test_issue546_ledger_concurrency.py +81 -0
- package/md_cg/test_issue547_append_jsonl_whitelist.py +115 -0
- package/md_cg/test_issue547_audit_durable.py +96 -0
- package/md_cg/test_issue549_durable_append.py +143 -0
- package/md_cg/test_issue557_r1_credential_value_forms.py +76 -0
- package/md_cg/test_issue557_r2_write_entries_size_gate.py +84 -0
- package/md_cg/test_issue557_r3_pem_block_redaction.py +75 -0
- package/md_cg/test_issue559_refine_prefix.py +70 -0
- package/md_cg/test_issue561_pending_review.py +109 -0
- package/md_cg/test_issue562_retired_faces.py +150 -0
- package/md_cg/test_issue66_scrub_semantics.py +107 -0
- package/md_cg/test_issue67_scrub_control_pool.py +233 -0
- package/md_cg/test_issue69_audit_boundary.py +129 -0
- package/md_cg/test_issue71_queue_backlog.py +377 -0
- package/md_cg/test_issue72_coldverify_controlled_drain.py +420 -0
- package/md_cg/test_issue73_doc_chapter_refs.py +326 -0
- package/md_cg/test_issue75_neural_failure.py +582 -0
- package/md_cg/test_issue78_export_concurrency.py +334 -0
- package/md_cg/test_issue80_residual_flush.py +169 -0
- package/md_cg/test_issue82_credential_rules.py +271 -0
- package/md_cg/test_issue82_layer_and_size.py +120 -0
- package/md_cg/test_issue84_portability.py +95 -0
- package/md_cg/test_issue85_export_overwrite.py +113 -0
- package/md_cg/test_issue86_runtime_revoke.py +110 -0
- package/md_cg/test_issue87_default_sensitivity_single.py +201 -0
- package/md_cg/test_issue88_conclusion_face.py +280 -0
- package/md_cg/test_issue88_execution_conflicts.py +87 -0
- package/md_cg/test_issue89_zero_score.py +169 -0
- package/md_cg/test_issue89a_score_cap_tiebreak.py +279 -0
- package/md_cg/test_issue91_verifier_sandbox.py +314 -0
- package/md_cg/test_issue91_verifier_timeout.py +528 -0
- package/md_cg/test_issue93_remember_redaction.py +282 -0
- package/md_cg/test_issue96_key_fallback.py +102 -0
- package/md_cg/test_issue98_tenant_root.py +104 -0
- package/md_cg/test_jsonl_commit_watermark.py +221 -0
- package/md_cg/test_live_validity_cache.py +256 -0
- package/md_cg/test_logref.py +1141 -1109
- package/md_cg/test_mcp_call_argument_types.py +99 -0
- package/md_cg/test_md_access_parity.py +330 -255
- package/md_cg/test_md_cg_no_hive_dep.py +140 -0
- package/md_cg/test_memory_rsi.py +215 -0
- package/md_cg/test_memory_rsi_adversarial.py +91 -0
- package/md_cg/test_mode_parity.py +1413 -1393
- package/md_cg/test_mutation_residue.py +120 -0
- package/md_cg/test_mutation_rollback.py +835 -805
- package/md_cg/test_n123_derive_expiry_chain.py +210 -205
- package/md_cg/test_n130_verify_falsified_protect.py +185 -185
- package/md_cg/test_n209_verify_write_face_gates.py +466 -461
- package/md_cg/test_n230_dirty_replay.py +52 -9
- package/md_cg/test_n62_tenant_bind_failclosed.py +200 -200
- package/md_cg/test_neg_coverage_budget.py +500 -0
- package/md_cg/test_neg_tail_honesty.py +733 -663
- package/md_cg/test_p0.py +16 -2
- package/md_cg/test_p14_root_fallback.py +373 -0
- package/md_cg/test_p1x_ref_root.py +47 -10
- package/md_cg/test_p21_tokens.py +294 -293
- package/md_cg/test_p24_evidence.py +232 -227
- package/md_cg/test_p26_refindex.py +20 -3
- package/md_cg/test_p27_docindex.py +792 -777
- package/md_cg/test_p28_refcheck.py +1053 -1032
- package/md_cg/test_p29_session_ingest_export.py +6 -0
- package/md_cg/test_p2_mcp.py +474 -453
- package/md_cg/test_p2_six_elements.py +476 -463
- package/md_cg/test_p3.py +370 -261
- package/md_cg/test_p33_ccg_wiring.py +300 -293
- package/md_cg/test_p3_legacy_closure.py +438 -433
- package/md_cg/test_p40_refine_worklist.py +244 -241
- package/md_cg/test_p47_session_view.py +25 -9
- package/md_cg/test_p4_freshness.py +781 -774
- package/md_cg/test_p83_jsonrpc_parse_error.py +124 -0
- package/md_cg/test_p9c_dedup_hints.py +288 -283
- package/md_cg/test_policy_required_ccg.py +444 -432
- package/md_cg/test_postings_digest_memo.py +139 -0
- package/md_cg/test_predict_beta.py +138 -135
- package/md_cg/test_qmd.py +500 -0
- package/md_cg/test_query_excerpt.py +209 -0
- package/md_cg/test_reach.py +456 -378
- package/md_cg/test_reach_snapshot_admission.py +218 -0
- package/md_cg/test_readcache_prodpath.py +4 -0
- package/md_cg/test_recall_face_guards.py +854 -812
- package/md_cg/test_recall_metadata.py +422 -0
- package/md_cg/test_retr_s5.py +312 -213
- package/md_cg/test_retr_score_once.py +14 -2
- package/md_cg/test_review_cli_attribution.py +11 -7
- package/md_cg/test_review_queue_batch.py +123 -0
- package/md_cg/test_review_queue_index.py +130 -0
- package/md_cg/test_rrf_candidates.py +292 -0
- package/md_cg/test_rrf_qualified_topk.py +214 -0
- package/md_cg/test_rrf_unique_votes.py +218 -0
- package/md_cg/test_s1_ast_judge.py +110 -0
- package/md_cg/test_scrub_ledger.py +413 -0
- package/md_cg/test_security_audit_b26.py +34 -3
- package/md_cg/test_security_audit_v21.py +1 -2
- package/md_cg/test_sensitivity_predicate_single_point.py +184 -0
- package/md_cg/test_session5_identity.py +93 -0
- package/md_cg/test_session_attribution_consistency.py +813 -0
- package/md_cg/test_session_identity_tristate.py +486 -0
- package/md_cg/test_sleep_p1.py +35 -3
- package/md_cg/test_state_event_op.py +521 -514
- package/md_cg/test_state_projection_streaming.py +136 -0
- package/md_cg/test_stg_interval_candidates.py +280 -0
- package/md_cg/test_stg_time_blocks.py +239 -0
- package/md_cg/test_stgidx_index_parity.py +1024 -1020
- package/md_cg/test_subgraph_bounded_bfs.py +227 -0
- package/md_cg/test_sustain_bounded.py +652 -624
- package/md_cg/test_task_memory.py +394 -0
- package/md_cg/test_text_projection_mutations.py +101 -0
- package/md_cg/test_time_anchor.py +421 -0
- package/md_cg/test_time_core_lint.py +1024 -975
- package/md_cg/test_tool_face.py +200 -189
- package/md_cg/test_trust_bounded_frontier.py +248 -0
- package/md_cg/test_trust_edge_reachability.py +405 -0
- package/md_cg/test_w5_path_fingerprint.py +381 -0
- package/md_cg/test_w7_redlines.py +659 -0
- package/md_cg/test_write_return_shape.py +919 -0
- package/md_cg/test_writepipe.py +228 -226
- package/md_cg/timeanchor.py +534 -0
- package/md_cg/tokens.py +68 -5
- package/md_cg/tool_face.py +273 -260
- package/md_cg/trust.py +1114 -1030
- package/md_cg/twophase.py +303 -231
- package/md_cg/weights.py +642 -638
- package/md_cg/whitebox.py +536 -527
- package/md_cg/whitebox_kb/aeis_core/core.py +5026 -5026
- package/md_cg/whitebox_kb/wisdom/chat_engine.py +1504 -1506
- package/md_cg/whitebox_kb/wisdom/code_compose.py +876 -837
- package/md_cg/whitebox_kb/wisdom/compiler_code_units.py +3040 -3033
- package/md_cg/whitebox_kb/wisdom/compose_engine.py +1634 -1633
- package/md_cg/whitebox_kb/wisdom/core/condition_vm.py +318 -298
- package/md_cg/whitebox_kb/wisdom/core/parser.py +1082 -1065
- package/md_cg/whitebox_kb/wisdom/csre.py +412 -264
- package/md_cg/whitebox_kb/wisdom/csre_session.py +97 -0
- package/md_cg/whitebox_kb/wisdom/md_access.py +490 -470
- package/md_cg/whitebox_kb/wisdom/migrate_wisdom.py +497 -476
- package/md_cg/whitebox_kb/wisdom/mini_python.py +995 -958
- package/md_cg/whitebox_kb/wisdom/navigate.py +249 -248
- package/md_cg/whitebox_kb/wisdom/net_units.py +2900 -2895
- package/md_cg/whitebox_kb/wisdom/neural_retrieve.py +100 -10
- package/md_cg/whitebox_kb/wisdom/semantic_translate.py +4060 -4054
- package/md_cg/whitebox_kb/wisdom/test_ecosystem_assembly.py +158 -158
- package/md_cg/whitebox_kb/wisdom/test_safe_module_proxy.py +147 -0
- package/md_cg/whitebox_kb/wisdom/verifier.py +1918 -1297
- package/md_cg/writelimit.py +584 -390
- package/md_cg/writepipe.py +1337 -826
- package/package.json +14 -9
- package/scripts/workspace_index.py +328 -0
- package/src/bridge.ts +34 -11
- package/src/cli.ts +75 -65
- package/src/hooks.ts +932 -776
- package/src/index.ts +30 -7
- package/src/init.ts +362 -361
- package/src/lib/datapath.ts +419 -326
- package/src/lib/hook_audit.ts +42 -3
- package/src/lib/mdcg_client.ts +469 -461
- package/src/lib/mutual.ts +481 -432
- package/src/lib/prompt_safety.ts +178 -149
- package/src/lib/roleplay_web.ts +30 -2
- package/src/lib/token_store.ts +274 -52
- package/src/tools.ts +271 -223
- package/zcode/README.md +1 -1
- package/docs/GBrain/345/217/257/345/200/237/351/211/264/347/202/271_/347/201/265/346/236/242/350/220/275/347/202/271/344/272/244/346/216/245_20260919.md +0 -169
- package/docs/Pi/345/217/257/345/255/246/344/271/240/344/274/230/347/202/271_/347/201/265/346/236/242/345/244/247/350/204/221/346/224/271/350/277/233/344/272/244/346/216/245_20260915.md +0 -144
- package/docs/eval/AGI/344/270/203/347/273/264/350/257/204/345/210/206/346/212/245/345/221/212_md_cg_v1.0.md +0 -299
- package/docs/eval/AGI/344/270/203/347/273/264/350/257/204/345/210/206/346/212/245/345/221/212_md_cg_v2.0.md +0 -239
- package/docs/eval/AGI/344/270/203/347/273/264/350/257/204/345/210/206/346/212/245/345/221/212_/345/205/255/345/256/266/350/256/260/345/277/206/347/263/273/347/273/237/345/220/214/345/260/272/345/256/236/346/265/213_v3.1.md +0 -340
- package/docs/eval/DSH/346/227/245/345/277/227/347/264/242/345/274/225v2_/345/217/202/350/200/203dsh-TUI_v1.0.md +0 -248
- package/docs/eval/DSH/346/227/245/345/277/227/347/264/242/345/274/225/346/225/210/346/236/234/351/252/214/350/257/201_v1.0.md +0 -209
- package/docs/eval/DSH/347/253/257/347/274/272/351/231/267/344/270/223/351/241/271_v1.0.md +0 -254
- package/docs/eval/N225_/347/264/242/345/274/225/346/227/245/345/277/227/351/235/236/345/257/271/350/261/241/350/243/205/350/275/275/351/235/242/347/261/273/345/236/213/351/227/270_v1.0.md +0 -418
- package/docs/eval/P1b2_/350/257/273/351/235/242/344/273/243/351/231/205/344/277/256/345/244/215_v1.0.md +0 -124
- package/docs/eval/bench_lingshu_self/bench_self.py +0 -140
- package/docs/eval/bench_lingshu_self/self_bench_result.json +0 -404
- package/docs/eval/bench_lingshu_self//347/201/265/346/236/242/350/207/252/345/272/223/347/253/257/345/210/260/347/253/257/346/243/200/347/264/242/345/256/236/346/265/213_v1.0.md +0 -38
- package/docs/eval/cons200_/345/206/262/347/252/201/346/243/200/346/265/213/351/200/211/351/235/242_/345/256/236/346/226/275/350/256/260/345/275/225_v1.0.md +0 -223
- package/docs/eval/cons200_/345/206/262/347/252/201/346/243/200/346/265/213/351/200/211/351/235/242_/345/256/236/346/226/275/350/256/260/345/275/225_v1.1.md +0 -340
- package/docs/eval/issue43_/351/273/230/350/256/244/347/255/226/347/225/245/344/270/216/351/224/256/347/261/273/345/236/213/351/227/270_v1.1.md +0 -441
- package/docs/eval/issue43_/351/273/230/350/256/244/347/255/226/347/225/245/345/212/240/350/275/275/344/270/216/345/207/255/346/215/256/346/230/216/346/226/207/351/230/237/345/210/227_v1.0.md +0 -398
- package/docs/eval/issue50_/345/205/203/346/225/260/346/215/256/351/200/217/344/274/240/344/270/216/345/205/234/345/272/225_/344/277/256/345/244/215/350/256/260/345/275/225_v1.0.md +0 -437
- package/docs/eval/issue50_/345/215/212/351/207/215/345/244/215/345/276/205/345/256/232/345/244/215/346/240/270_/344/277/256/345/244/215/350/256/260/345/275/225_v1.0.md +0 -488
- package/docs/eval/issue50_/345/276/205/345/256/232/345/244/215/346/240/270/345/205/245/351/230/237_/344/277/256/345/244/215/350/256/260/345/275/225_v1.0.md +0 -428
- package/docs/eval/issue50_/350/257/273/351/235/242/344/277/235/346/212/244/345/217/252/350/256/244/346/230/276/345/274/217/346/235/245/346/272/220_/344/277/256/345/244/215/350/256/260/345/275/225_v1.0.md +0 -271
- package/docs/eval/issue50_/351/207/215/350/246/201/345/272/246/345/220/214/346/272/220/344/270/216/344/277/235/346/212/244/350/257/255/344/271/211_/344/277/256/345/244/215/350/256/260/345/275/225_v1.0.md +0 -306
- package/docs/eval/issue51_/344/270/200/351/224/256/345/256/211/350/243/205/345/244/261/350/264/245_/345/275/222/345/261/236/345/210/244/345/256/232_v1.0.md +0 -58
- package/docs/eval/issue52_/346/235/241/344/273/266/345/205/210/350/241/214/344/270/216/346/210/252/346/226/255/345/217/257/350/247/202/346/265/213_/344/277/256/345/244/215/350/256/260/345/275/225_v1.0.md +0 -360
- package/docs/eval/issue64_replay_check_/346/240/270/351/252/214/344/270/216/344/277/256/345/244/215_v1.0.md +0 -284
- package/docs/eval/issue65_session_recall_/344/274/232/350/257/235/351/232/224/347/246/273_/346/240/270/351/252/214/344/270/216/344/277/256/345/244/215_v1.0.md +0 -329
- package/docs/eval//344/270/211/346/241/243/350/207/252/346/262/273_/346/255/245/351/252/244/342/221/241/346/241/243/344/275/215/345/215/225/344/270/200/345/205/245/345/217/243_/350/220/275/347/240/201/350/256/260/345/275/225_v1.0.md +0 -651
- package/docs/eval//344/270/211/346/241/243/350/207/252/346/262/273_/346/255/245/351/252/244/342/221/242/345/217/230/346/233/264/345/215/225/344/270/216/345/233/236/346/273/232/345/216/237/350/257/255_/350/220/275/347/240/201/350/256/260/345/275/225_v1.0.md +0 -439
- package/docs/eval//344/270/211/346/241/243/350/207/252/346/262/273_/346/255/245/351/252/244/342/221/243/345/207/206/345/205/245/350/257/273/346/225/260_/350/220/275/347/240/201/350/256/260/345/275/225_v1.0.md +0 -1337
- package/docs/eval//344/270/211/346/241/243/350/207/252/346/262/273_/346/255/245/351/252/244/342/221/244/346/224/266/345/256/230/344/270/216/345/205/250/351/223/276/351/252/214/346/224/266_/350/220/275/347/240/201/350/256/260/345/275/225_v1.0.md +0 -1817
- package/docs/eval//344/270/215/345/217/257/351/235/240/346/200/247/350/220/275/345/234/260_P0_v1.0.md +0 -185
- package/docs/eval//344/274/230/345/214/226/347/254/254/344/270/200/346/211/271_/346/216/245/347/272/277/344/270/216/347/255/211/344/273/267/345/217/230/346/215/242_v1.0.md +0 -498
- package/docs/eval//344/274/230/345/214/226/347/254/254/344/270/211/346/211/271_/351/227/250/347/246/201/350/275/254/346/255/243/344/270/216/350/260/203/345/272/246/346/255/242/350/241/200/344/270/216/351/227/250/346/216/247/346/224/266/345/217/243_v1.0.md +0 -669
- package/docs/eval//344/274/230/345/214/226/347/254/254/344/272/214/346/211/271_/344/276/235/350/265/226/351/200/217/344/274/240/344/270/216/345/257/271/346/213/215/345/217/243/345/276/204/344/270/216/350/264/237/347/274/223/345/255/230_v1.0.md +0 -683
- package/docs/eval//345/207/272/350/264/247/351/235/242/345/206/222/347/203/237_/350/277/233/350/264/247/351/227/250/347/246/201_v1.0.md +0 -460
- package/docs/eval//345/217/221/345/270/20306_/344/270/200/351/224/256/351/205/215/347/275/256/344/270/216DSH0172_v1.0.md +0 -171
- package/docs/eval//345/217/221/345/270/20307_/345/244/226/351/203/250/346/212/245/345/221/212/345/233/233/346/211/271/344/277/256/345/244/215/344/270/216/346/217/222/344/273/266/351/235/242/345/212/240/345/233/272_v1.0.md +0 -209
- package/docs/eval//345/217/221/345/270/20308_/350/207/252/350/277/255/344/273/243/344/270/216/347/235/241/347/234/240_/345/233/276/346/243/200/347/264/242/350/267/257/344/270/216/346/235/203/351/207/215_v1.0.md +0 -97
- package/docs/eval//345/217/221/345/270/20309_/346/243/200/347/264/242/351/235/242/344/270/211/346/211/271/346/224/266/345/217/243_v1.0.md +0 -66
- package/docs/eval//345/217/221/345/270/20310_/350/257/273/351/235/242/346/215/237/345/235/217UTF8/345/256/266/346/227/217/346/224/266/345/217/243_v1.0.md +0 -71
- package/docs/eval//345/217/221/345/270/20311_/345/244/226/346/212/245/345/205/255/350/277/236/344/277/256/344/270/216/344/270/226/347/225/214/346/250/241/345/236/213/345/272/225/345/272/247_v1.0.md +0 -91
- package/docs/eval//345/217/221/345/270/20312_/344/270/226/347/225/214/346/250/241/345/236/213/345/212/237/350/203/275/347/253/257_v1.0.md +0 -108
- package/docs/eval//345/217/221/345/270/20313_/344/270/212/344/270/213/346/226/207/350/207/252/347/256/241/347/220/206/346/234/272/345/210/266_v1.0.md +0 -114
- package/docs/eval//345/217/221/345/270/20314_/350/272/253/344/275/223/303/227/350/204/221/347/273/204/345/220/210_v1.0.md +0 -160
- package/docs/eval//345/217/221/345/270/20315_/344/274/232/350/257/235/351/232/224/347/246/273/344/277/256/345/244/215/344/270/216/350/272/253/344/275/223/345/272/223/345/261/225/347/244/272_v1.0.md +0 -88
- package/docs/eval//345/256/236/351/252/214/346/226/271/346/241/210_/345/255/246/344/271/240/351/227/255/347/216/257AB/344/270/216/346/250/252/350/257/204_v1.0.md +0 -174
- package/docs/eval//345/275/222/344/270/200/345/261/202/347/274/272/347/234/201/347/277/273/345/205/263_/344/277/256/345/244/215/350/256/260/345/275/225_v1.0.md +0 -458
- package/docs/eval//346/200/247/350/203/275/344/270/223/351/241/271_/345/206/267/346/237/245/350/257/242/344/270/216/345/206/205/345/255/230_v1.0.md +0 -218
- package/docs/eval//346/225/205/351/232/234/346/263/250/345/205/245/345/256/236/346/265/213_v1.0.md +0 -432
- package/docs/eval//346/250/252/350/257/204_/345/205/255/345/256/266100/351/242/230/344/270/255/350/213/261/345/217/214/346/237/245_v1.0.md +0 -223
- package/docs/eval//347/224/250/346/210/267/350/243/201/345/206/263/345/244/204/347/275/256_/347/254/254/344/270/200/346/211/271_v1.0.md +0 -195
- package/docs/eval//347/224/250/346/210/267/350/243/201/345/206/263/345/244/204/347/275/256_/347/254/254/344/272/214/346/211/271B6_/346/234/254/346/234/272/350/267/257/345/276/204/347/233/270/345/257/271/345/214/226_v1.0.md +0 -78
- package/docs/eval//347/253/257/345/210/260/347/253/257LoCoMoQA/345/220/214/345/217/243/345/276/204/345/257/271/347/205/247_v1.0.md +0 -100
- package/docs/eval//347/253/257/345/210/260/347/253/257/345/271/262/346/211/260/346/261/240/350/257/204/346/265/213_/347/241/256/345/256/232/346/200/247/350/243/201/345/206/263vsLLM_judge_v1.1.md +0 -197
- package/docs/eval//347/254/2543/345/261/202stg/347/273/223/346/236/204/347/264/242/345/274/225_/345/256/236/346/226/275/350/256/260/345/275/225_v1.0.md +0 -250
- package/docs/eval//347/254/254/344/270/211/346/226/271/351/252/214/350/257/201/346/212/245/345/221/212_LoCoMo_/347/201/265/346/236/242_.png +0 -0
- package/docs/eval//347/254/254/344/270/211/346/226/271/351/252/214/350/257/201/346/212/245/345/221/212_/347/201/265/346/236/242_vs_dejavu_/347/273/237/344/270/200/350/257/204/345/210/206_v7.md +0 -202
- package/docs/eval//347/274/226/347/240/201/351/235/242/345/211/215/347/275/256_/345/205/245/345/217/243/350/207/252/344/277/235/350/257/201UTF8/344/270/216/346/226/207/346/234/254open/345/256/210/345/215/253_v1.0.md +0 -646
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v1.md +0 -156
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v10.md +0 -210
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v11.md +0 -227
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v12.md +0 -203
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v13.md +0 -233
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v14.md +0 -191
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v15.md +0 -213
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v16.md +0 -214
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v17.md +0 -199
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v18.md +0 -183
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v19.md +0 -207
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v2.md +0 -156
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v20.md +0 -283
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v21.md +0 -224
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v22.md +0 -223
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v23.md +0 -293
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v24.md +0 -303
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v25.md +0 -230
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v26.md +0 -196
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v27.md +0 -259
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v28.md +0 -217
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v29.md +0 -270
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v3.md +0 -152
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v30.md +0 -231
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v31.md +0 -281
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v4.md +0 -128
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v5.md +0 -114
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v6.md +0 -192
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v7.md +0 -187
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v8.md +0 -207
- package/docs/eval//347/274/272/351/231/267/346/214/226/346/216/230_/350/207/252/344/270/273/350/277/255/344/273/243_v9.md +0 -203
- package/docs/eval//350/257/255/344/271/211/346/227/266/347/251/272/345/233/276/350/241/245/345/205/250_P1_/346/212/275/345/217/226/345/231/250v4_/344/270/211/347/216/207/350/257/273/346/225/260_v1.0.md +0 -231
- package/docs/eval//351/262/270/345/250/230/344/270/203/347/261/273/347/212/266/346/200/201/350/277/275/350/270/252_v1.0.md +0 -176
- package/docs/hive//344/270/273/344/273/243/347/220/206/345/255/220/344/273/243/347/220/206/350/256/260/345/277/206/346/236/266/346/236/204/350/256/276/350/256/241.md +0 -385
- package/docs/hive//344/273/244/347/211/214/344/270/216/350/247/222/350/211/262/346/235/203/350/201/214/345/210/206/347/246/273_v0.1.md +0 -165
- package/docs/hive//344/273/244/347/211/214/350/257/255/344/271/211/344/277/256/346/255/243_/344/273/262/350/243/201/344/275/215_v0.1.md +0 -76
- package/docs/hive//345/244/232/347/253/257harness/351/200/232/344/277/241/345/245/221/347/272/246_v0.1.md +0 -42
- package/docs/hive//345/255/220/344/273/243/347/220/206/351/205/215/347/275/256/346/240/207/345/207/206_v0.5.md +0 -236
- package/docs/hive//345/256/211/345/205/250/345/256/241/350/256/241/345/256/236/351/224/232_v0.1.md +0 -202
- package/docs/hive//346/243/200/347/264/242/346/224/266/346/225/233/345/256/236/346/265/213/344/270/216S1b/350/256/276/350/256/241_v0.1.md +0 -43
- package/docs/hive//346/243/200/347/264/242/347/256/227/346/263/225/345/217/243/345/276/204/345/257/271/347/205/247_v0.1.md +0 -210
- package/docs/hive//346/243/200/347/264/242/350/267/257/345/276/204/344/270/216/350/256/244/347/237/245/347/273/223/346/236/204/345/245/221/347/272/246_v0.1.md +0 -152
- package/docs/hive//350/234/202/345/267/242M6_ingest/345/256/236/346/226/275/350/256/241/345/210/222_v0.1.md +0 -47
- package/docs/hive//350/234/202/345/267/242/345/217/214/345/256/236/344/276/213/344/272/222/351/252/214_/350/256/276/350/256/241/345/256/232/347/250/277.md +0 -519
- package/docs/hive//350/234/202/345/267/242/345/267/245/344/275/234/350/256/260/345/277/206_/351/241/271/347/233/256/350/256/241/345/210/222.md +0 -85
- package/docs/hive//350/234/202/345/267/242/350/256/276/350/256/241_/347/220/206/350/256/272/345/257/271/351/275/220_v0.1.md +0 -191
- package/docs/hive//350/234/202/345/267/242/350/277/255/344/273/243_/345/256/217/350/247/202/344/270/216/347/276/244/344/275/223/350/260/203/345/272/246_v0.1.md +0 -90
- package/docs/images/lingshu-moonlight-covenant-poster-preview.jpg +0 -0
- package/docs/images/lingshu-moonlight-covenant-poster.png +0 -0
- package/docs/mdcg/D_meta_/345/267/245/347/250/213/345/214/226/346/226/271/346/241/210_v0.2.md +0 -216
- package/docs/mdcg/README/350/257/246/347/273/206/347/211/210_v0.4.10.md +0 -746
- package/docs/mdcg/guardrail-charter.md +0 -164
- package/docs/mdcg/lingshu_tutorial.html +0 -14956
- package/docs/mdcg/memory-assessment.html +0 -923
- package/docs/mdcg/memory_score.html +0 -213
- package/docs/mdcg/memory_score.png +0 -0
- package/docs/mdcg/release_v0.3.0.md +0 -36
- package/docs/mdcg/release_v0.4.11.md +0 -49
- package/docs/mdcg/release_v0.4.5.md +0 -55
- package/docs/mdcg/tool_table_v0.3.0.md +0 -117
- package/docs/mdcg//344/270/226/347/225/214/346/250/241/345/236/213/345/212/237/350/203/275/347/253/257_/344/275/277/347/224/250/344/270/216/350/277/220/347/273/264_v1.0.md +0 -384
- package/docs/mdcg//344/273/243/347/240/201/350/257/204/345/256/241/344/270/216/346/235/241/344/273/266/345/214/226/346/263/250/351/207/212_/345/245/221/347/272/246_v0.1.md +0 -82
- package/docs/mdcg//345/205/250/345/272/223/344/273/243/347/240/201/350/257/204/345/256/241/344/270/216/346/235/241/344/273/266/345/214/226/346/263/250/351/207/212_/350/256/241/345/210/222_v0.1.md +0 -600
- package/docs/mdcg//345/212/237/350/203/275/350/260/203/347/224/250/346/230/240/345/260/204/350/241/250_v0.1.md +0 -266
- package/docs/mdcg//345/215/225/345/205/203/350/207/252/346/210/221/351/224/232/347/202/271_/347/263/273/347/273/237/346/217/220/347/244/272/350/257/215/346/240/207/345/207/206_v0.3.md +0 -275
- package/docs/mdcg//345/217/221/345/270/203/351/227/250/347/246/201/351/223/276_v0.1.md +0 -134
- package/docs/mdcg//346/272/220/347/240/201/347/272/247/346/236/266/346/236/204/345/256/241/350/256/241_GPT/346/211/271/350/257/204/345/257/271/347/205/247_v1.0.md +0 -130
- package/docs/mdcg//347/201/265/346/236/24282/345/267/245/345/205/267_/345/212/237/350/203/275/346/225/264/347/220/206/344/270/216/350/277/201/347/247/273/346/230/240/345/260/204_v0.1.md +0 -278
- package/docs/mdcg//347/201/265/346/236/242MCP/345/267/245/345/205/267/346/200/273/350/241/250_v3.4.md +0 -164
- package/docs/mdcg//347/201/265/346/236/242_/350/207/252/346/210/221/345/261/202/345/256/232/344/271/211.md +0 -43
- package/docs/mdcg//347/201/265/346/236/242/344/270/211/345/261/202/346/213/206/345/210/206/350/247/204/345/210/222_v0.1.md +0 -181
- package/docs/mdcg//347/201/265/346/236/242/350/256/260/345/277/206/345/212/250/350/257/215/345/215/217/350/256/256_v1.0-draft.md +0 -177
- package/docs/mdcg//347/216/257/344/272/214_/347/231/275/347/256/261/345/241/253/345/205/205/346/265/201/346/260/264/347/272/277_v0.1.md +0 -31
- package/docs/mdcg//347/235/241/347/234/240/345/221/250/346/234/237_/350/277/220/347/273/264/345/211/215/346/217/220/344/270/216/347/273/264/346/212/244/346/214/207/345/215/227_v1.0.md +0 -337
- package/docs/mdcg//347/274/272/345/217/243/345/215/225_P0/346/224/266/345/217/243_v0.1.md +0 -270
- package/docs/mdcg//350/256/244/347/237/245/345/233/276_G4-G8/347/274/272/345/217/243/350/243/201/345/256/232/345/215/225_v0.1.md +0 -491
- package/docs/mdcg//350/256/244/347/237/245/345/233/276_MD/347/233/256/345/275/225/346/226/271/346/241/210_v0.1.md +0 -552
- package/docs/mdcg//350/256/244/347/237/245/345/233/276_/346/235/241/344/273/266/347/251/272/351/227/264/345/220/210/346/210/220/344/270/216/347/224/237/346/225/210/346/235/241/344/273/266/345/217/243/345/276/204_v0.1.md +0 -340
- package/docs/mdcg//350/256/244/347/237/245/345/233/276_/347/264/242/345/274/225/344/270/216/345/267/245/347/250/213/350/247/204/350/214/203/345/214/226_/350/256/241/345/210/222_v0.1.md +0 -624
- package/docs/mdcg//350/256/244/347/237/245/345/233/276/344/275/234/344/270/272/350/256/260/345/277/206/346/223/215/344/275/234/347/263/273/347/273/237_/350/257/204/344/274/260/344/270/216/350/267/257/347/272/277_v0.1.md +0 -94
- package/docs/mdcg//350/256/260/345/277/206/346/223/215/344/275/234/347/263/273/347/273/237_MdCGOS/344/270/216MCP/346/216/245/345/205/245_v0.1.md +0 -281
- package/docs/mdcg//350/267/250/347/253/257/351/252/214/350/257/201/344/270/216/345/220/214/346/255/245/345/215/217/350/256/256_v0.1.md +0 -93
- package/docs/plans/GridWorld/346/234/200/345/260/217/351/227/255/347/216/257/350/247/204/346/240/274_v0.1.md +0 -176
- package/docs/plans/stg/346/235/241/344/273/266/345/214/226/344/270/216/347/273/223/346/236/204/347/264/242/345/274/225_/350/256/276/350/256/241_v0.1.md +0 -124
- package/docs/plans//345/205/250/344/270/255/346/226/207/347/274/226/347/240/201/344/270/216/350/234/202/345/267/242/344/273/273/345/212/241/346/240/207/350/257/206/345/245/221/347/272/246_v2.0.md +0 -220
- package/docs/plans//345/221/275/345/220/215/346/262/273/347/220/206_/351/241/271/347/233/256/350/256/241/345/210/222.md +0 -81
- package/docs/plans//345/244/232/344/270/273/344/275/223/344/270/226/347/225/214/346/250/241/345/236/213_/345/257/271/351/275/220/350/257/204/344/274/260/344/270/216/350/220/275/345/234/260/350/256/276/350/256/241_v0.1.md +0 -141
- package/docs/plans//347/201/265/346/236/242/350/272/253/344/275/223/303/227/350/204/221_/344/270/226/347/225/214/346/250/241/345/236/213/345/257/271/346/216/245/350/256/276/350/256/241_v0.1.md +0 -119
- package/docs/plans//347/234/237/346/272/220/347/264/242/345/274/225_/351/200/232/347/224/250/346/234/272/345/210/266_v0.3.md +0 -227
- package/docs/plans//347/235/241/347/234/240/344/270/216/350/207/252/350/277/255/344/273/243_/345/212/237/350/203/275/344/274/230/345/214/226/350/256/276/350/256/241_v0.4.md +0 -547
- package/docs/plans//347/237/245/350/257/206/345/233/276/350/260/261/351/241/271/347/233/256_/344/272/244/346/216/245/346/226/207/346/241/243_20260914.md +0 -308
- package/docs/plans//350/234/202/345/267/242/346/250/241/345/236/213/345/257/206/351/222/245/351/205/215/347/275/256/351/235/242_v1.0.md +0 -385
- package/docs/plans//350/256/260/345/277/206/350/207/252/345/244/204/347/220/206/344/270/211/346/241/243/350/207/252/346/262/273_/350/256/276/350/256/241_v0.2.md +0 -207
- package/docs/plans//350/257/255/344/271/211/346/227/266/347/251/272/345/233/276/350/241/245/345/205/250_/344/270/226/347/225/214/346/250/241/345/236/213/345/212/237/350/203/275/347/253/257_/350/256/276/350/256/241_v0.1.md +0 -107
- package/docs/promo/bilibili-cover.jpg +0 -0
- package/docs/promo/promo-visual-1.png +0 -0
- package/docs/promo/promo-visual-2.png +0 -0
- package/docs/promo/promo-visual-3.png +0 -0
- package/docs/promo/promo-visual-4.png +0 -0
- package/docs/promo/promo-visual-5.png +0 -0
- package/docs/swarm//350/234/202/347/276/244/344/272/222/350/201/224_v0.1.md +0 -704
- package/docs/swarm//350/234/202/347/276/244/345/220/214/351/224/231/346/243/200/346/265/213/345/256/236/351/252/214/345/215/217/350/256/256_v0.1.md +0 -242
- package/docs/swarm//350/234/202/347/276/244/345/244/232/346/231/272/350/203/275/344/275/223_/344/275/277/347/224/250/346/212/245/345/221/212_20260913.md +0 -68
- package/docs/swarm//350/234/202/347/276/244/345/244/232/346/231/272/350/203/275/344/275/223_/345/212/237/350/203/275/350/257/264/346/230/216_v0.6.md +0 -130
- package/docs/swarm//350/234/202/347/276/244/347/233/262/345/214/272/346/240/207/350/256/260_v1.0.md +0 -89
- package/docs/theory//344/270/215/345/217/257/351/235/240/345/256/232/347/220/206/344/270/216/345/244/261/346/225/210/344/274/230/345/205/210/346/241/206/346/236/266_v0.3.md +0 -365
- package/docs/theory//344/277/241/346/201/257/345/267/256/344/270/272/344/273/200/344/271/210/345/277/205/347/204/266/345/255/230/345/234/250/344/270/224/350/207/252/347/204/266/346/211/251/345/244/247.md +0 -239
- package/docs/theory//345/215/225/347/272/277/347/250/213/344/270/216/346/263/250/346/204/217/345/212/233/351/233/206/344/270/255_/346/227/240/344/272/211/350/256/256/347/220/206/350/256/272/346/226/207/346/241/243_v1.0.md +0 -129
- package/docs/theory//345/271/266/345/217/221/345/277/205/347/204/266/346/200/247/347/220/206/350/256/272_v0.2.md +0 -134
- package/docs/theory//346/231/272/350/203/275/347/232/204/345/205/254/347/220/206/345/214/226/345/237/272/347/237/263.md +0 -789
- package/docs/theory//346/231/272/350/203/275/347/232/204/350/256/244/347/237/245/350/277/207/347/250/213.md +0 -379
- package/docs/theory//346/231/272/350/203/275/350/256/2723.4.md +0 -5260
- package/docs/theory//346/246/202/345/277/265/345/210/206/345/261/202/345/257/271/351/275/220/350/241/250_v0.1.md +0 -145
- package/docs/theory//347/220/206/350/256/272_/346/234/272/345/210/266_/344/273/243/347/240/201_/345/256/236/351/252/214_/347/274/272/345/217/243/347/237/251/351/230/265_v0.1.md +0 -144
- package/docs/theory//347/220/206/350/256/272/344/273/223/346/213/206/345/210/206/344/270/216/347/231/275/347/256/261/347/237/245/350/257/206/345/272/223/345/206/205/350/277/201_v0.1.md +0 -198
- package/docs/theory//347/231/275/347/256/261/346/231/272/350/203/275/346/230/257/344/273/200/344/271/210/357/274/237.md +0 -812
- package/docs/theory//347/231/275/347/256/261/346/231/272/350/203/275/347/263/273/345/210/227/302/267/347/254/254/344/272/224/347/257/207/357/274/232/350/256/251AI/347/234/237/346/255/243/350/243/205/344/270/212/350/256/260/345/277/206.md +0 -421
- package/docs/theory//350/256/244/347/237/245/344/273/243/347/220/206/344/270/216/346/224/266/346/225/233/347/273/223/346/236/204_/346/235/241/344/273/266/350/256/272/351/207/215/346/236/204_v0.1.md +0 -221
- package/docs/world_model//344/270/226/347/225/214/346/250/241/345/236/213_/347/245/236/347/273/217/347/275/221/347/273/234/345/272/225/345/261/202/346/236/266/346/236/204_v1.0.md +0 -237
- package/docs//344/270/215/345/217/257/351/235/240/346/200/247/347/220/206/350/256/272_v0.1.md +0 -343
- package/docs//345/217/221/345/270/203/344/273/266/345/233/236/346/272/257/350/257/264/346/230/216_v0.1.md +0 -82
- package/docs//345/255/230/347/256/227/344/270/200/344/275/223/346/236/266/346/236/204/345/255/246/344/271/240_/345/244/226/351/203/250/347/220/206/350/256/272/345/257/271/347/205/247/344/270/216/350/267/257/347/272/277/344/272/244/346/216/245_20260916.md +0 -98
- package/docs//345/255/230/347/256/227/344/270/200/344/275/223/347/245/236/347/273/217/347/275/221/347/273/234/346/236/266/346/236/204_/346/200/273/347/272/262/344/270/216/344/272/244/346/216/245_20260916.md +0 -109
- package/docs//347/201/265/346/236/242/350/207/252/346/210/221/346/224/271/350/277/233/345/267/245/344/275/234/350/256/241/345/210/222_/345/244/226/351/203/250/347/240/224/347/251/266/347/263/273/345/210/227/345/220/270/346/224/266_v1_20260919.md +0 -286
- package/docs//347/231/275/347/256/261/345/214/226/347/272/262/351/242/206_/344/270/211/351/241/271/347/233/256/345/255/246/344/271/240/346/200/273/346/236/266/346/236/204/344/270/216Ghidra/344/272/244/346/216/245_20260916.md +0 -104
- package/docs//347/247/244_/350/256/260/345/277/206/347/263/273/347/273/237/350/257/204/346/265/213/350/247/204/350/214/203_v2.1.md +0 -225
- package/docs//350/256/260/345/277/206/347/263/273/347/273/237/345/215/263/347/231/275/347/256/261/345/214/226/346/231/272/350/203/275_/346/236/266/346/236/204/345/257/271/347/205/247_v0.2.md +0 -121
- package/docs//350/256/260/345/277/206/347/263/273/347/273/237/345/215/263/347/231/275/347/256/261/345/214/226/346/231/272/350/203/275_/346/236/266/346/236/204/345/257/271/347/205/247_v0.3.md +0 -117
- package/docs//350/256/260/345/277/206/350/257/204/345/256/241/347/263/273/347/273/237_/347/253/213/351/241/271/350/256/276/350/256/241/344/270/216/346/226/275/345/267/245/344/272/244/346/216/245_20260915.md +0 -183
package/md_cg/fsutil.py
CHANGED
|
@@ -1,824 +1,1120 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
|
-
"""md 认知图 · 文件系统原语(原子写 / 跨进程锁 / append-only 日志)
|
|
3
|
-
|
|
4
|
-
纯标准库(D-005)。三个原语都直接对应竞品踩过的坑:
|
|
5
|
-
|
|
6
|
-
1. 原子写:临时名必须唯一。deja-vu 的 atomicfile 记录了固定临时名的后果——
|
|
7
|
-
两个写者共享同一临时文件,第二个截断第一个还在写的内容,第一个把半截文件
|
|
8
|
-
rename 到位(18049 次读里 180 次读到不可解析的记录)。
|
|
9
|
-
2. Windows rename:另一个进程持有打开句柄时 os.replace 会被拒绝。deja-vu 在
|
|
10
|
-
windows CI 上实测「四个并发写者有三个被拒」,解法是短重试。
|
|
11
|
-
3. 跨进程锁:threading.Lock 只管本进程;灵枢是多进程共享库,必须用 OS 级锁。
|
|
12
|
-
"""
|
|
13
|
-
import os
|
|
14
|
-
import sys
|
|
15
|
-
import time
|
|
16
|
-
import uuid
|
|
17
|
-
import errno
|
|
18
|
-
import json
|
|
19
|
-
import tempfile
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
#
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
os.
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
tmp
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
self.
|
|
126
|
-
self.
|
|
127
|
-
self.
|
|
128
|
-
self.
|
|
129
|
-
self.
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
self.
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
os.
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
def
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
return
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
#
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
#
|
|
317
|
-
def
|
|
318
|
-
"""
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
"""
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
#
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
#
|
|
422
|
-
#
|
|
423
|
-
#
|
|
424
|
-
#
|
|
425
|
-
#
|
|
426
|
-
#
|
|
427
|
-
#
|
|
428
|
-
#
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
#
|
|
438
|
-
#
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
#
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
#
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
"""
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
#
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
#
|
|
546
|
-
#
|
|
547
|
-
#
|
|
548
|
-
#
|
|
549
|
-
#
|
|
550
|
-
#
|
|
551
|
-
#
|
|
552
|
-
#
|
|
553
|
-
#
|
|
554
|
-
#
|
|
555
|
-
#
|
|
556
|
-
#
|
|
557
|
-
#
|
|
558
|
-
#
|
|
559
|
-
#
|
|
560
|
-
#
|
|
561
|
-
#
|
|
562
|
-
#
|
|
563
|
-
#
|
|
564
|
-
#
|
|
565
|
-
|
|
566
|
-
#
|
|
567
|
-
|
|
568
|
-
#
|
|
569
|
-
|
|
570
|
-
#
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
"""
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
#
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
#
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""md 认知图 · 文件系统原语(原子写 / 跨进程锁 / append-only 日志)
|
|
3
|
+
|
|
4
|
+
纯标准库(D-005)。三个原语都直接对应竞品踩过的坑:
|
|
5
|
+
|
|
6
|
+
1. 原子写:临时名必须唯一。deja-vu 的 atomicfile 记录了固定临时名的后果——
|
|
7
|
+
两个写者共享同一临时文件,第二个截断第一个还在写的内容,第一个把半截文件
|
|
8
|
+
rename 到位(18049 次读里 180 次读到不可解析的记录)。
|
|
9
|
+
2. Windows rename:另一个进程持有打开句柄时 os.replace 会被拒绝。deja-vu 在
|
|
10
|
+
windows CI 上实测「四个并发写者有三个被拒」,解法是短重试。
|
|
11
|
+
3. 跨进程锁:threading.Lock 只管本进程;灵枢是多进程共享库,必须用 OS 级锁。
|
|
12
|
+
"""
|
|
13
|
+
import os
|
|
14
|
+
import sys
|
|
15
|
+
import time
|
|
16
|
+
import uuid
|
|
17
|
+
import errno
|
|
18
|
+
import json
|
|
19
|
+
import tempfile
|
|
20
|
+
import threading
|
|
21
|
+
|
|
22
|
+
IS_WIN = sys.platform == "win32"
|
|
23
|
+
if IS_WIN:
|
|
24
|
+
import msvcrt
|
|
25
|
+
else:
|
|
26
|
+
import fcntl
|
|
27
|
+
|
|
28
|
+
_RENAME_TRIES = 20
|
|
29
|
+
_RENAME_WAIT = 0.005
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
# 生效条件:tmp 与 path 给定后循环至多 _RENAME_TRIES 次调用 os.replace(tmp, path),成功即返回;仅捕获 PermissionError,非最后一次则 time.sleep(_RENAME_WAIT) 重试,最后一次仍 PermissionError 则抛出。
|
|
33
|
+
def _publish(tmp: str, path: str):
|
|
34
|
+
"""把临时文件 rename 到位,Windows 上短重试。"""
|
|
35
|
+
for i in range(_RENAME_TRIES):
|
|
36
|
+
try:
|
|
37
|
+
os.replace(tmp, path)
|
|
38
|
+
return
|
|
39
|
+
except PermissionError:
|
|
40
|
+
if i == _RENAME_TRIES - 1:
|
|
41
|
+
raise
|
|
42
|
+
time.sleep(_RENAME_WAIT)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
# 生效条件:行为与 _publish 完全一致(直接委派并返回其结果)——公开名,供包内
|
|
46
|
+
# 各「tmp + os.replace」原子写点位统一改走带 Windows 短重试的实现。
|
|
47
|
+
def publish(tmp: str, path: str):
|
|
48
|
+
"""`os.replace` 的公开安全版:目标被 Defender/索引器短暂持锁时短重试。
|
|
49
|
+
|
|
50
|
+
2026-09-25 全量回归实测:裸 `os.replace` 在 Windows 上随机抛
|
|
51
|
+
`PermissionError: [WinError 5]`(retr_s7 的 _postings_meta.json 改名中招,
|
|
52
|
+
失败者随机分布)——md_cg 内原子写一律经本函数,不再各写各的裸 replace。
|
|
53
|
+
"""
|
|
54
|
+
return _publish(tmp, path)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
# 生效条件:path 与 data 给定时取 path 所在目录 d 建目录,用 tempfile.mkstemp 在 d 内建临时文件按 encoding 写入 data,durable 为真才 flush+os.fsync(假值不 fsync),再经 _publish(tmp, path) 替换;任一步失败时 finally 里若 tmp 仍非 None 且 os.path.exists(tmp) 为真则 os.remove(OSError 忽略)。
|
|
58
|
+
def atomic_write(path: str, data: str, encoding: str = "utf-8", durable: bool = False):
|
|
59
|
+
"""整文件替换。临时文件与目标同目录(保证同一文件系统,rename 才原子),
|
|
60
|
+
临时名唯一(并发写者不共享),失败即清理而不是留在可能刚写满的磁盘上。
|
|
61
|
+
|
|
62
|
+
durable=False(默认):不做 fsync。
|
|
63
|
+
崩溃一致性由「临时文件 + rename」保证——读者要么看到旧内容、要么看到
|
|
64
|
+
新内容,永远看不到半截文件;fsync 多保证的只是"断电后新内容不丢"。
|
|
65
|
+
实测每次 fsync 让写入从 ~2000 节点/秒掉到 17 节点/秒(3048 节点迁移
|
|
66
|
+
要 3 分钟),而节点 .md 丢失的代价只是丢那一个节点,且索引可重建。
|
|
67
|
+
deja-vu 的 atomicfile 同样只做 Close + Rename,不 fsync。
|
|
68
|
+
durable=True:留给确实需要断电存活的调用方。
|
|
69
|
+
"""
|
|
70
|
+
d = os.path.dirname(os.path.abspath(path)) or "."
|
|
71
|
+
os.makedirs(d, exist_ok=True)
|
|
72
|
+
fd, tmp = tempfile.mkstemp(prefix="." + os.path.basename(path) + ".tmp-", dir=d)
|
|
73
|
+
try:
|
|
74
|
+
with os.fdopen(fd, "w", encoding=encoding, newline="\n") as f:
|
|
75
|
+
f.write(data)
|
|
76
|
+
if durable:
|
|
77
|
+
f.flush()
|
|
78
|
+
os.fsync(f.fileno())
|
|
79
|
+
_publish(tmp, path)
|
|
80
|
+
tmp = None
|
|
81
|
+
finally:
|
|
82
|
+
if tmp and os.path.exists(tmp):
|
|
83
|
+
try:
|
|
84
|
+
os.remove(tmp)
|
|
85
|
+
except OSError:
|
|
86
|
+
pass
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
# 生效条件:os.path.isdir(d) 为假时直接返回;否则对 d 下名字以 "." 开头且含 ".tmp-" 的条目,当 now - os.path.getmtime(p) > older_than 时 os.remove(p)(OSError 忽略),其余条目不动。
|
|
90
|
+
def sweep_stale_temps(d: str, older_than: float = 3600):
|
|
91
|
+
"""清理被杀死的进程留下的唯一命名临时文件(它们不会被下一个写者复用清掉)。"""
|
|
92
|
+
if not os.path.isdir(d):
|
|
93
|
+
return
|
|
94
|
+
now = time.time()
|
|
95
|
+
for name in os.listdir(d):
|
|
96
|
+
if ".tmp-" not in name or not name.startswith("."):
|
|
97
|
+
continue
|
|
98
|
+
p = os.path.join(d, name)
|
|
99
|
+
try:
|
|
100
|
+
if now - os.path.getmtime(p) > older_than:
|
|
101
|
+
os.remove(p)
|
|
102
|
+
except OSError:
|
|
103
|
+
pass
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
# 生效条件:path 加 ".lock" 后缀作为锁文件,进入时按 timeout 秒内以 poll 间隔轮询获取 OS 级排它锁(IS_WIN 用 msvcrt.locking 锁首字节,否则 fcntl.flock),超时仍未获锁时 strict 为真抛 TimeoutError、否则返回自身放行。
|
|
107
|
+
class FileLock:
|
|
108
|
+
"""跨进程排它锁(OS 级)。
|
|
109
|
+
|
|
110
|
+
Windows 用 msvcrt.locking 锁首字节,Unix 用 fcntl.flock。两者语义不同
|
|
111
|
+
(前者是强制字节范围锁、后者是建议性文件锁),但对「同一把锁文件、所有
|
|
112
|
+
写者都主动获取」这个用法是等价的。
|
|
113
|
+
|
|
114
|
+
超时后放弃并放行(best-effort):写被拒绝的代价大于一次竞态——这与
|
|
115
|
+
deja-vu 对 usage 日志锁的取舍一致("a racing write beats a lost injection")。
|
|
116
|
+
|
|
117
|
+
strict=True 反转该取舍:超时抛 TimeoutError 而非放行。适用于**不能丢的写**
|
|
118
|
+
(审核队列 inbox/decisions、裁决记录)——锁竞争失败时显式报错让调用方重试,
|
|
119
|
+
好过静默放行后退化为无锁并发(丢一条提案/裁决比让写入者等一下代价大)。
|
|
120
|
+
"""
|
|
121
|
+
|
|
122
|
+
# 生效条件:path 加 ".lock" 后缀存入 self.path,timeout、poll、strict 原样保存,并置 self._f = None、self.acquired = False。
|
|
123
|
+
def __init__(self, path: str, timeout: float = 10.0, poll: float = 0.01,
|
|
124
|
+
strict: bool = False):
|
|
125
|
+
self.path = path + ".lock"
|
|
126
|
+
self.timeout = timeout
|
|
127
|
+
self.poll = poll
|
|
128
|
+
self.strict = strict
|
|
129
|
+
self._f = None
|
|
130
|
+
self.acquired = False
|
|
131
|
+
|
|
132
|
+
# 生效条件:先按 self.path 建父目录并 open(self.path, "a+b"),再在 self.timeout 到期前每 self.poll 秒尝试加锁(IS_WIN 用 msvcrt.locking(LK_NBLCK),否则 fcntl.flock(LOCK_EX|LOCK_NB));成功即置 self.acquired=True 并返回 self;OSError 的 errno 不在 (EACCES, EAGAIN, EDEADLK) 时直接 raise,超时后 self.strict 为真抛 TimeoutError、否则返回 self 放行。
|
|
133
|
+
def __enter__(self):
|
|
134
|
+
os.makedirs(os.path.dirname(os.path.abspath(self.path)) or ".", exist_ok=True)
|
|
135
|
+
self._f = open(self.path, "a+b")
|
|
136
|
+
deadline = time.time() + self.timeout
|
|
137
|
+
while True:
|
|
138
|
+
try:
|
|
139
|
+
if IS_WIN:
|
|
140
|
+
self._f.seek(0)
|
|
141
|
+
msvcrt.locking(self._f.fileno(), msvcrt.LK_NBLCK, 1)
|
|
142
|
+
else:
|
|
143
|
+
fcntl.flock(self._f.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
144
|
+
self.acquired = True
|
|
145
|
+
return self
|
|
146
|
+
except OSError as e:
|
|
147
|
+
if e.errno not in (errno.EACCES, errno.EAGAIN, errno.EDEADLK):
|
|
148
|
+
raise
|
|
149
|
+
if time.time() > deadline:
|
|
150
|
+
if self.strict:
|
|
151
|
+
raise TimeoutError(
|
|
152
|
+
f"FileLock 超时未获锁(strict):{self.path}")
|
|
153
|
+
return self # 放行,不阻断写路径
|
|
154
|
+
time.sleep(self.poll)
|
|
155
|
+
|
|
156
|
+
# 生效条件:self.acquired 为真时按 IS_WIN 用 msvcrt.locking(LK_UNLCK) 或 fcntl.flock(LOCK_UN) 解锁(OSError 被吞掉);finally 中只要 self._f 为真就 close,随后 self._f=None、self.acquired=False。
|
|
157
|
+
def __exit__(self, *exc):
|
|
158
|
+
try:
|
|
159
|
+
if self.acquired:
|
|
160
|
+
if IS_WIN:
|
|
161
|
+
self._f.seek(0)
|
|
162
|
+
msvcrt.locking(self._f.fileno(), msvcrt.LK_UNLCK, 1)
|
|
163
|
+
else:
|
|
164
|
+
fcntl.flock(self._f.fileno(), fcntl.LOCK_UN)
|
|
165
|
+
except OSError:
|
|
166
|
+
pass
|
|
167
|
+
finally:
|
|
168
|
+
if self._f:
|
|
169
|
+
self._f.close()
|
|
170
|
+
self._f = None
|
|
171
|
+
self.acquired = False
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
# 生效条件:作为 `with` 的空上下文——进入返回自身、退出返回 False(不吞异常)。
|
|
175
|
+
class _NullContext:
|
|
176
|
+
"""空上下文(无 root 的纯内存实例:无需跨进程互斥)。"""
|
|
177
|
+
|
|
178
|
+
def __enter__(self):
|
|
179
|
+
return self
|
|
180
|
+
|
|
181
|
+
def __exit__(self, *exc):
|
|
182
|
+
return False
|
|
183
|
+
|
|
184
|
+
#: 本进程内**已持有**的跨进程锁(key -> 重入深度)。为什么需要:写面有两个天然的
|
|
185
|
+
#: 嵌套关系——`writepipe.execute`(整条链:判存在/保护闸/落盘/可见性)与它内部的
|
|
186
|
+
#: `MdCGOS.add`(同一 node_id)、以及 `remember_gated` 与它内部的 `add`。OS 级文件锁
|
|
187
|
+
#: 在同一进程内对同一文件**第二次**获取会自我阻塞(Windows msvcrt 锁同一字节范围、
|
|
188
|
+
#: POSIX flock 换 fd 亦互斥)⇒ 必须按「同线程重入」记账,否则修复本身变成死锁。
|
|
189
|
+
#: 只按线程记账(threading.local):跨线程仍各自排队,跨进程永远走 OS 锁。
|
|
190
|
+
_LOCK_TLS = threading.local()
|
|
191
|
+
|
|
192
|
+
# 生效条件:返回本线程的 key->深度字典(首次访问时创建),不触盘、不加锁。
|
|
193
|
+
def _held_locks():
|
|
194
|
+
d = getattr(_LOCK_TLS, "held", None)
|
|
195
|
+
if d is None:
|
|
196
|
+
d = {}
|
|
197
|
+
_LOCK_TLS.held = d
|
|
198
|
+
return d
|
|
199
|
+
|
|
200
|
+
# 生效条件:作为 `with` 使用——同线程已持有同 key 时只加深度并返回自身(不重复获取 OS 锁);
|
|
201
|
+
# 否则按 path 建 FileLock(timeout, strict=True) 并进入,成功后记深度 1;FileLock 抛错时
|
|
202
|
+
# 不记账(失败不留下「已持有」的假象),异常原样上抛。退出时深度递减,归零才真正释放。
|
|
203
|
+
class _ReentrantFileLock:
|
|
204
|
+
"""`FileLock` 的同线程重入包装(跨进程互斥语义完全由内层 FileLock 提供)。"""
|
|
205
|
+
|
|
206
|
+
def __init__(self, path: str, timeout: float):
|
|
207
|
+
self.path = path
|
|
208
|
+
self.key = os.path.normcase(os.path.abspath(path))
|
|
209
|
+
self.timeout = timeout
|
|
210
|
+
self._inner = None
|
|
211
|
+
|
|
212
|
+
def __enter__(self):
|
|
213
|
+
held = _held_locks()
|
|
214
|
+
if held.get(self.key):
|
|
215
|
+
held[self.key] += 1
|
|
216
|
+
return self
|
|
217
|
+
inner = FileLock(self.path, timeout=self.timeout, strict=True)
|
|
218
|
+
inner.__enter__()
|
|
219
|
+
self._inner = inner
|
|
220
|
+
held[self.key] = 1
|
|
221
|
+
return self
|
|
222
|
+
|
|
223
|
+
def __exit__(self, *exc):
|
|
224
|
+
held = _held_locks()
|
|
225
|
+
depth = held.get(self.key, 0)
|
|
226
|
+
if depth > 1:
|
|
227
|
+
held[self.key] = depth - 1
|
|
228
|
+
return False
|
|
229
|
+
held.pop(self.key, None)
|
|
230
|
+
if self._inner is not None:
|
|
231
|
+
inner, self._inner = self._inner, None
|
|
232
|
+
return inner.__exit__(*exc)
|
|
233
|
+
return False
|
|
234
|
+
|
|
235
|
+
# 生效条件:cg 具备非空 root 时返回以 `<root>/_node_locks/<node_id>` 为锁文件(FileLock 自动补 `.lock`)的跨进程排它锁(timeout 秒内未获锁抛 TimeoutError,strict 语义:「不能丢的写」宁可显式失败也不无锁放行;同线程重入按深度计数);root 为假值返回不做任何事的空上下文;node_id 含路径分隔符/`..`/`:` 等不安全字符时改用其 sha256 前 32 位作锁名——同一 id 在任何进程里都映射到同一把锁,但绝不把调用方可控串拼进路径。
|
|
236
|
+
def node_write_lock(cg, node_id, timeout: float = 30.0):
|
|
237
|
+
"""同一 `node_id` 的写入跨进程串行化(issue #322)。
|
|
238
|
+
|
|
239
|
+
**不得用进程内锁冒充**:多智能体共享同一 root 时,写者可能是不同进程;
|
|
240
|
+
threading.Lock 对它们零作用。本函数一律走 `FileLock`(OS 级字节范围锁)。
|
|
241
|
+
|
|
242
|
+
锁粒度按 **node_id**:不同 id 的写入互不阻塞(写面瓶颈 S2/S4),而同一 id
|
|
243
|
+
的「判存在 → 保护闸 → 落盘 → 索引可见(flush)」必须在同一临界区内完成——
|
|
244
|
+
否则两个写者都判「不存在」,后写整份覆盖前写,两边都回执 committed:true
|
|
245
|
+
(#322 的静默丢正文);anchor/knowledge 同 id 并发时两份文件并存,保护闸
|
|
246
|
+
在两侧都判不到对方(#322 的绕过 anchor)。
|
|
247
|
+
"""
|
|
248
|
+
root = getattr(cg, "root", None)
|
|
249
|
+
if not root:
|
|
250
|
+
return _NullContext()
|
|
251
|
+
# 非字符串/空 id 一律由 add 的既有白名单 fail-closed 拒绝(它们根本走不到
|
|
252
|
+
# 落盘),此处不为它们造锁文件——空锁名还会把目录本身当文件打开。
|
|
253
|
+
if not isinstance(node_id, str) or not node_id:
|
|
254
|
+
return _NullContext()
|
|
255
|
+
name = node_id
|
|
256
|
+
if ("/" in name or "\\" in name or ".." in name or ":" in name):
|
|
257
|
+
import hashlib
|
|
258
|
+
name = "sha256_" + hashlib.sha256(
|
|
259
|
+
name.encode("utf-8", "replace")).hexdigest()[:32]
|
|
260
|
+
return _ReentrantFileLock(os.path.join(root, "_node_locks", name),
|
|
261
|
+
timeout)
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def merge_rmw_lock(cg, timeout: float = 30.0):
|
|
265
|
+
"""MERGE/CONVERGE 目标「读 → 聚合 → 整份覆写」的跨进程锁(issue #348-B)。
|
|
266
|
+
|
|
267
|
+
按**根**一把:`forgetting.reinforce` 与 `writelimit.converge_into` 的目标
|
|
268
|
+
由近重复判定选出、可能不同,但读改写窗口只有一处语义(目标正文 + merge_count),
|
|
269
|
+
根级锁让所有聚合落点共用同一临界区,避免「按目标加锁」在选目标阶段就漏掉
|
|
270
|
+
竞态。锁文件 `_merge_rmw.lock` 在 `sleep.FORBIDDEN_SUFFIXES` 内,不进版本库。
|
|
271
|
+
|
|
272
|
+
`strict=True`:锁竞争超时显式失败(调用方/重试层可见),绝不退化为无锁并发
|
|
273
|
+
——后者正是本缺陷的形态(后写者用旧正文整份覆盖前者刚追加的聚合行)。
|
|
274
|
+
"""
|
|
275
|
+
root = getattr(cg, "root", None)
|
|
276
|
+
if not root:
|
|
277
|
+
return _NullContext()
|
|
278
|
+
return _ReentrantFileLock(os.path.join(root, "_merge_rmw"), timeout)
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
# 生效条件:os.path.getsize(path) 为 0 时返回 False;否则二进制打开 path 并从 size-1 处读 1 字节,返回 f.read(1) != b"\n";getsize/open/seek/read 抛 OSError 时返回 False。
|
|
282
|
+
def ends_mid_line(path: str) -> bool:
|
|
283
|
+
"""行式日志的最后一字节是否不是换行——即上一个写者被杀死留下的半截记录。
|
|
284
|
+
追加者若不先补一个换行,新记录会粘在这行上,两条都解析不出来。"""
|
|
285
|
+
try:
|
|
286
|
+
size = os.path.getsize(path)
|
|
287
|
+
if size == 0:
|
|
288
|
+
return False
|
|
289
|
+
with open(path, "rb") as f:
|
|
290
|
+
f.seek(size - 1)
|
|
291
|
+
return f.read(1) != b"\n"
|
|
292
|
+
except OSError:
|
|
293
|
+
return False
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
# 生效条件:record 序列化为 json.dumps(record, ensure_ascii=False, separators=(",", ":")) + "\n";若 ends_mid_line(path) 为真则在行首再补一个 "\n";
|
|
297
|
+
# 随后建父目录并以 O_CREAT|O_WRONLY|O_APPEND、权限 0o600 打开 path 写入该行 UTF-8 字节后关闭;成功返回 True;
|
|
298
|
+
# **durable=True 时整段「追加」收进 FileLock(path, strict=True) 临界区**(issue #549:审计账本等不可丢的写面);
|
|
299
|
+
# os.makedirs/os.open/os.write 抛 OSError(磁盘满/只读介质/权限拒绝/路径过长)时**兜住并返回 False**
|
|
300
|
+
# (best-effort 契约:簿记写入失败不得升级为业务路径失败,issue #326-④);KeyboardInterrupt/SystemExit 是
|
|
301
|
+
# BaseException、不在兜住面内,照常上抛。
|
|
302
|
+
# 参数关系(两批改动正交,合并保留):strict=「失败是否上抛」(#326-④) · durable=「是否进临界区」(#549)。
|
|
303
|
+
#: 写失败累计计数 + **有界**样本(issue #326-④ B 面)。
|
|
304
|
+
#: 为什么在源头记账:`append_jsonl` 的失败契约是**返回值**(不上抛)——若无本地捕获的
|
|
305
|
+
#: 调用点只是忽略返回值,失败就**静默**了(本仓 121 个调用点属此类)。把失败信号收成
|
|
306
|
+
#: 本模块的**单点可观测项**后,「静默」在结构上不再可能(运维/守卫读 APPEND_FAILURES
|
|
307
|
+
#: 与 _APPEND_FAIL_SAMPLES 即可;口径同 crypto._note_audit_write_failure 的 N126)。
|
|
308
|
+
APPEND_FAILURES = 0 # 累计**写失败次数**(非按路径去重)
|
|
309
|
+
_APPEND_FAIL_SAMPLES = [] # 有界样本:(path, 异常类型名, 截断文本)
|
|
310
|
+
_APPEND_FAIL_SAMPLE_CAP = 8
|
|
311
|
+
_APPEND_FAIL_WARN_CAP = 8 # stderr 告警同样有界(防刷屏)
|
|
312
|
+
|
|
313
|
+
|
|
314
|
+
# 生效条件:path/exc 给定时把 APPEND_FAILURES 累加 1、在样本未满 _APPEND_FAIL_SAMPLE_CAP 时
|
|
315
|
+
# 追加一条 (path, 类型名, 文本[:120])、并在累计不超 _APPEND_FAIL_WARN_CAP 时向 stderr 写一行
|
|
316
|
+
# 告警;**无条件返回 False**(供 append_jsonl 直接 `return`)。不抛任何异常。
|
|
317
|
+
def _note_append_failure(path, exc):
|
|
318
|
+
"""登记一次 append_jsonl 写失败 + 有界告警(容忍 ≠ 静默)。"""
|
|
319
|
+
global APPEND_FAILURES
|
|
320
|
+
APPEND_FAILURES += 1
|
|
321
|
+
if len(_APPEND_FAIL_SAMPLES) < _APPEND_FAIL_SAMPLE_CAP:
|
|
322
|
+
_APPEND_FAIL_SAMPLES.append((str(path), type(exc).__name__, str(exc)[:120]))
|
|
323
|
+
if APPEND_FAILURES <= _APPEND_FAIL_WARN_CAP:
|
|
324
|
+
sys.stderr.write(
|
|
325
|
+
"[fsutil] append_jsonl 写失败(best-effort 已容忍,返回 False)%s:%s: %s"
|
|
326
|
+
"——进程内累计 %d 次;已登记 fsutil.APPEND_FAILURES / "
|
|
327
|
+
"_APPEND_FAIL_SAMPLES 供审计读取。排查方向:磁盘满 / 只读介质 / "
|
|
328
|
+
"权限拒绝 / 路径过长。\n"
|
|
329
|
+
% (path, type(exc).__name__, str(exc)[:120], APPEND_FAILURES))
|
|
330
|
+
return False
|
|
331
|
+
|
|
332
|
+
|
|
333
|
+
def append_jsonl(path: str, record: dict, strict: bool = False,
|
|
334
|
+
durable: bool = False) -> bool:
|
|
335
|
+
"""向 append-only 日志追加一条记录(best-effort 语义):成功 True、写失败 False。
|
|
336
|
+
|
|
337
|
+
注意 O_APPEND 的原子性是**平台相关**的:POSIX 保证「定位到末尾 + 写入」是
|
|
338
|
+
一个原子操作,Windows CRT 的 _O_APPEND 则是 lseek(END) + write 两步,并发下
|
|
339
|
+
会偶发交错丢记录(实测 6 进程 × 40 条,每轮丢 ~1 条)。
|
|
340
|
+
|
|
341
|
+
因此本函数只用于**丢一条无所谓**的簿记(访问计数)。任何不能丢的东西
|
|
342
|
+
(比如索引增量)必须用 ShardedLog——每个写者独占一个分片,不共享写入点。
|
|
343
|
+
|
|
344
|
+
失败契约(issue #326-④):本文档自陈「丢一条无所谓」,原实现却只有
|
|
345
|
+
try/finally——os.makedirs/os.open/os.write 的 OSError(磁盘满、只读介质、
|
|
346
|
+
权限拒绝、路径过长)会**击穿**调用方,把**簿记写入失败升级为业务路径失败**。
|
|
347
|
+
现按本文档兜住 OSError 并返回 False;**只**兜 OSError——KeyboardInterrupt /
|
|
348
|
+
SystemExit 是 BaseException,Type/ValueError(如 record 不可 JSON 序列化)
|
|
349
|
+
是编程错误,一概照常上抛,不把「写失败」与「用错」混为一谈。
|
|
350
|
+
|
|
351
|
+
strict(issue #426-C,2026-10-17 U06 槽内回触 U05 面):默认 False ⇒ **既有 129 处
|
|
352
|
+
调用点行为逐位不变**(含不可写路径下「返回 False 且不抛」的旧语义)。
|
|
353
|
+
`strict=True` 供**不能丢**的队列写入(inbox / decisions)使用:失败时**先记账**
|
|
354
|
+
(_note_append_failure,保证「不静默」)**再上抛 OSError**,让调用方无法把
|
|
355
|
+
「没落盘」当成功回执发出去。**不新增第四种失败编码**:上抛的异常由工具面按仓内
|
|
356
|
+
既有三种编码(业务失败 isError=false+JSON / 异常 isError=true+JSON / read 缺失
|
|
357
|
+
null)中的「异常」形态呈现,不自创形态。
|
|
358
|
+
|
|
359
|
+
因此**缺省**(`durable=False`)仍只用于**丢一条无所谓**的簿记(访问计数);
|
|
360
|
+
任何不能丢的东西(索引增量)**首选** ShardedLog——每个写者独占一个分片,
|
|
361
|
+
不共享写入点,物理上无从冲突。
|
|
362
|
+
|
|
363
|
+
issue #549(2026-10-14):但**审计账本**(`MdCGOS._audit` 的 `_audit.jsonl`)
|
|
364
|
+
此前也走本函数缺省路径,实测 6 进程 × 20 条在多写者下静默丢
|
|
365
|
+
**29.2% / 36.7% / 39.2%**(三轮),且 `add` 仍返回成功、worker 零报错 ——
|
|
366
|
+
而该账本被 `admission` 等消费做状态推断,缺行使审计追溯出现
|
|
367
|
+
**无法察觉的空洞**(「发生过」与「没发生过」不可区分)。
|
|
368
|
+
审计账本的写点分散且要参与轮转,改 ShardedLog 需动读面与轮转器;
|
|
369
|
+
故此处提供 **`durable=True`**:把「判半截行 + 追加」整段收进
|
|
370
|
+
`FileLock(strict=True)` 临界区(该锁的既有语义即「不能丢的写」),
|
|
371
|
+
以最小改动关掉静默丢行。
|
|
372
|
+
|
|
373
|
+
`durable=True` 的代价是**锁定串行化**(并发写审计会排队)。
|
|
374
|
+
按既有价值序(`FileLock` 注释:写被拒绝的代价大于一次竞态,而**不能丢的写**
|
|
375
|
+
宁可让写入者等一下)——审计账本属「不能丢」,故取 strict 语义:
|
|
376
|
+
锁竞争超时**显式抛错**给调用方,而不是静默退化为无锁并发再丢行。
|
|
377
|
+
"""
|
|
378
|
+
line = json.dumps(record, ensure_ascii=False, separators=(",", ":")) + "\n"
|
|
379
|
+
d = os.path.dirname(os.path.abspath(path)) or "."
|
|
380
|
+
try:
|
|
381
|
+
os.makedirs(d, exist_ok=True)
|
|
382
|
+
if durable:
|
|
383
|
+
# issue #549:整段临界区——判半截行与追加必须原子(否则两个写者可同时
|
|
384
|
+
# 判定「上一行完整」而各自直接追加,交错仍会丢)。
|
|
385
|
+
# 注:不加 ".lock" 后缀的第二把锁——FileLock 内部即以 path + ".lock" 命名,
|
|
386
|
+
# 与本文件其他写点(inbox/decisions)同构。
|
|
387
|
+
# 失败契约与下方共用:临界区内抛 OSError 同样落到 _note_append_failure;
|
|
388
|
+
# `strict=True` 时同一条腿再上抛(两参数正交,见上方契约注释)。
|
|
389
|
+
with FileLock(path, strict=True):
|
|
390
|
+
if ends_mid_line(path):
|
|
391
|
+
line = "\n" + line
|
|
392
|
+
fd = os.open(path, os.O_CREAT | os.O_WRONLY | os.O_APPEND, 0o600)
|
|
393
|
+
try:
|
|
394
|
+
os.write(fd, line.encode("utf-8"))
|
|
395
|
+
finally:
|
|
396
|
+
os.close(fd)
|
|
397
|
+
return True
|
|
398
|
+
if ends_mid_line(path):
|
|
399
|
+
line = "\n" + line
|
|
400
|
+
fd = os.open(path, os.O_CREAT | os.O_WRONLY | os.O_APPEND, 0o600)
|
|
401
|
+
except OSError as e:
|
|
402
|
+
_note = _note_append_failure(path, e)
|
|
403
|
+
if strict:
|
|
404
|
+
raise # strict:失败必须让调用方看见(先记账,再上抛)
|
|
405
|
+
return _note
|
|
406
|
+
try:
|
|
407
|
+
os.write(fd, line.encode("utf-8"))
|
|
408
|
+
return True
|
|
409
|
+
except OSError as e:
|
|
410
|
+
_note = _note_append_failure(path, e)
|
|
411
|
+
if strict:
|
|
412
|
+
raise # strict:同上(记账在 _note_append_failure 内完成)
|
|
413
|
+
return _note
|
|
414
|
+
finally:
|
|
415
|
+
try:
|
|
416
|
+
os.close(fd)
|
|
417
|
+
except OSError:
|
|
418
|
+
pass
|
|
419
|
+
|
|
420
|
+
|
|
421
|
+
# issue #546:审计/台账面的**独立写入点**(每进程一个分片)+ 读面合并。
|
|
422
|
+
#
|
|
423
|
+
# 为什么不是给 append_jsonl 加锁:它的 docstring 已规定「本函数只用于**丢一条
|
|
424
|
+
# 无所谓**的簿记;任何不能丢的东西必须用 ShardedLog——每个写者独占一个分片,
|
|
425
|
+
# **不共享写入点**」。所以病在「调用面选错了函数」,不在「这个函数缺锁」。
|
|
426
|
+
#
|
|
427
|
+
# 实测(6 写者 × 400 行):共享单文件丢 538 行(22.4%)且全部报成功;
|
|
428
|
+
# 每写者独立文件丢 0 行(0.0%)。唯一变量就是写入点是否共享。
|
|
429
|
+
def ledger_shard_path(base: str, pid=None) -> str:
|
|
430
|
+
"""单写者的分片路径:<base 去掉扩展名>.<pid><扩展名>。"""
|
|
431
|
+
root, ext = os.path.splitext(base)
|
|
432
|
+
if not ext:
|
|
433
|
+
ext = ".jsonl"
|
|
434
|
+
return "%s.%d%s" % (root, os.getpid() if pid is None else int(pid), ext)
|
|
435
|
+
|
|
436
|
+
|
|
437
|
+
# 生效条件:base 为台账基准路径;把记录追加到**本进程独占的分片**,复用 append_jsonl
|
|
438
|
+
# 的既有失败契约(best-effort 返回 bool;strict 由调用方决定)。与 append_jsonl 同族:
|
|
439
|
+
# **调用方必须消费返回值**(E11 钉子同时覆盖两者,issue #546)。
|
|
440
|
+
def append_jsonl_ledger(base: str, record: dict, strict: bool = False) -> bool:
|
|
441
|
+
"""追加到本进程独占分片(issue #546)。"""
|
|
442
|
+
return append_jsonl(ledger_shard_path(base), record, strict=strict)
|
|
443
|
+
|
|
444
|
+
|
|
445
|
+
# 生效条件:base 为台账基准路径;读面**同时**读旧单文件 base 与全部分片
|
|
446
|
+
# <stem>.<pid><ext>(旧单文件 + 新分片共存时历史不断代)。返回按**时间序**拼接的记录
|
|
447
|
+
# 列表(旧单文件在最前——它是旧版本写的、必早于任何分片;分片按 pid **数值**升序);
|
|
448
|
+
# 单文件/单分片的读失败不阻断其它(各自 try)。
|
|
449
|
+
#
|
|
450
|
+
# issue #556 修两处错(病象:旧库升级后 `self_state.refresh(strict=True)` 恒
|
|
451
|
+
# `chain_broken`;非 strict 时版本号倒退、两条 version=2):
|
|
452
|
+
# ① **旧单文件被读两次** —— 旧匹配条件 `<stem>.` 前缀 + `.jsonl` 后缀把 `<stem>.jsonl`
|
|
453
|
+
# 自己也算了进去(它同时满足两者)。改为「前缀之后必须是纯数字 pid」,base 自身
|
|
454
|
+
# 因而不命中。
|
|
455
|
+
# ② **分片按文件名字典序** —— `_self_state.10000.jsonl` 排在 `_self_state.9999.jsonl`
|
|
456
|
+
# 之前,于是「最新一条」取错。改为解析 pid 后**按数值**排序。
|
|
457
|
+
# 两处都只在「旧单文件与分片共存」或「pid 跨位数」时显形,全新单进程库看不出来。
|
|
458
|
+
def read_jsonl_ledger(base: str) -> list:
|
|
459
|
+
"""读旧单文件 + 全部分片(issue #546 的读面合并;#556 修重复读与排序)。"""
|
|
460
|
+
out = []
|
|
461
|
+
try:
|
|
462
|
+
out.extend(read_jsonl(base))
|
|
463
|
+
except OSError:
|
|
464
|
+
pass
|
|
465
|
+
root, ext = os.path.splitext(base)
|
|
466
|
+
if not ext:
|
|
467
|
+
ext = ".jsonl"
|
|
468
|
+
d = os.path.dirname(os.path.abspath(base)) or "."
|
|
469
|
+
try:
|
|
470
|
+
names = os.listdir(d)
|
|
471
|
+
except OSError:
|
|
472
|
+
return out
|
|
473
|
+
prefix = os.path.basename(root) + "."
|
|
474
|
+
shards = []
|
|
475
|
+
for name in names:
|
|
476
|
+
if not (name.startswith(prefix) and name.endswith(ext)):
|
|
477
|
+
continue
|
|
478
|
+
mid = name[len(prefix):len(name) - len(ext)]
|
|
479
|
+
if not mid.isdigit(): # #556 ①:排除旧单文件与任何非 <pid> 形态
|
|
480
|
+
continue
|
|
481
|
+
shards.append((int(mid), name))
|
|
482
|
+
shards.sort() # #556 ②:按 pid 数值升序,非字典序
|
|
483
|
+
for _pid, name in shards:
|
|
484
|
+
try:
|
|
485
|
+
out.extend(read_jsonl(os.path.join(d, name)))
|
|
486
|
+
except OSError:
|
|
487
|
+
continue
|
|
488
|
+
return out
|
|
489
|
+
|
|
490
|
+
|
|
491
|
+
# 生效条件:os.path.exists(path) 为假时生成器直接结束不产出;否则逐行 strip,空行跳过,json.loads 成功则 yield 该对象,抛 ValueError 的行跳过,其余异常不捕获。
|
|
492
|
+
def read_jsonl(path: str):
|
|
493
|
+
"""读 append-only 日志,跳过被截断/粘连的坏行(不因自身簿记而失败)。"""
|
|
494
|
+
if not os.path.exists(path):
|
|
495
|
+
return
|
|
496
|
+
with open(path, "r", encoding="utf-8", errors="replace") as f:
|
|
497
|
+
for line in f:
|
|
498
|
+
line = line.strip()
|
|
499
|
+
if not line:
|
|
500
|
+
continue
|
|
501
|
+
try:
|
|
502
|
+
yield json.loads(line)
|
|
503
|
+
except ValueError:
|
|
504
|
+
continue
|
|
505
|
+
|
|
506
|
+
|
|
507
|
+
# 生效条件:文件不存在返回 ([], None);从 offset 读到 EOF,但只解析最后一个换行之前的完整记录;完整坏行跳过,未完成尾行留待后续重读。返回记录与已消费的完整行字节水位,不以读前或读后的 stat size 代替。
|
|
508
|
+
def read_jsonl_tail(path: str, offset: int):
|
|
509
|
+
"""Incremental JSONL records and the last committed newline offset.
|
|
510
|
+
|
|
511
|
+
A writer may still be completing JSON/UTF-8 when this reader reaches EOF.
|
|
512
|
+
Advancing beyond that suffix permanently loses the eventual record in the
|
|
513
|
+
four inbox/decisions caches. Keep the cursor at its starting line boundary.
|
|
514
|
+
Complete corrupt lines still advance the cursor; append_jsonl's crash
|
|
515
|
+
resynchronization newline therefore cannot block later valid records.
|
|
516
|
+
|
|
517
|
+
A complete append that races with the initial stat is consumed exactly
|
|
518
|
+
once: the returned cursor is measured from bytes actually read. A JSON
|
|
519
|
+
object without its terminating newline is pending, even if parseable.
|
|
520
|
+
"""
|
|
521
|
+
if not os.path.exists(path):
|
|
522
|
+
return [], None
|
|
523
|
+
recs = []
|
|
524
|
+
with open(path, "rb") as f:
|
|
525
|
+
f.seek(offset)
|
|
526
|
+
data = f.read()
|
|
527
|
+
# Only complete lines are committed. A concurrent writer may still be in
|
|
528
|
+
# the middle of JSON or a UTF-8 code point: retain that suffix for the next
|
|
529
|
+
# read, rather than permanently advancing the consumer beyond it.
|
|
530
|
+
committed = data.rfind(b"\n") + 1
|
|
531
|
+
for b in data[:committed].split(b"\n"):
|
|
532
|
+
line = b.decode("utf-8", "replace").strip()
|
|
533
|
+
if not line:
|
|
534
|
+
continue
|
|
535
|
+
try:
|
|
536
|
+
recs.append(json.loads(line))
|
|
537
|
+
except ValueError:
|
|
538
|
+
continue
|
|
539
|
+
return recs, offset + committed
|
|
540
|
+
|
|
541
|
+
|
|
542
|
+
_COUNT_CACHE = {} # abspath -> (bytes_scanned, mtime_ns, lines)
|
|
543
|
+
|
|
544
|
+
|
|
545
|
+
# ---------- N225(2026-09-29):非对象记录行的可观测记账 ----------
|
|
546
|
+
#
|
|
547
|
+
# 分片日志的读面必须**容忍**非对象行(`null` / `[]` / `123` / `"abc"` 都是
|
|
548
|
+
# 合法 JSON,`read_jsonl` 按「JSON 合法性」收行 ⇒ 它们会进调用方的记录
|
|
549
|
+
# 列表),否则排序键 `r.get("_t", 0)` 抛 AttributeError,索引装载 / compact
|
|
550
|
+
# 整链断裂(实测:MdCG 构造失败、常驻服务起不来)。但**容忍 ≠ 静默**:
|
|
551
|
+
# 跳过必须留下机器可读的痕迹——否则「日志里混进了坏行」与「记录本就不
|
|
552
|
+
# 存在」不可区分,索引静默少几条,运维与守卫都看不见。故此处是跳过面的
|
|
553
|
+
# 唯一记账点:模块级累计计数 + 有界样本 + stderr 汇总告警(每条分片一行,
|
|
554
|
+
# 不逐行刷屏)。
|
|
555
|
+
#
|
|
556
|
+
# N225 补强(2026-09-29,深度 2 同族面):同一记账面再收两类**dict 行内**的坏型
|
|
557
|
+
# (N225 v1.0 报告 §0「已知缺口」点名的深度 2 复发面,此前不在任何断言面内):
|
|
558
|
+
# ① 载荷坏型:分片记录 `e` 非 dict 且非 None——判据在重放面(mdcg 的两条重放
|
|
559
|
+
# 路径同一实现),经 note_bad_payload_rows 记账;
|
|
560
|
+
# ② 排序键坏型:`_t` / `_s` 非数值(或 NaN)——read_all 的排序键健壮化与记账
|
|
561
|
+
# 必须同时发生,经 note_bad_sortkey_rows 记账。
|
|
562
|
+
# 三类**合计**进 NONOBJECT_ROW_SKIPS(这就是「同一记账面」的字面口径),各子面
|
|
563
|
+
# 另留独立计数与样本,运维据此区分「行根本不是对象」/「行是对象而载荷坏了」/
|
|
564
|
+
# 「行的排序键坏了」——三者排查方向不同。kind 名:非对象行 / 载荷坏型 / 排序键坏型。
|
|
565
|
+
NONOBJECT_ROW_SKIPS = 0 # 累计跳过的坏记录行数(三类合计,进程内)
|
|
566
|
+
NONOBJECT_ROW_SAMPLES = [] # 非对象行样本:(分片路径, 行号, JSON 类型名)
|
|
567
|
+
BAD_PAYLOAD_ROW_SKIPS = 0 # 载荷坏型子面:(e 非 dict 且非 None) 条数
|
|
568
|
+
BAD_PAYLOAD_ROW_SAMPLES = [] # 载荷坏型样本:(来源标签, 记录 id, 载荷类型名)
|
|
569
|
+
BAD_SORTKEY_ROW_SKIPS = 0 # 排序键坏型子面:(_t/_s 非数值或 NaN) 条数
|
|
570
|
+
BAD_SORTKEY_ROW_SAMPLES = [] # 排序键坏型样本:(分片路径, 行号, (t 类型名, s 类型名))
|
|
571
|
+
_NONOBJECT_SAMPLE_CAP = 32 # 样本上限:记账要有界,不随坏行线性涨
|
|
572
|
+
|
|
573
|
+
|
|
574
|
+
# 生效条件:skips 为 (分片路径, 行号, 值) 三元组序列且非空时,把条数累加进 NONOBJECT_ROW_SKIPS、按 _NONOBJECT_SAMPLE_CAP 上限补样本,并向 sys.stderr 写一行汇总告警(含分片名、本次条数、首条行号与类型名、累计条数);skips 为空序列或假值时立即返回、不写任何输出。
|
|
575
|
+
def note_nonobject_rows(skips):
|
|
576
|
+
"""登记一批被跳过的非对象记录行 + stderr 告警(N225 可观测面)。"""
|
|
577
|
+
if not skips:
|
|
578
|
+
return
|
|
579
|
+
global NONOBJECT_ROW_SKIPS
|
|
580
|
+
NONOBJECT_ROW_SKIPS += len(skips)
|
|
581
|
+
for item in skips:
|
|
582
|
+
if len(NONOBJECT_ROW_SAMPLES) >= _NONOBJECT_SAMPLE_CAP:
|
|
583
|
+
break
|
|
584
|
+
NONOBJECT_ROW_SAMPLES.append((item[0], item[1],
|
|
585
|
+
type(item[2]).__name__))
|
|
586
|
+
path, lineno, value = skips[0]
|
|
587
|
+
sys.stderr.write(
|
|
588
|
+
"[fsutil] ShardedLog.read_all 跳过 %d 条非对象记录行(分片 %s,"
|
|
589
|
+
"首条 行%d 类型=%s)——坏行不进索引重放;本次后进程内累计 %d 条。"
|
|
590
|
+
"排查方向:该分片被非本协议写入方污染,或发生过截断 / 粘连。\n"
|
|
591
|
+
% (len(skips), os.path.basename(path), lineno,
|
|
592
|
+
type(value).__name__, NONOBJECT_ROW_SKIPS))
|
|
593
|
+
|
|
594
|
+
|
|
595
|
+
# 生效条件:无入参,返回二元组 (累计坏行条数, 非对象行样本元组副本)——累计数为三类合计(非对象行 + 载荷坏型 + 排序键坏型),样本元素为 (分片路径, 行号, JSON 类型名) 且只含非对象行一类,副本只读、调用方改动不影响记账面。
|
|
596
|
+
def nonobject_row_stats():
|
|
597
|
+
"""读坏行记账合计(只读):(累计条数, 非对象行样本元组)。
|
|
598
|
+
|
|
599
|
+
累计数是**同一记账面**的合计(三类 kind 都计入);按 kind 的分项见
|
|
600
|
+
bad_payload_row_stats / bad_sortkey_row_stats。
|
|
601
|
+
"""
|
|
602
|
+
return NONOBJECT_ROW_SKIPS, tuple(NONOBJECT_ROW_SAMPLES)
|
|
603
|
+
|
|
604
|
+
|
|
605
|
+
# 生效条件:skips 为 (来源标签, 记录 id, 载荷值) 三元组序列且非空时,把 len(skips) 同时累加进 NONOBJECT_ROW_SKIPS(同一记账面)与 BAD_PAYLOAD_ROW_SKIPS(子面),按 _NONOBJECT_SAMPLE_CAP 上限补 BAD_PAYLOAD_ROW_SAMPLES(元素为 (来源标签, str(记录 id), 载荷类型名)),并向 sys.stderr 写一行载荷坏型汇总告警(含来源基名、本次条数、首条 id 与类型名、进程内累计条数);skips 为空序列或假值时立即返回——不写输出、不动任何计数。
|
|
606
|
+
def note_bad_payload_rows(skips):
|
|
607
|
+
"""登记一批被跳过的**载荷坏型**记录(`e` 非 dict 且非 None)+ stderr 告警。
|
|
608
|
+
|
|
609
|
+
与 note_nonobject_rows 同一个记账面(合计进 NONOBJECT_ROW_SKIPS),但样本
|
|
610
|
+
与告警文案分开:运维据此区分「行本身不是对象」与「行是对象而载荷坏了」。
|
|
611
|
+
记录**不落进索引**由调用方(重放路径)负责,本函数只记账。计数只在这里加,
|
|
612
|
+
调用方不得另立计数器(`_count_buckets` 那条下游不做类型检查,正是靠这里
|
|
613
|
+
把坏载荷挡在 `nodes` 之外)。
|
|
614
|
+
"""
|
|
615
|
+
if not skips:
|
|
616
|
+
return
|
|
617
|
+
global NONOBJECT_ROW_SKIPS, BAD_PAYLOAD_ROW_SKIPS
|
|
618
|
+
NONOBJECT_ROW_SKIPS += len(skips)
|
|
619
|
+
BAD_PAYLOAD_ROW_SKIPS += len(skips)
|
|
620
|
+
for item in skips:
|
|
621
|
+
if len(BAD_PAYLOAD_ROW_SAMPLES) >= _NONOBJECT_SAMPLE_CAP:
|
|
622
|
+
break
|
|
623
|
+
BAD_PAYLOAD_ROW_SAMPLES.append((item[0], str(item[1]),
|
|
624
|
+
type(item[2]).__name__))
|
|
625
|
+
source, nid, value = skips[0]
|
|
626
|
+
sys.stderr.write(
|
|
627
|
+
"[fsutil] 分片重放跳过 %d 条载荷坏型记录(e 非 dict 且非 None,来源 %s,"
|
|
628
|
+
"首条 id=%s 载荷类型=%s)——坏载荷不进索引节点;本次后进程内累计 %d 条"
|
|
629
|
+
"(含非对象行)。排查方向:该分片被按别的协议写入,或载荷被外部改写。\n"
|
|
630
|
+
% (len(skips), os.path.basename(str(source)), nid,
|
|
631
|
+
type(value).__name__, NONOBJECT_ROW_SKIPS))
|
|
632
|
+
|
|
633
|
+
|
|
634
|
+
# 生效条件:无入参,返回二元组 (载荷坏型累计条数, 样本元组副本)——样本元素为 (来源标签, 记录 id, 载荷类型名),副本只读、调用方改动不影响记账面。
|
|
635
|
+
def bad_payload_row_stats():
|
|
636
|
+
"""读载荷坏型子面(只读):(累计条数, 样本元组)。"""
|
|
637
|
+
return BAD_PAYLOAD_ROW_SKIPS, tuple(BAD_PAYLOAD_ROW_SAMPLES)
|
|
638
|
+
|
|
639
|
+
|
|
640
|
+
# 生效条件:skips 为 (分片路径, 行号, (t 值, s 值)) 三元组序列且非空时,把 len(skips) 同时累加进 NONOBJECT_ROW_SKIPS(同一记账面)与 BAD_SORTKEY_ROW_SKIPS(子面),按 _NONOBJECT_SAMPLE_CAP 上限补 BAD_SORTKEY_ROW_SAMPLES(元素为 (分片路径, 行号, (t 类型名, s 类型名))),并向 sys.stderr 写一行排序键坏型汇总告警(含分片基名、本次条数、首条行号与两个类型名、进程内累计条数);skips 为空序列或假值时立即返回——不写输出、不动任何计数。
|
|
641
|
+
def note_bad_sortkey_rows(skips):
|
|
642
|
+
"""登记一批**排序键坏型**记录(`_t` / `_s` 非数值或 NaN)+ stderr 告警。
|
|
643
|
+
|
|
644
|
+
与 note_nonobject_rows 同一记账面。**记录本身不丢**(仍是 dict,照常回放,
|
|
645
|
+
只在排序键上按「缺键取 0」的既有口径归一)——故本函数只记账、不改记录:
|
|
646
|
+
丢掉一条合法 dict 记录是比重排它更糟的事。
|
|
647
|
+
"""
|
|
648
|
+
if not skips:
|
|
649
|
+
return
|
|
650
|
+
global NONOBJECT_ROW_SKIPS, BAD_SORTKEY_ROW_SKIPS
|
|
651
|
+
NONOBJECT_ROW_SKIPS += len(skips)
|
|
652
|
+
BAD_SORTKEY_ROW_SKIPS += len(skips)
|
|
653
|
+
for item in skips:
|
|
654
|
+
if len(BAD_SORTKEY_ROW_SAMPLES) >= _NONOBJECT_SAMPLE_CAP:
|
|
655
|
+
break
|
|
656
|
+
raw = item[2]
|
|
657
|
+
BAD_SORTKEY_ROW_SAMPLES.append(
|
|
658
|
+
(item[0], item[1], (type(raw[0]).__name__, type(raw[1]).__name__)))
|
|
659
|
+
path, lineno, raw = skips[0]
|
|
660
|
+
sys.stderr.write(
|
|
661
|
+
"[fsutil] ShardedLog.read_all 排序键坏型 %d 条(分片 %s,首条 行%d "
|
|
662
|
+
"_t=%s _s=%s)——坏型槽记 0 参与排序(记录不丢);本次后进程内累计 %d 条"
|
|
663
|
+
"(含非对象行)。排查方向:该分片被非本协议写入方污染。\n"
|
|
664
|
+
% (len(skips), os.path.basename(path), lineno,
|
|
665
|
+
type(raw[0]).__name__, type(raw[1]).__name__, NONOBJECT_ROW_SKIPS))
|
|
666
|
+
|
|
667
|
+
|
|
668
|
+
# 生效条件:无入参,返回二元组 (排序键坏型累计条数, 样本元组副本)——样本元素为 (分片路径, 行号, (t 类型名, s 类型名)),副本只读、调用方改动不影响记账面。
|
|
669
|
+
def bad_sortkey_row_stats():
|
|
670
|
+
"""读排序键坏型子面(只读):(累计条数, 样本元组)。"""
|
|
671
|
+
return BAD_SORTKEY_ROW_SKIPS, tuple(BAD_SORTKEY_ROW_SAMPLES)
|
|
672
|
+
|
|
673
|
+
|
|
674
|
+
# 生效条件:无入参,把三类累计条数(NONOBJECT_ROW_SKIPS / BAD_PAYLOAD_ROW_SKIPS / BAD_SORTKEY_ROW_SKIPS)全置 0 并把三个样本列表原地清空(del [:],不换对象);守卫与运维建立观测基线时用,生产读路径不调用。
|
|
675
|
+
def reset_nonobject_row_stats():
|
|
676
|
+
"""清空整个坏行记账面(含两类子面;守卫 / 运维的观测基线用)。"""
|
|
677
|
+
global NONOBJECT_ROW_SKIPS, BAD_PAYLOAD_ROW_SKIPS, BAD_SORTKEY_ROW_SKIPS
|
|
678
|
+
NONOBJECT_ROW_SKIPS = 0
|
|
679
|
+
BAD_PAYLOAD_ROW_SKIPS = 0
|
|
680
|
+
BAD_SORTKEY_ROW_SKIPS = 0
|
|
681
|
+
del NONOBJECT_ROW_SAMPLES[:]
|
|
682
|
+
del BAD_PAYLOAD_ROW_SAMPLES[:]
|
|
683
|
+
del BAD_SORTKEY_ROW_SAMPLES[:]
|
|
684
|
+
|
|
685
|
+
|
|
686
|
+
# ---------- C-3(FI-M02 / N134,2026-09-29):读失败的**瞬时/终态单点判别**与记账 ----------
|
|
687
|
+
#
|
|
688
|
+
# 缺陷(N134,docs/eval/缺陷挖掘_自主迭代_v16.md:86):`MdCG._read` 把 OSError
|
|
689
|
+
# (含**瞬态**失败:独占句柄 / 资源剥夺)与「文件不存在 / 越界」一律归成
|
|
690
|
+
# `(None, None)`,readcache 又把任何返回值——包括 `(None, None)`——一律当正常值
|
|
691
|
+
# 写入缓存 ⇒ 一次瞬态 OS 失败被固化成「该节点从检索面永久消失,直到进程重启或
|
|
692
|
+
# 该 path 再写盘」(cache 条目字面 `(gen, (None, None))`);而 `cg.get` 直读不走
|
|
693
|
+
# 缓存照常可读 ⇒「get 能读、search 搜不到」撕裂(P1 fail-closed 缺席 + T4 静默损伤)。
|
|
694
|
+
#
|
|
695
|
+
# 修法要点:**判别只此一处**——读路径在唯一捕获 OSError 的点上调用本函数拿到
|
|
696
|
+
# 「瞬时 / 终态」标签,缓存层按标签决定接纳与否,**不再自行回看异常类型**
|
|
697
|
+
# (否则就是判据的第二份副本;本仓 N133 与 `_ccg_line` 的教训都是「副本自称同源」)。
|
|
698
|
+
# · 终态(READ_FAIL_ABSENT):`FileNotFoundError`——`atomic_write` 是
|
|
699
|
+
# tmp + os.replace(见 `_publish`:读者只可能看到旧值或新值,不会看到
|
|
700
|
+
# 「写一半的缺文件」),故 ENOENT 是**真缺**,可照旧入缓存(不构成重读风暴)。
|
|
701
|
+
# 越界(`_node_disk_path` 抛 ValueError)与「有节点但无密钥」同属终态,
|
|
702
|
+
# 且不经本函数——两条既有语义逐位不变。
|
|
703
|
+
# · 瞬时(READ_FAIL_TRANSIENT):其余 OSError——Windows 上独占句柄
|
|
704
|
+
# (WinError 32 共享冲突)、权限剥夺、同名目录顶位、磁盘瞬时故障。
|
|
705
|
+
# **判据方向是 fail-closed 的**:判不准(非 FileNotFoundError 的一切)
|
|
706
|
+
# 一律归「瞬时」——宁可多读一次,不可把可读节点判死。
|
|
707
|
+
#
|
|
708
|
+
# ③「不得静默」:与 N225 三类坏行记账同风格——模块级累计计数 + 有界样本 +
|
|
709
|
+
# stderr 汇总告警;计数面经 `transient_read_stats()` 可被守卫/运维读取。
|
|
710
|
+
READ_FAIL_ABSENT = "absent" # 终态:真缺 / 越界 / 无密钥 → 可入缓存
|
|
711
|
+
READ_FAIL_TRANSIENT = "transient" # 瞬时:可重试 → **不得**以「新鲜」身份固化
|
|
712
|
+
|
|
713
|
+
TRANSIENT_READ_FAILURES = 0 # 累计瞬时读失败次数(进程内)
|
|
714
|
+
TRANSIENT_READ_SAMPLES = [] # 样本:(节点 path, 异常类型名, errno)
|
|
715
|
+
_TRANSIENT_READ_SAMPLE_CAP = 32 # 样本上限:记账要有界,不随失败次数线性涨
|
|
716
|
+
# stderr 告警上限:热路径(全池检索每查询逐条读)防刷屏;**计数面恒完整**,
|
|
717
|
+
# 上限只压告警行数,不减信息可观测性(守卫读的是计数与样本,不是 stderr)。
|
|
718
|
+
_TRANSIENT_READ_WARN_CAP = 32
|
|
719
|
+
|
|
720
|
+
|
|
721
|
+
# 生效条件:exc 为读节点文件时捕获的异常对象;是 FileNotFoundError(含其子类)返回 READ_FAIL_ABSENT(终态:真缺,可入缓存),否则返回 READ_FAIL_TRANSIENT(瞬时:可重试,不得入缓存);非 OSError 入参同样按瞬时返回(判不准即保守,绝不判死节点)。
|
|
722
|
+
def classify_read_failure(exc) -> str:
|
|
723
|
+
"""**单点**判别:读失败是瞬时的还是终态的(唯一真源,缓存层不得再猜一遍)。
|
|
724
|
+
|
|
725
|
+
调用面恒为读路径捕获 OSError 的那一处(`MdCG._note_read_oserror`)。
|
|
726
|
+
"""
|
|
727
|
+
if isinstance(exc, FileNotFoundError):
|
|
728
|
+
return READ_FAIL_ABSENT
|
|
729
|
+
return READ_FAIL_TRANSIENT
|
|
730
|
+
|
|
731
|
+
|
|
732
|
+
# 生效条件:path 为节点文件路径(任意值,str() 后取基名入样本)、exc 为捕获到的异常;无条件把 TRANSIENT_READ_FAILURES 累加 1、按 _TRANSIENT_READ_SAMPLE_CAP 上限补样本,并在累计次数不超过 _TRANSIENT_READ_WARN_CAP 时向 sys.stderr 写一行汇总告警;返回是否写了告警行(bool)。
|
|
733
|
+
def note_transient_read_failure(path, exc) -> bool:
|
|
734
|
+
"""登记一次**瞬时读失败** + stderr 告警(C-3 可观测面,N225 同风格)。
|
|
735
|
+
|
|
736
|
+
只记账,**不改任何调用方的控制流**——读不到仍是读不到,只是不再静默。
|
|
737
|
+
"""
|
|
738
|
+
global TRANSIENT_READ_FAILURES
|
|
739
|
+
TRANSIENT_READ_FAILURES += 1
|
|
740
|
+
if len(TRANSIENT_READ_SAMPLES) < _TRANSIENT_READ_SAMPLE_CAP:
|
|
741
|
+
TRANSIENT_READ_SAMPLES.append(
|
|
742
|
+
(str(path), type(exc).__name__, getattr(exc, "errno", None)))
|
|
743
|
+
if TRANSIENT_READ_FAILURES > _TRANSIENT_READ_WARN_CAP:
|
|
744
|
+
return False
|
|
745
|
+
sys.stderr.write(
|
|
746
|
+
"[fsutil] 节点读失败(瞬时,可重试)%s:%s errno=%s——本次不计入检索面,"
|
|
747
|
+
"且**不当作「不存在」固化**(C-3:负结果不入读缓存);进程内累计 %d 次。"
|
|
748
|
+
"排查方向:独占句柄(Windows 共享冲突)/ 权限剥夺 / 同名目录顶位 / "
|
|
749
|
+
"磁盘瞬时故障。\n"
|
|
750
|
+
% (os.path.basename(str(path)), type(exc).__name__,
|
|
751
|
+
getattr(exc, "errno", None), TRANSIENT_READ_FAILURES))
|
|
752
|
+
return True
|
|
753
|
+
|
|
754
|
+
|
|
755
|
+
# 生效条件:无入参,返回二元组 (累计瞬时读失败次数, 样本元组副本)——样本元素为 (节点 path, 异常类型名, errno),副本只读、调用方改动不影响记账面。
|
|
756
|
+
def transient_read_stats():
|
|
757
|
+
"""读瞬时读失败记账(只读):(累计次数, 样本元组)。"""
|
|
758
|
+
return TRANSIENT_READ_FAILURES, tuple(TRANSIENT_READ_SAMPLES)
|
|
759
|
+
|
|
760
|
+
|
|
761
|
+
# 生效条件:无入参,把 TRANSIENT_READ_FAILURES 置 0 并原地清空 TRANSIENT_READ_SAMPLES(del [:],不换对象);守卫/运维建立观测基线时用,生产读路径不调用。
|
|
762
|
+
def reset_transient_read_stats():
|
|
763
|
+
"""清空瞬时读失败记账面(守卫 / 运维的观测基线用)。"""
|
|
764
|
+
global TRANSIENT_READ_FAILURES
|
|
765
|
+
TRANSIENT_READ_FAILURES = 0
|
|
766
|
+
del TRANSIENT_READ_SAMPLES[:]
|
|
767
|
+
|
|
768
|
+
|
|
769
|
+
# 生效条件:os.stat(os.path.abspath(path or "")) 抛 OSError 时返回 0;缓存命中且已扫字节数与 mtime_ns 均与 stat 一致时直接返回缓存计数;若缓存已扫字节 < 当前 size 且 mtime_ns 不同则从该偏移起按 chunk 分块累计 b"\n" 个数并加上缓存值;读文件抛 OSError 时返回 total or 0(已累计值为假则返回 0)。
|
|
770
|
+
def count_jsonl(path: str, chunk: int = 1 << 20) -> int:
|
|
771
|
+
"""数 append-only 日志的行数——**流式计数、不物化**(内存 O(1))。
|
|
772
|
+
|
|
773
|
+
存在的唯一理由:`len(list(read_jsonl(p)))` 是**危险的默认写法**。它把整份
|
|
774
|
+
日志解析成对象列表,日志一长就是灾难——本机实测(2026-09-16):审计日志
|
|
775
|
+
4.3 GB,一次 health/盘点调用解析速率 ~1 MB/s、RSS 涨到 5 GB+ 仍在涨、
|
|
776
|
+
数十分钟不返回;调用方超时重试又在别的进程里再排一次队,最终把整条 MCP
|
|
777
|
+
通道堵死(一个只读的「体检」把服务打死,代价与收益完全不成比例)。
|
|
778
|
+
这里只数字节里的换行:不解析、不驻留,速度只受磁盘限制;进程内按
|
|
779
|
+
(已扫字节数, mtime) 缓存,文件只增长时只扫新增字节(O(增量))。
|
|
780
|
+
|
|
781
|
+
语义边界(诚实声明):数的是**换行符**,不是 JSON 记录——
|
|
782
|
+
① 完整写入的日志(`append_jsonl` 每条尾带 `\\n`)换行数 == 记录数,与
|
|
783
|
+
`len(list(read_jsonl(p)))` 逐位相等(test_health_scale ⑦ 守卫);
|
|
784
|
+
② 末尾**未终止的半截行**不计入(下界,最多差 1 行;见 append_jsonl 的
|
|
785
|
+
ends_mid_line 修补分支——并发交错被杀的进程会留下这种尾巴);
|
|
786
|
+
③ 非法 JSON 行计入行数而 `read_jsonl` 会跳过——健康度是量级指标,不做
|
|
787
|
+
逐行校验,逐行解析正是上面那场事故的根因。
|
|
788
|
+
"""
|
|
789
|
+
key = os.path.abspath(path or "")
|
|
790
|
+
try:
|
|
791
|
+
st = os.stat(key)
|
|
792
|
+
except OSError:
|
|
793
|
+
return 0
|
|
794
|
+
start, total = 0, 0
|
|
795
|
+
prev = _COUNT_CACHE.get(key)
|
|
796
|
+
if prev:
|
|
797
|
+
p_scan, p_mtime, p_count = prev
|
|
798
|
+
if p_scan == st.st_size and p_mtime == st.st_mtime_ns:
|
|
799
|
+
return p_count # 完全未变:零 IO
|
|
800
|
+
if p_scan < st.st_size and p_mtime != st.st_mtime_ns:
|
|
801
|
+
start, total = p_scan, p_count # 只增长:扫增量
|
|
802
|
+
scanned, lines = start, 0
|
|
803
|
+
try:
|
|
804
|
+
with open(key, "rb") as f:
|
|
805
|
+
if start:
|
|
806
|
+
f.seek(start)
|
|
807
|
+
while True:
|
|
808
|
+
buf = f.read(chunk)
|
|
809
|
+
if not buf:
|
|
810
|
+
break
|
|
811
|
+
scanned += len(buf)
|
|
812
|
+
lines += buf.count(b"\n")
|
|
813
|
+
except OSError:
|
|
814
|
+
return total or 0
|
|
815
|
+
total += lines
|
|
816
|
+
_COUNT_CACHE[key] = (scanned, st.st_mtime_ns, total)
|
|
817
|
+
return total
|
|
818
|
+
|
|
819
|
+
|
|
820
|
+
# ---------- B4(2026-09-30):分片目录缺失自愈(容忍 ≠ 静默;失败结构化) ----------
|
|
821
|
+
#
|
|
822
|
+
# 病灶:`ShardedLog` 只在 `__init__` 里建目录,`append` 首次 `open(self.path, "a")`
|
|
823
|
+
# 前不复查。**长驻进程**(生产形态 `MdCGSecure(root, principal, autoflush=1)`——
|
|
824
|
+
# `MdCG.flush` 每批写完全部 `close()` 分片句柄,故下次 append 必经重开)持有过
|
|
825
|
+
# `ShardedLog` 实例之后目录被删(外部清理脚本 / 误删 / 备份还原),此后每次
|
|
826
|
+
# append 都在 open 处抛 `FileNotFoundError [Errno 2]`,**且永不恢复**:目录三次
|
|
827
|
+
# 都不重建、`close()` 抛同异常、`_index.json` 从未落成,而正文 .md 已在盘上
|
|
828
|
+
# (`MdCG.add` → `_write_node` 先写盘)——索引静默落后于盘面。不自愈的必要条件
|
|
829
|
+
# 是「本进程已持有 ShardedLog 实例」:`MdCG.flush` 只在该属性为 None 时建实例
|
|
830
|
+
# (`mdcg.py` 内 `ShardedLog(...)` 构造点全仓唯一),全新短命进程做同一操作时
|
|
831
|
+
# `__init__` 的 makedirs 生效、目录会被重建
|
|
832
|
+
# (2026-09-30 沙箱实测两侧,见 `test_b4_shard_dir_selfheal.py` 头注)。
|
|
833
|
+
#
|
|
834
|
+
# 收口口径(本块是**唯一**实现;第二处调用点一律委托 `ensure_shard_dir`):
|
|
835
|
+
# ① 判据 = 「分片目录不存在就重建」——`isdir` 快路径 → `makedirs(exist_ok=True)`
|
|
836
|
+
# → 失败抛结构化错误;`__init__`(首建)与 `append`(打开前)都走它。
|
|
837
|
+
# ② **容忍 ≠ 静默**(本仓 N225 已确立的判据):重建必须可观测——模块级累计
|
|
838
|
+
# 计数 + 有界样本 + stderr 汇总告警(与 `note_nonobject_rows` 同一风格:
|
|
839
|
+
# 计数只在这里加,调用方不得另立计数器)。**首建不记账**(`where="init"`:
|
|
840
|
+
# 建库时目录本就不存在,那是正常路径,不是「被删后自愈」的病态事件——
|
|
841
|
+
# 计入会让计数面变成噪声、真事件被淹没)。
|
|
842
|
+
# ③ 重建**失败**(父只读 / 权限不足 / 有文件占着该路径):抛 `ShardDirError`
|
|
843
|
+
# —— 可机读 `code` + 可操作 `hint`,不让裸 `FileNotFoundError` 冒到 MCP
|
|
844
|
+
# 出口(出口只渲染 `f"{type(exc).__name__}: {exc}"` + `getattr(exc,
|
|
845
|
+
# "hint")`,裸异常在那里既无 code 也无 hint)。失败**同样记账 + 告警**:
|
|
846
|
+
# 异常可能被上层兜底吞掉(`MdCG.close` 的 `except (OSError, ValueError)`),
|
|
847
|
+
# 痕迹不得只存在于异常里。
|
|
848
|
+
# ④ `read_all` 既有语义**一字不改**:directory 不是目录 → 返回 `[]`(那是读面
|
|
849
|
+
# 的降级契约,与本块写面自愈无关)。
|
|
850
|
+
SHARD_DIR_REBUILDS = 0 # 累计「目录缺失→重建成功」次数(进程内)
|
|
851
|
+
SHARD_DIR_REBUILD_SAMPLES = [] # 重建样本:(目录绝对路径, 调用点标签)
|
|
852
|
+
SHARD_DIR_HEAL_FAILURES = 0 # 累计「目录缺失且重建失败」次数(进程内)
|
|
853
|
+
SHARD_DIR_HEAL_FAILURE_SAMPLES = [] # 失败样本:(目录绝对路径, 调用点, 异常类型名)
|
|
854
|
+
_SHARD_DIR_SAMPLE_CAP = 8 # 样本上限:记账有界,不随事件线性涨
|
|
855
|
+
|
|
856
|
+
# 结构化失败的稳定 code(进 message 首字段 ⇒ MCP 出口的 error 串里可机读;
|
|
857
|
+
# 出口不改形状,与 AccessDenied 的 hint 同口径只透 `hint`)。
|
|
858
|
+
SHARD_DIR_ERR_CODE = "E_SHARD_DIR_UNREBUILDABLE"
|
|
859
|
+
|
|
860
|
+
|
|
861
|
+
# 生效条件:msg 为必填字符串(经 super().__init__ 原样成为 str(e)),code / hint / path 任选(缺省 None);构造出的是 OSError 子类实例,三者分别存入 self.code / self.hint / self.path,不校验取值、不读盘、不抛异常。
|
|
862
|
+
class ShardDirError(OSError):
|
|
863
|
+
"""分片目录缺失且无法重建——结构化失败(可机读 code + 可操作 hint)。
|
|
864
|
+
|
|
865
|
+
为什么继承 `OSError`(而不是 RuntimeError):既有调用面按 `except OSError`
|
|
866
|
+
收敛 I/O 失败(`MdCG.close` 的兜底、各处 `except (OSError, ValueError)`),
|
|
867
|
+
换基类会让这些既有的降级/兜底面行为漂移;本类要补的是**信息**(code /
|
|
868
|
+
hint),不是新的异常族。`.hint` 与 `security.AccessDenied` 同口径——MCP
|
|
869
|
+
出口的 `getattr(exc, "hint")` 会把它渲染进工具错误结果。
|
|
870
|
+
"""
|
|
871
|
+
|
|
872
|
+
# 生效条件:msg 为必填字符串(经 super().__init__ 原样成为 str(e)),code / hint / path 任选(缺省 None);随后把三者分别存入 self.code / self.hint / self.path,不校验取值、不读盘。
|
|
873
|
+
def __init__(self, msg: str, code: str = None, hint: str = None,
|
|
874
|
+
path: str = None):
|
|
875
|
+
super().__init__(msg)
|
|
876
|
+
self.code = code
|
|
877
|
+
self.hint = hint
|
|
878
|
+
self.path = path
|
|
879
|
+
|
|
880
|
+
|
|
881
|
+
# 生效条件:where 非 "init" 时把 SHARD_DIR_REBUILDS 累加 1、按 _SHARD_DIR_SAMPLE_CAP 上限补 (目录绝对路径, where) 样本,并向 sys.stderr 写一行含目录名/调用点/累计次数与排查方向的重建告警;where == "init" 时立即返回(首建是正常路径,不记账不告警);stderr 写失败被吞掉(告警面不得反向破坏写路径),计数与样本不受影响。
|
|
882
|
+
def note_shard_dir_rebuild(directory, where: str = "append"):
|
|
883
|
+
"""登记一次分片目录重建(成功)+ stderr 告警(B4 可观测面)。"""
|
|
884
|
+
if where == "init":
|
|
885
|
+
return
|
|
886
|
+
global SHARD_DIR_REBUILDS
|
|
887
|
+
SHARD_DIR_REBUILDS += 1
|
|
888
|
+
if len(SHARD_DIR_REBUILD_SAMPLES) < _SHARD_DIR_SAMPLE_CAP:
|
|
889
|
+
SHARD_DIR_REBUILD_SAMPLES.append((os.path.abspath(directory), where))
|
|
890
|
+
try:
|
|
891
|
+
sys.stderr.write(
|
|
892
|
+
"[fsutil] ShardedLog 分片目录缺失,已重建(目录 %s,调用点 %s)——"
|
|
893
|
+
"本进程持有的分片实例不自愈;本次后进程内累计重建 %d 次。"
|
|
894
|
+
"排查方向:库根 _index_log/ 被外部清理/还原删掉,或库根被换过;"
|
|
895
|
+
"正文 .md 未受影响,索引缺口由重放/全库扫描补齐。\n"
|
|
896
|
+
% (os.path.basename(os.path.abspath(directory)), where,
|
|
897
|
+
SHARD_DIR_REBUILDS))
|
|
898
|
+
except Exception: # noqa: BLE001 —— 告警面不反向破坏写路径
|
|
899
|
+
pass
|
|
900
|
+
|
|
901
|
+
|
|
902
|
+
# 生效条件:无条件把 SHARD_DIR_HEAL_FAILURES 累加 1、按 _SHARD_DIR_SAMPLE_CAP 上限补 (目录绝对路径, where, 异常类型名) 样本,并向 sys.stderr 写一行含失败原因类型与 hint 指向的告警;exc 为 None 时原因类型按 "NoneType" 渲染;stderr 写失败被吞掉(计数与样本不受影响),本函数不抛异常(真正的失败由调用方抛 ShardDirError)。
|
|
903
|
+
def note_shard_dir_heal_failure(directory, where, exc=None):
|
|
904
|
+
"""登记一次「目录缺失且重建失败」+ stderr 告警(B4 可观测面)。"""
|
|
905
|
+
global SHARD_DIR_HEAL_FAILURES
|
|
906
|
+
SHARD_DIR_HEAL_FAILURES += 1
|
|
907
|
+
if len(SHARD_DIR_HEAL_FAILURE_SAMPLES) < _SHARD_DIR_SAMPLE_CAP:
|
|
908
|
+
SHARD_DIR_HEAL_FAILURE_SAMPLES.append(
|
|
909
|
+
(os.path.abspath(directory), where,
|
|
910
|
+
type(exc).__name__ if exc is not None else "NoneType"))
|
|
911
|
+
try:
|
|
912
|
+
sys.stderr.write(
|
|
913
|
+
"[fsutil] ShardedLog 分片目录缺失且**重建失败**(目录 %s,调用点 %s,"
|
|
914
|
+
"原因 %s: %s)——抛 %s(code=%s),不降级为裸 FileNotFoundError;"
|
|
915
|
+
"本次后进程内累计失败 %d 次。\n"
|
|
916
|
+
% (os.path.basename(os.path.abspath(directory)), where,
|
|
917
|
+
type(exc).__name__ if exc is not None else "NoneType", exc,
|
|
918
|
+
ShardDirError.__name__, SHARD_DIR_ERR_CODE,
|
|
919
|
+
SHARD_DIR_HEAL_FAILURES))
|
|
920
|
+
except Exception: # noqa: BLE001 —— 告警面不反向破坏写路径
|
|
921
|
+
pass
|
|
922
|
+
|
|
923
|
+
|
|
924
|
+
# 生效条件:无入参,返回四元组 (重建累计次数, 重建样本元组, 失败累计次数, 失败样本元组)——样本元素分别为 (目录绝对路径, 调用点标签) 与 (目录绝对路径, 调用点标签, 异常类型名);元组为副本,调用方改动不影响记账面。
|
|
925
|
+
def shard_dir_stats():
|
|
926
|
+
"""读分片目录自愈记账(只读):(重建数, 重建样本, 失败数, 失败样本)。"""
|
|
927
|
+
return (SHARD_DIR_REBUILDS, tuple(SHARD_DIR_REBUILD_SAMPLES),
|
|
928
|
+
SHARD_DIR_HEAL_FAILURES, tuple(SHARD_DIR_HEAL_FAILURE_SAMPLES))
|
|
929
|
+
|
|
930
|
+
|
|
931
|
+
# 生效条件:无入参、无返回值;把 SHARD_DIR_REBUILDS / SHARD_DIR_HEAL_FAILURES 归零并清空两个样本列表(守卫与运维读面前的重置点),只动本进程计数面、不触盘面。
|
|
932
|
+
def reset_shard_dir_stats():
|
|
933
|
+
"""清零分片目录自愈记账(守卫/运维用;只影响本进程计数面)。"""
|
|
934
|
+
global SHARD_DIR_REBUILDS, SHARD_DIR_HEAL_FAILURES
|
|
935
|
+
SHARD_DIR_REBUILDS = 0
|
|
936
|
+
SHARD_DIR_HEAL_FAILURES = 0
|
|
937
|
+
del SHARD_DIR_REBUILD_SAMPLES[:]
|
|
938
|
+
del SHARD_DIR_HEAL_FAILURE_SAMPLES[:]
|
|
939
|
+
|
|
940
|
+
|
|
941
|
+
# 生效条件:directory 为分片目录路径时,os.path.isdir(directory) 为真立即返回 False(零动作、零记账);为假则 os.makedirs(directory, exist_ok=True)——成功时经 note_shard_dir_rebuild 按 where 记账("init" 不记)+ stderr 告警并返回 True;makedirs 抛 OSError(权限不足 / 有文件占着该路径 / 路径不可达)时先 note_shard_dir_heal_failure 记账 + 告警,再抛 ShardDirError(code=SHARD_DIR_ERR_CODE、message 首字段为 [code]、hint 含目录与三步处置、path=directory),cause 链(raise ... from)保留原异常;本函数是分片目录存在性的唯一实现点。
|
|
942
|
+
def ensure_shard_dir(directory: str, where: str = "append") -> bool:
|
|
943
|
+
"""`ShardedLog` 分片目录的**唯一**存在性保证点(B4)。
|
|
944
|
+
|
|
945
|
+
返回 True 表示本次**建了目录**(缺失→重建),False 表示目录本来就在。
|
|
946
|
+
where 只影响记账口径("init" = `ShardedLog.__init__` 首建,正常路径不记;
|
|
947
|
+
"append" = 打开分片前的病态自愈面,记账 + 告警)。
|
|
948
|
+
"""
|
|
949
|
+
if os.path.isdir(directory):
|
|
950
|
+
return False
|
|
951
|
+
try:
|
|
952
|
+
os.makedirs(directory, exist_ok=True)
|
|
953
|
+
except OSError as exc:
|
|
954
|
+
note_shard_dir_heal_failure(directory, where, exc)
|
|
955
|
+
raise ShardDirError(
|
|
956
|
+
"[%s] 分片目录不存在且重建失败:%s(底层 %s: %s)"
|
|
957
|
+
% (SHARD_DIR_ERR_CODE, directory, type(exc).__name__, exc),
|
|
958
|
+
code=SHARD_DIR_ERR_CODE,
|
|
959
|
+
hint=("分片目录 %s 不存在且无法重建(父目录不可写 / 权限不足 / 有文件"
|
|
960
|
+
"占着该路径)。处置:①确认库根可写(Windows:去掉只读属性或"
|
|
961
|
+
"改用有写权限的账号;POSIX:chmod/所有权);②查是否有进程或"
|
|
962
|
+
"杀毒软件把该目录删掉或占住;③索引日志是派生面——正文 .md 未"
|
|
963
|
+
"受影响,修好权限后重开进程即可由重放 / 全库扫描恢复索引,"
|
|
964
|
+
"无需重写节点。" % directory),
|
|
965
|
+
path=directory) from exc
|
|
966
|
+
note_shard_dir_rebuild(directory, where)
|
|
967
|
+
return True
|
|
968
|
+
|
|
969
|
+
|
|
970
|
+
# 生效条件:directory 经 abspath 存入 self.dir 后经 ensure_shard_dir(self.dir, "init") 保证目录存在(首建不记账;失败抛 ShardDirError 而非裸 OSError),实例分片文件名由 os.getpid() 与 uuid.uuid4().hex[:8] 拼成 "{pid}-{hex8}.log",append 时各写者只写自己这一分片,从而不共享写入点。
|
|
971
|
+
class ShardedLog:
|
|
972
|
+
"""每写者独占一个分片的 append-only 日志——不能丢记录时用它。
|
|
973
|
+
|
|
974
|
+
单文件 append 要靠 O_APPEND 的原子性,而那在 Windows 上不成立。分片则连
|
|
975
|
+
「共享写入点」都没有:进程 A 写 A 的文件,进程 B 写 B 的文件,物理上无从冲突。
|
|
976
|
+
代价是读取要合并 N 个分片,靠记录里的单调序号 (t, seq) 恢复全局写入顺序。
|
|
977
|
+
"""
|
|
978
|
+
|
|
979
|
+
# 生效条件:directory 经 abspath 存入 self.dir 并经 ensure_shard_dir(self.dir, "init") 保证存在(唯一实现点;首建不记账,失败抛 ShardDirError),self.path 为 self.dir 下 "{os.getpid()}-{uuid.uuid4().hex[:8]}.log",并置 self._seq = 0、self._fh = None。
|
|
980
|
+
def __init__(self, directory: str):
|
|
981
|
+
self.dir = os.path.abspath(directory)
|
|
982
|
+
# B4:建目录单点收口到 ensure_shard_dir(不再各写各的 makedirs)——
|
|
983
|
+
# where="init" 是首建(正常路径,不记账);重建失败的记账 + 告警与
|
|
984
|
+
# 结构化错误由该单点负责(原来这里是裸 OSError 直冒调用方)。
|
|
985
|
+
ensure_shard_dir(self.dir, "init")
|
|
986
|
+
self.path = os.path.join(
|
|
987
|
+
self.dir, f"{os.getpid()}-{uuid.uuid4().hex[:8]}.log")
|
|
988
|
+
self._seq = 0
|
|
989
|
+
self._fh = None
|
|
990
|
+
|
|
991
|
+
# 生效条件:self._seq 先自增 1,record 被 dict(record, _t=time.time(), _s=self._seq) 复制;self._fh 为 None 时先经 ensure_shard_dir(self.dir, "append") 复查并(缺失即)重建分片目录——重建成功记账 + stderr 告警、失败抛 ShardDirError——再以 "a"、encoding="utf-8"、newline="\n" 打开 self.path(路径照常,不换分片名),随后写入 json.dumps(ensure_ascii=False, separators=(",", ":")) + "\n" 并 flush。
|
|
992
|
+
def append(self, record: dict):
|
|
993
|
+
self._seq += 1
|
|
994
|
+
record = dict(record, _t=time.time(), _s=self._seq)
|
|
995
|
+
if self._fh is None:
|
|
996
|
+
# B4(2026-09-30):**每次重开分片前**复查目录。病灶:目录在进程
|
|
997
|
+
# 存活期间被删后,原实现在这里 `open` 抛 FileNotFoundError 且此后
|
|
998
|
+
# 永不恢复(实测三连抛、目录三次都不重建、close 抛同异常)。缺失
|
|
999
|
+
# 即重建,路径照常仍是 self.path(同一分片名,只补回目录这一层)。
|
|
1000
|
+
# 边界(诚实声明):本条只在**句柄为 None 的重开点**复查——已持有
|
|
1001
|
+
# 打开句柄期间目录被删不在本守卫面内(Windows 上被打开的文件无法
|
|
1002
|
+
# 删除,生产写点 `MdCG.flush` 每批写完即 close ⇒ 必经此重开点)。
|
|
1003
|
+
ensure_shard_dir(self.dir, "append")
|
|
1004
|
+
self._fh = open(self.path, "a", encoding="utf-8", newline="\n")
|
|
1005
|
+
self._fh.write(json.dumps(record, ensure_ascii=False,
|
|
1006
|
+
separators=(",", ":")) + "\n")
|
|
1007
|
+
self._fh.flush()
|
|
1008
|
+
|
|
1009
|
+
# 生效条件:幂等;self._fh 为真值时 flush 并关闭句柄、再把 self._fh 置 None,为 None 时直接返回不报错;
|
|
1010
|
+
def close(self):
|
|
1011
|
+
if self._fh:
|
|
1012
|
+
self._fh.close()
|
|
1013
|
+
self._fh = None
|
|
1014
|
+
|
|
1015
|
+
@staticmethod
|
|
1016
|
+
# 生效条件:directory 是目录时,按 sorted(os.listdir(directory)) 顺序对每个以 ".log" 结尾的文件读取汇总(单分片 PermissionError 时以 5ms×8 短重试等 Windows delete-pending 窗口过去、窗口后 FileNotFoundError 视为已被 compact 并入快照清走而跳过、重试耗尽照常 raise),逐行只收 dict 记录——非对象行(null/[]/123/"abc" 等合法 JSON)跳过并经 note_nonobject_rows 记账 + stderr 告警(不得静默);随后按排序键 (_norm(r.get("_t", 0)), _norm(r.get("_s", 0))) 排序,其中 _norm 只把「数值(int/float,含 bool)且非 NaN」原样透传、其余槽(str/list/dict/None/NaN)一律记 0——坏型槽经 note_bad_sortkey_rows 记账 + stderr 告警(不得静默),记录本身不丢;合法记录(_t/_s 均数值,缺键按默认值 0)的排序键逐位等于旧键 (r.get("_t", 0), r.get("_s", 0)),故其相对次序与改前逐位一致;directory 不是目录时直接返回 []。
|
|
1017
|
+
def read_all(directory: str):
|
|
1018
|
+
"""按全局写入顺序回放所有分片。
|
|
1019
|
+
|
|
1020
|
+
N170(2026-09-27):并发 compact 的 delete-pending 窗口容忍。他进程
|
|
1021
|
+
close→compact_index 的 ShardedLog.clear(mdcg.py:1137,锁内「先并快照
|
|
1022
|
+
再删分片」)与本读方的 listdir→open 交错时,Windows 上 open 命中
|
|
1023
|
+
「已 remove、名未消」的 delete-pending 态 → PermissionError [Errno 13]
|
|
1024
|
+
(test_review_conformance【9】4 decide worker 同根并发实测复现:
|
|
1025
|
+
worker 在 MdCGOS.__init__ 崩溃、stdout 空,父进程 json.loads("")
|
|
1026
|
+
二次崩成 JSONDecodeError)。短重试等窗口过去:窗口过后文件要么可读、
|
|
1027
|
+
要么已真删。已真删(FileNotFoundError)跳过是**安全**的——clear 的
|
|
1028
|
+
契约是分片记录先并入快照再删(mdcg.py:1136-1137 顺序),本读方的
|
|
1029
|
+
快照基底的陈旧读界与既有「他进程未 flush 写入不可见」边界同格,
|
|
1030
|
+
由 _index_signature 指纹机制在下次重载收敛;重试耗尽的 PermissionError
|
|
1031
|
+
照常上抛,真权限问题不掩盖。
|
|
1032
|
+
|
|
1033
|
+
N225(2026-09-29):**非对象行容忍 + 可观测**。`read_jsonl` 按 JSON
|
|
1034
|
+
合法性收行,`null` / `[]` / `123` / `"abc"` 都会原样产出;直接进
|
|
1035
|
+
`recs` 会让下面的 `r.get("_t", 0)` 抛 AttributeError(实测:MdCG
|
|
1036
|
+
构造 + compact_index 双崩)。此处逐行分流:dict 进回放列表,非对象行
|
|
1037
|
+
进记账面(note_nonobject_rows → 模块级计数 + 样本 + stderr 告警)。
|
|
1038
|
+
分流必须在本层做——`read_jsonl` 的通用契约(其它消费面)不在本次
|
|
1039
|
+
范围,不得改。整片读成功才记账:半途重试不重复计数。
|
|
1040
|
+
|
|
1041
|
+
N225 补强(2026-09-29,**排序键健壮化**):行是 dict 不等于键可排序
|
|
1042
|
+
——`_t` / `_s` 被外部写成 `"abc"` / `[]` / `null` 时,旧键
|
|
1043
|
+
`(r.get("_t", 0), r.get("_s", 0))` 在 sort 里抛 TypeError
|
|
1044
|
+
(`'<' not supported between instances of 'str' and 'float'`),
|
|
1045
|
+
而 except 面只有 ValueError/OSError ⇒ 与 N225 原始缺陷同一条断链,
|
|
1046
|
+
只是深度 2(N225 v1.0 报告 §0 已点名的缺口)。现按**单槽归一**处理:
|
|
1047
|
+
数值(int/float,含 bool)原样,其余槽记 0——与「缺键取 0」同一口径,
|
|
1048
|
+
不新造第二套默认值。归一化真的发生时(归一结果 ≠ 原值对)进
|
|
1049
|
+
note_bad_sortkey_rows 记账(**容忍 ≠ 静默**,记录不丢)。
|
|
1050
|
+
次序不变性:合法记录的排序键逐位等于旧键 ⇒ 同一稳定排序算法在同一
|
|
1051
|
+
输入序列上产出同一次序,故相对次序与改前逐位一致(守卫 F2 用真实
|
|
1052
|
+
分片记录与旧键 oracle 逐条对照)。
|
|
1053
|
+
"""
|
|
1054
|
+
if not os.path.isdir(directory):
|
|
1055
|
+
return []
|
|
1056
|
+
|
|
1057
|
+
# 生效条件:v 为 int/float(含 bool)且 v == v(非 NaN)时原样返回 v,其余取值(str/list/dict/None/NaN)一律返回 0——排序键单槽归一,NaN 必须排除(`nan == nan` 为假、与任何数比较恒假:它不抛但让次序不确定)。
|
|
1058
|
+
def _norm(v):
|
|
1059
|
+
return v if (isinstance(v, (int, float)) and v == v) else 0
|
|
1060
|
+
|
|
1061
|
+
recs = []
|
|
1062
|
+
for fn in sorted(os.listdir(directory)):
|
|
1063
|
+
if not fn.endswith(".log"):
|
|
1064
|
+
continue
|
|
1065
|
+
p = os.path.join(directory, fn)
|
|
1066
|
+
for _attempt in range(8):
|
|
1067
|
+
try:
|
|
1068
|
+
batch, skips, badkeys = [], [], []
|
|
1069
|
+
for lineno, rec in enumerate(read_jsonl(p), 1):
|
|
1070
|
+
if not isinstance(rec, dict):
|
|
1071
|
+
skips.append((p, lineno, rec))
|
|
1072
|
+
continue
|
|
1073
|
+
batch.append(rec)
|
|
1074
|
+
raw = (rec.get("_t", 0), rec.get("_s", 0))
|
|
1075
|
+
if (_norm(raw[0]), _norm(raw[1])) != raw:
|
|
1076
|
+
badkeys.append((p, lineno, raw))
|
|
1077
|
+
recs.extend(batch)
|
|
1078
|
+
note_nonobject_rows(skips)
|
|
1079
|
+
note_bad_sortkey_rows(badkeys)
|
|
1080
|
+
break
|
|
1081
|
+
except FileNotFoundError:
|
|
1082
|
+
break # 已被 compact 清走(记录已并入快照)
|
|
1083
|
+
except PermissionError:
|
|
1084
|
+
if _attempt == 7:
|
|
1085
|
+
raise
|
|
1086
|
+
time.sleep(0.005)
|
|
1087
|
+
recs.sort(key=lambda r: (_norm(r.get("_t", 0)), _norm(r.get("_s", 0))))
|
|
1088
|
+
return recs
|
|
1089
|
+
|
|
1090
|
+
@staticmethod
|
|
1091
|
+
# 生效条件:directory 是目录时,遍历 os.listdir(directory),对以 ".log" 结尾且不满足「keep 为真值且 os.path.abspath(p) == keep」的条目调用 os.remove(keep 为 None/空串等假值时该排除条件恒不成立,所有 ".log" 条目都会被删),删除时的 OSError 被忽略;directory 不是目录时直接返回。
|
|
1092
|
+
def clear(directory: str, keep: str = None):
|
|
1093
|
+
"""合并进快照后清理分片。keep 用于保留当前进程正在写的那个。"""
|
|
1094
|
+
if not os.path.isdir(directory):
|
|
1095
|
+
return
|
|
1096
|
+
for fn in os.listdir(directory):
|
|
1097
|
+
p = os.path.join(directory, fn)
|
|
1098
|
+
if not fn.endswith(".log") or (keep and os.path.abspath(p) == keep):
|
|
1099
|
+
continue
|
|
1100
|
+
try:
|
|
1101
|
+
os.remove(p)
|
|
1102
|
+
except OSError:
|
|
1103
|
+
pass
|
|
1104
|
+
|
|
1105
|
+
#: issue #443:审计/台账记录单条的**总字节上限**(调用方可控字段可无限长 ⇒ 1:1 写入放大)。
|
|
1106
|
+
#: 与 state_events._MAX_FIELD_BYTES(8192/字段)不同:这里的落点是「整条记录」,
|
|
1107
|
+
#: 故按序列化后的总字节判;取 65536 B(单条簿记不该接近一页文本)。
|
|
1108
|
+
_MAX_RECORD_BYTES = 65536
|
|
1109
|
+
|
|
1110
|
+
|
|
1111
|
+
# 生效条件:rec 可 json 序列化时按其 UTF-8 字节数判:<= _MAX_RECORD_BYTES 返回 True;
|
|
1112
|
+
# 超限或序列化失败返回 False(调用方据此**跳过写入**,不抛——本函数的调用点全是
|
|
1113
|
+
# best-effort 审计路径,抛错会被上层 except 吞掉从而变静默)。
|
|
1114
|
+
def bounded_record(rec) -> bool:
|
|
1115
|
+
"""issue #443:单条台账记录的总字节闸(返回可否写入)。"""
|
|
1116
|
+
try:
|
|
1117
|
+
raw = json.dumps(rec, ensure_ascii=False, default=str).encode('utf-8')
|
|
1118
|
+
except Exception: # noqa: BLE001
|
|
1119
|
+
return False
|
|
1120
|
+
return len(raw) <= _MAX_RECORD_BYTES
|