@furongjun1999/dsh-memory 0.9.6 → 0.9.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (233) hide show
  1. package/README.md +150 -662
  2. package/codebuddy/CODEBUDDY.md +333 -204
  3. package/data/policy.json +38 -38
  4. package/docs/README.md +18 -0
  5. package/docs/discipline/harnesses.yaml +11 -3
  6. package/docs/discipline/templates/compact.txt.tmpl +20 -18
  7. package/docs/discipline/templates/full.md.tmpl +63 -61
  8. package/docs/discipline/templates/rules.mdc.tmpl +5 -3
  9. package/docs/discipline/templates/skill.md.tmpl +25 -23
  10. package/docs/discipline/templates/spec.txt.tmpl +2 -2
  11. package/docs/discipline/templates/zcode-user.md.tmpl +1 -1
  12. package/dsh/cordis.yml.example +139 -139
  13. package/lib/hooks.d.ts +19 -0
  14. package/lib/hooks.js +30 -7
  15. package/lib/index.js +18 -3
  16. package/lib/init.js +19 -4
  17. package/lib/lib/hook_audit.js +64 -2
  18. package/lib/lib/prompt_safety.js +14 -3
  19. package/lib/tools.d.ts +7 -0
  20. package/lib/tools.js +22 -1
  21. package/md_cg/_guard_kit.py +121 -121
  22. package/md_cg/anchor.py +114 -114
  23. package/md_cg/audit.py +15 -2
  24. package/md_cg/chain.py +3 -2
  25. package/md_cg/config_registry_guard.py +487 -487
  26. package/md_cg/conformance.py +747 -747
  27. package/md_cg/consolidate.py +1793 -1793
  28. package/md_cg/crypto.py +794 -794
  29. package/md_cg/hyperedge.py +262 -262
  30. package/md_cg/interface_schema.py +146 -146
  31. package/md_cg/lifecycle.py +426 -426
  32. package/md_cg/mdcos.py +12 -8
  33. package/md_cg/predict.py +1103 -1103
  34. package/md_cg/protect.py +654 -654
  35. package/md_cg/provenance.py +606 -606
  36. package/md_cg/queue_health.py +130 -130
  37. package/md_cg/refine.py +613 -613
  38. package/md_cg/retrieve.py +128 -128
  39. package/md_cg/revise.py +140 -140
  40. package/md_cg/rollback.py +657 -657
  41. package/md_cg/rollback_cli.py +124 -124
  42. package/md_cg/rotate.py +346 -346
  43. package/md_cg/routing.py +490 -490
  44. package/md_cg/self_surface.py +50 -50
  45. package/md_cg/semantic/canonical.py +144 -144
  46. package/md_cg/semantic/zh_en_atoms.py +141 -141
  47. package/md_cg/sources.py +1091 -1091
  48. package/md_cg/stg.py +3 -2
  49. package/md_cg/sustain.py +10 -1
  50. package/md_cg/test_audit_rotate_parity.py +279 -279
  51. package/md_cg/test_autonomy_modes.py +2013 -2013
  52. package/md_cg/test_cn_recall_grams.py +206 -206
  53. package/md_cg/test_code_kind_forbidden.py +101 -0
  54. package/md_cg/test_condaccept.py +156 -156
  55. package/md_cg/test_condgate.py +141 -141
  56. package/md_cg/test_heartbeat_ledger.py +9 -3
  57. package/md_cg/test_i50e_readside_protection.py +658 -658
  58. package/md_cg/test_issue111_gate_order.py +76 -76
  59. package/md_cg/test_issue111_scoring_contract.py +86 -86
  60. package/md_cg/test_issue121_calibration_bounds.py +293 -293
  61. package/md_cg/test_issue145_sleep_no_silent_rollback.py +167 -167
  62. package/md_cg/test_issue147_pack_id_escape.py +378 -378
  63. package/md_cg/test_issue148_merge_downgrade_leak.py +192 -192
  64. package/md_cg/test_issue149_accel_default_face.py +248 -248
  65. package/md_cg/test_issue150_empty_query_terms.py +328 -328
  66. package/md_cg/test_issue151_153_157_write_cache_index.py +151 -151
  67. package/md_cg/test_issue152_sealed_postings.py +309 -309
  68. package/md_cg/test_issue153b_index_generation_invalidation.py +142 -142
  69. package/md_cg/test_issue157b_index_remainder.py +344 -344
  70. package/md_cg/test_issue189_conclusion_bidirectional.py +223 -223
  71. package/md_cg/test_issue191_import_trust.py +382 -382
  72. package/md_cg/test_issue193_write_scope.py +378 -378
  73. package/md_cg/test_issue212_writeface_gaps.py +349 -349
  74. package/md_cg/test_issue214_reject_sensitivity.py +266 -266
  75. package/md_cg/test_issue215_condition_source_gap.py +272 -272
  76. package/md_cg/test_issue216_clash_routes.py +214 -214
  77. package/md_cg/test_issue217_slot_routes.py +248 -248
  78. package/md_cg/test_issue218_postcondition_value_blind.py +222 -222
  79. package/md_cg/test_issue258_pack_sensitivity.py +468 -468
  80. package/md_cg/test_issue265_coldverify_multiproc.py +438 -438
  81. package/md_cg/test_issue274_read_node_id_none.py +341 -341
  82. package/md_cg/test_issue275_layer_param.py +250 -250
  83. package/md_cg/test_issue277_node_id_traversal.py +384 -384
  84. package/md_cg/test_issue278_neg_separated_tension.py +271 -271
  85. package/md_cg/test_issue278_neg_window.py +294 -294
  86. package/md_cg/test_issue278_negative_layering.py +83 -83
  87. package/md_cg/test_issue284_skip_not_pass.py +291 -291
  88. package/md_cg/test_issue294_flywheel_sensitivity.py +573 -573
  89. package/md_cg/test_issue301_baseline_corpus_gate.py +245 -245
  90. package/md_cg/test_issue303_design_boundaries.py +262 -262
  91. package/md_cg/test_issue315_merge_branch_sensitivity.py +383 -383
  92. package/md_cg/test_issue319_tenant_unregistered_bypass.py +256 -256
  93. package/md_cg/test_issue321_mcp_deep_json.py +304 -304
  94. package/md_cg/test_issue322_node_write_lock.py +727 -727
  95. package/md_cg/test_issue324_write_size_fields.py +214 -214
  96. package/md_cg/test_issue325_audit_poison.py +229 -229
  97. package/md_cg/test_issue326_ingest_auto_whitelist.py +331 -331
  98. package/md_cg/test_issue326_key_perms.py +355 -355
  99. package/md_cg/test_issue326_twin_parser_parity.py +323 -323
  100. package/md_cg/test_issue326_vm_call_frame.py +569 -569
  101. package/md_cg/test_issue345_writelimit_effective_sens.py +233 -233
  102. package/md_cg/test_issue346_overwrite_sensitivity_gate.py +424 -424
  103. package/md_cg/test_issue348_insight_verify_op.py +199 -199
  104. package/md_cg/test_issue348_merge_rmw_lock.py +345 -345
  105. package/md_cg/test_issue350_casefold_collision.py +235 -235
  106. package/md_cg/test_issue351_ingest_root_fallback.py +201 -201
  107. package/md_cg/test_issue352_imgaudit.py +418 -418
  108. package/md_cg/test_issue352_pathsjson.py +537 -537
  109. package/md_cg/test_issue352_refindex.py +412 -412
  110. package/md_cg/test_issue352_rollback.py +604 -604
  111. package/md_cg/test_issue354_p2_writelimit_toctou.py +327 -327
  112. package/md_cg/test_issue354_rust_index_containment.py +571 -571
  113. package/md_cg/test_issue354_rust_wal_containment.py +606 -606
  114. package/md_cg/test_issue358_truncation_observable.py +247 -247
  115. package/md_cg/test_issue360_tidy_private_excerpt.py +195 -195
  116. package/md_cg/test_issue361_blindspot_not_silent_drop.py +494 -494
  117. package/md_cg/test_issue362_neg_condition.py +468 -468
  118. package/md_cg/test_issue363_lexical_idf.py +275 -275
  119. package/md_cg/test_issue370_readcache_assembly.py +488 -488
  120. package/md_cg/test_issue376_heartbeat_failed.py +293 -293
  121. package/md_cg/test_issue377_layer_protect_stamp.py +324 -324
  122. package/md_cg/test_issue378_protected_floor.py +310 -310
  123. package/md_cg/test_issue383_state_event_write_gate.py +462 -462
  124. package/md_cg/test_issue397_sens_eff_wiring.py +435 -435
  125. package/md_cg/test_issue398_toolface_op_parity.py +511 -511
  126. package/md_cg/test_issue399_hive_json_shape.py +317 -317
  127. package/md_cg/test_issue401_wisdom_codegen.py +295 -295
  128. package/md_cg/test_issue404_heal_receipt.py +389 -389
  129. package/md_cg/test_issue409_compact_index_completeness.py +509 -509
  130. package/md_cg/test_issue410_verify_demote_write.py +366 -366
  131. package/md_cg/test_issue411_scrub_idempotence.py +575 -575
  132. package/md_cg/test_issue412_discipline_producer_parity.py +311 -311
  133. package/md_cg/test_issue413_preimage_failclosed.py +239 -239
  134. package/md_cg/test_issue413_preimage_twin.py +339 -339
  135. package/md_cg/test_issue415_deps_gate_probe.py +357 -357
  136. package/md_cg/test_issue416_entry_key_subset.py +525 -525
  137. package/md_cg/test_issue417_twophase_merge_landed.py +423 -423
  138. package/md_cg/test_issue425_ingest_library_path.py +331 -331
  139. package/md_cg/test_issue426c_enqueue_not_silent.py +346 -346
  140. package/md_cg/test_issue426e_restore_overwrite_live.py +310 -310
  141. package/md_cg/test_issue426f_paths_gbk_failclosed.py +301 -301
  142. package/md_cg/test_issue427_serve_status_shape.py +191 -191
  143. package/md_cg/test_issue431_432_468_protect_single_point.py +322 -322
  144. package/md_cg/test_issue433_outlook_threshold_single_point.py +278 -278
  145. package/md_cg/test_issue434_435_float_gate.py +90 -90
  146. package/md_cg/test_issue439_sibling_sensitivity_inherit.py +448 -448
  147. package/md_cg/test_issue43_default_policy.py +874 -874
  148. package/md_cg/test_issue444_scrub_evidence_plaintext.py +281 -281
  149. package/md_cg/test_issue445_metacognition_ledger_gate.py +454 -454
  150. package/md_cg/test_issue446_453_403_verified_evidence.py +382 -382
  151. package/md_cg/test_issue450_sensitivity_fallback_reason.py +51 -51
  152. package/md_cg/test_issue450_writegate_unknown_sens.py +203 -203
  153. package/md_cg/test_issue451_audit_actor.py +313 -313
  154. package/md_cg/test_issue452_ingest_root_sensitivity.py +299 -299
  155. package/md_cg/test_issue463_ghost_prune_guard.py +259 -259
  156. package/md_cg/test_issue466_basis_registry.py +469 -469
  157. package/md_cg/test_issue467_identity_sensitivity_parity.py +360 -360
  158. package/md_cg/test_issue469_scrub_override_passthrough.py +126 -126
  159. package/md_cg/test_issue470_wal_event_durable.py +427 -427
  160. package/md_cg/test_issue471_identity_readable.py +136 -136
  161. package/md_cg/test_issue472_audit_chain.py +90 -90
  162. package/md_cg/test_issue472_prune_ledger.py +105 -105
  163. package/md_cg/test_issue472_rollback_degraded.py +81 -81
  164. package/md_cg/test_issue491_compose_sandbox.py +460 -460
  165. package/md_cg/test_issue498_merge_trace_stale_index.py +280 -280
  166. package/md_cg/test_issue511_scoreboard_bad_files.py +256 -256
  167. package/md_cg/test_issue521_rotate_lock_suffix.py +110 -110
  168. package/md_cg/test_issue524_orphan_noise_no_demote.py +88 -88
  169. package/md_cg/test_issue525_read_null_semantics.py +52 -52
  170. package/md_cg/test_issue547_append_jsonl_whitelist.py +114 -114
  171. package/md_cg/test_issue547_audit_durable.py +96 -96
  172. package/md_cg/test_issue557_r2_write_entries_size_gate.py +83 -83
  173. package/md_cg/test_issue559_refine_prefix.py +69 -69
  174. package/md_cg/test_issue562_retired_faces.py +150 -0
  175. package/md_cg/test_issue71_queue_backlog.py +376 -376
  176. package/md_cg/test_issue72_coldverify_controlled_drain.py +420 -420
  177. package/md_cg/test_issue82_credential_rules.py +271 -271
  178. package/md_cg/test_issue91_verifier_sandbox.py +314 -314
  179. package/md_cg/test_issue93_remember_redaction.py +282 -282
  180. package/md_cg/test_logref.py +1141 -1141
  181. package/md_cg/test_md_access_parity.py +330 -330
  182. package/md_cg/test_md_cg_no_hive_dep.py +139 -139
  183. package/md_cg/test_mutation_residue.py +119 -119
  184. package/md_cg/test_mutation_rollback.py +835 -835
  185. package/md_cg/test_n123_derive_expiry_chain.py +210 -210
  186. package/md_cg/test_n184_keys_concurrent_provision.py +1 -1
  187. package/md_cg/test_n209_verify_write_face_gates.py +466 -466
  188. package/md_cg/test_n62_tenant_bind_failclosed.py +200 -200
  189. package/md_cg/test_neg_tail_honesty.py +733 -733
  190. package/md_cg/test_p21_tokens.py +294 -294
  191. package/md_cg/test_p27_docindex.py +792 -792
  192. package/md_cg/test_p28_refcheck.py +1053 -1053
  193. package/md_cg/test_p2_six_elements.py +476 -476
  194. package/md_cg/test_p33_ccg_wiring.py +300 -300
  195. package/md_cg/test_p3_legacy_closure.py +438 -438
  196. package/md_cg/test_p40_refine_worklist.py +244 -244
  197. package/md_cg/test_p4_freshness.py +781 -781
  198. package/md_cg/test_p9c_dedup_hints.py +288 -288
  199. package/md_cg/test_policy_required_ccg.py +444 -444
  200. package/md_cg/test_read_scope_b27.py +2 -2
  201. package/md_cg/test_retr_s5.py +312 -312
  202. package/md_cg/test_s1_ast_judge.py +109 -109
  203. package/md_cg/test_security_audit_v21.py +249 -249
  204. package/md_cg/test_state_event_op.py +521 -521
  205. package/md_cg/test_stgidx_index_parity.py +1024 -1024
  206. package/md_cg/test_sustain_bad_stamp.py +178 -0
  207. package/md_cg/test_sustain_bounded.py +652 -652
  208. package/md_cg/test_trust_edge_reachability.py +405 -405
  209. package/md_cg/test_w7_redlines.py +659 -659
  210. package/md_cg/test_write_return_shape.py +919 -919
  211. package/md_cg/test_writepipe.py +228 -228
  212. package/md_cg/trust.py +1114 -1114
  213. package/md_cg/twophase.py +303 -303
  214. package/md_cg/whitebox.py +536 -536
  215. package/md_cg/whitebox_kb/wisdom/compose_engine.py +1634 -1634
  216. package/md_cg/whitebox_kb/wisdom/core/condition_vm.py +318 -318
  217. package/md_cg/whitebox_kb/wisdom/core/parser.py +1082 -1082
  218. package/md_cg/whitebox_kb/wisdom/md_access.py +490 -490
  219. package/md_cg/whitebox_kb/wisdom/migrate_wisdom.py +497 -497
  220. package/md_cg/whitebox_kb/wisdom/mini_python.py +995 -995
  221. package/md_cg/whitebox_kb/wisdom/net_units.py +2900 -2900
  222. package/md_cg/whitebox_kb/wisdom/semantic_translate.py +4060 -4060
  223. package/md_cg/writepipe.py +1337 -1337
  224. package/package.json +3 -2
  225. package/scripts/workspace_index.py +328 -0
  226. package/src/hooks.ts +932 -908
  227. package/src/index.ts +21 -6
  228. package/src/init.ts +16 -4
  229. package/src/lib/hook_audit.ts +48 -2
  230. package/src/lib/mdcg_client.ts +469 -469
  231. package/src/lib/prompt_safety.ts +17 -3
  232. package/src/tools.ts +23 -1
  233. package/zcode/AGENTS.md +333 -204
package/md_cg/rotate.py CHANGED
@@ -1,347 +1,347 @@
1
- # -*- coding: utf-8 -*-
2
- """分片日志轮转·参数化共享件(有界性的来源)。
3
-
4
- 背景(第 4 条取证,2026-09-16):审计日志无上界增长——单文件形态让任何「读它」
5
- 的操作随运行时长线性劣化。上一轮把体检读数降为 O(1) 元数据口径(`_log_scale`),
6
- 只让「指标与代价错配」不再显形;**有界性的来源是轮转**:单文件 ≤ rotate_bytes、
7
- 分片数 ≤ keep_shards ⇒ 单片读取代价与总量都有上界。
8
-
9
- 本模块是**审计面**轮转的参数化共享件(A2,2026-10-10 设计者裁定):`mdcos` 的
10
- 审计轮转(`_audit.jsonl`)与 `crypto.audit`(`_crypto.jsonl`)此前各自有无一份,
11
- 现统一改调 `Rotator`。参数化面:目录 + 基名 + 归档目录 + 索引名 + 阈值 + 保留数 +
12
- 探测节流 + 计数口径(`counter` / `scale_reader`)+ 自述记录形状(`mark_factory`)
13
- + 时钟(`clock`,取证/守卫用)。
14
-
15
- **范围订正(2026-10-10,DSH 端独立复核;原注释「唯一实现点」不实)**:本模块覆盖
16
- **审计(`_audit.jsonl`)与密文(`_crypto.jsonl`)两路**;**心跳台账另有一份独立轮转**
17
- ——`md_cg/sustain.py::rotate_heartbeat`(分片名 `_heartbeat.%06d.jsonl`,自带
18
- `_hb_next_seq` / `_hb_load_index` / `_hb_prune_shards`),它**不**经 `Rotator`。
19
- 本注释此前写「轮转机制唯一实现点」与该事实不符,现订正为「审计/密文两路共用此实现;
20
- 心跳另有一份独立轮转」。守卫 `test_audit_rotate_parity` 的 ③d 已能扫出「他处仍有
21
- 独立实现」并点名到文件,防止再新增第三份而不被发现。
22
-
23
- 设计边界(诚实面)
24
- ------------------
25
- · 分片内容与轮转前**逐行一致**(`publish` = rename,不动一个字节);
26
- · 并发由 `FileLock` + 「rename 前复检大小 / 失败即返回 None」兜住:抢输者不重复
27
- 切分,也不丢记录;
28
- · 保留策略只淘汰**分片**,淘汰名单写进自述记录(不静默丢证据);
29
- · 超大历史分片在轮转/体检路径上**只读量级**(`scale_reader`),不付 O(n) 全量解析
30
- ——「一次调用堵死通道」不在轮转路径上复现。
31
- """
32
- from __future__ import annotations
33
- import sys
34
- import io
35
-
36
- import json
37
- import os
38
- import time
39
-
40
- from .fsutil import (FileLock, append_jsonl, atomic_write, count_jsonl,
41
- publish, read_jsonl)
42
-
43
-
44
- # 生效条件:path 为日志路径、count_max_bytes 为规模阈值、est_sample 为尾部采样字节数、counter 为行数计数器(缺省 count_jsonl)时,先 os.stat;OSError 返回 {'bytes':0,'mtime':None,'events':0,'exact':True};size 不超过 count_max_bytes 时返回 counter(path) 的精确条数与 exact=True;否则读末尾 est_sample 字节估算 events、exact=False 并附 note(不触发任何全量计数)。
45
- def log_scale(path, count_max_bytes=64 << 20, est_sample=256 << 10,
46
- counter=None):
47
- """日志量级读数(O(1)):以元数据为主,条数只在与规模相称时才精确。"""
48
- counter = counter or count_jsonl
49
- try:
50
- st = os.stat(path)
51
- except OSError:
52
- return {"bytes": 0, "mtime": None, "events": 0, "exact": True}
53
- size = st.st_size
54
- out = {"bytes": size, "mtime": st.st_mtime}
55
- if size <= count_max_bytes:
56
- out.update({"events": counter(path), "exact": True})
57
- return out
58
- est = None
59
- try:
60
- with open(path, "rb") as f:
61
- f.seek(max(0, size - est_sample))
62
- tail = f.read()
63
- lines = [ln for ln in tail.split(b"\n") if ln.strip()]
64
- if lines:
65
- est = int(size / (len(tail) / float(len(lines))))
66
- except OSError:
67
- pass
68
- out.update({
69
- "events": est, "exact": False,
70
- "note": ("超过 %.0f MB 不给全量精确计数(O(n) 磁盘 IO 与量级体检不匹配);"
71
- "events 为尾部 %d KB 采样的估算值"
72
- % (count_max_bytes / 1048576.0, est_sample >> 10))})
73
- return out
74
-
75
-
76
- # 生效条件:注入的 clock 为 None 时用 lambda 延迟解析本模块 time(守卫冻结时钟可 patch);注入时原样使用。
77
- def _default_clock():
78
- return lambda: time.time()
79
-
80
-
81
- # 生效条件:以 root/basename/archive_name 及可选阈值与回调构造实例;active/archive/lock 路径由 root 与基名拼接,_index 缓存与 writes 计数初值为 None/0;不读盘、不建目录、不抛异常。
82
- PRUNE_LEDGER = "_prune_ledger.jsonl" #: #472:剪枝对账单(**不在 shards() 命名空间内**)
83
-
84
- class Rotator:
85
- """参数化分片轮转器:一次实现,审计面与 crypto 面共用。
86
-
87
- · `basename` —— 活动文件基名(`_audit` / `_crypto`)⇒ 活动文件
88
- `<root>/<basename>.jsonl`、分片 `<archive>/<basename>.<6位序号>.jsonl`。
89
- · `archive_name` —— 归档目录名(`_audit_archive` / `_crypto_archive`)。
90
- · `counter` —— 精确行数计数(默认 `fsutil.count_jsonl`,流式不物化)。
91
- · `scale_reader` —— 超大分片的量级读数(默认本模块 `log_scale`);mdcos
92
- 传自身 `_log_scale` 以保持既有读数口径逐位不变。
93
- · `mark_factory(name, size, events, pruned, reason)` —— 轮转自述记录
94
- (形状随消费方协议;返回 None 则不留痕,调用方自担)。
95
- · `clock` —— 时间源(默认本模块 `time.time`)。
96
- """
97
-
98
- # 生效条件:root、basename、archive_name 为必填;index_name/lock_name 缺省 "_index.json" 与 basename+".rotate.lock";rotate_bytes/keep_shards/probe_every/count_max_bytes 缺省 64<<20/8/32/64<<20;counter/scale_reader/mark_factory/clock 缺省见类文档;随后把路径与回调存入实例,_index=None、writes=0。
99
- def __init__(self, root, basename, archive_name, index_name="_index.json",
100
- lock_name=None, rotate_bytes=64 << 20, keep_shards=8,
101
- probe_every=32, count_max_bytes=64 << 20,
102
- counter=None, scale_reader=None, mark_factory=None,
103
- clock=None):
104
- self.root = root
105
- self.basename = basename
106
- self.active = os.path.join(root, basename + ".jsonl")
107
- self.archive = os.path.join(root, archive_name)
108
- self.index_name = index_name
109
- # issue #521:`FileLock` 自己会补 ".lock"(fsutil.py:124)⇒ 此处**只给不含 .lock 的基名**,
110
- # 否则 `basename.rotate.lock` + `.lock` = `…rotate.lock.lock`(后缀拼接职责重叠)。
111
- # ★ 形态(本条正文自带):「不在清单里 ≠ 不存在;做差集前必须先判『清单是否穷举』」。
112
- self.lock_path = os.path.join(root, lock_name or
113
- (basename + ".rotate"))
114
- self.rotate_bytes = rotate_bytes
115
- self.keep_shards = keep_shards
116
- self.probe_every = probe_every
117
- self.count_max_bytes = count_max_bytes
118
- self._counter = counter
119
- self._scale_reader = scale_reader
120
- self._mark = mark_factory
121
- self._clock = clock if clock is not None else _default_clock()
122
- self._index = None
123
- self.writes = 0
124
-
125
- # 生效条件:无入参,返回 self._clock() 的结果(默认 time.time())。
126
- def _now(self):
127
- return self._clock()
128
-
129
- # 生效条件:archive 可被 os.listdir 列出时返回其中以 "<basename>." 开头且以 ".jsonl" 结尾的名字升序列表;listdir 抛 OSError 时返回 []。
130
- def shards(self):
131
- """归档分片名,序号零填充 ⇒ 字典序 == 时间序。"""
132
- try:
133
- names = os.listdir(self.archive)
134
- except OSError:
135
- return []
136
- prefix = self.basename + "."
137
- return sorted(n for n in names
138
- if n.startswith(prefix) and n.endswith(".jsonl"))
139
-
140
- # 生效条件:无必需形参;按 shards() 的字典序(== 时间序)逐个读归档分片、最后读活动
141
- # 文件,返回记录列表;limit 为真时只取尾部 limit 条;单个文件不可读时按空计。
142
- def read_records(self, limit=0):
143
- """活动文件 + 全部分片的记录(**归档感知**读面,#212 缺口 4)。
144
-
145
- 为什么放在 Rotator 而不是各账本各写一份:shards() 已把「序号零填充 ⇒
146
- 字典序 == 时间序」这条口径钉在这里,读面必须与它同源,否则又会长出
147
- 「写侧按分片序号、读侧按 mtime」的第二口径。
148
- """
149
- recs = []
150
- for name in self.shards():
151
- recs.extend(read_jsonl(os.path.join(self.archive, name)))
152
- recs.extend(read_jsonl(self.active))
153
- if limit:
154
- recs = recs[-int(limit):]
155
- return recs
156
-
157
- # 生效条件:self._index 为 None 时尝试读取 archive 中 index_name 并过滤 value 为 dict,读取失败或数据非 dict 时为空字典,随后缓存并返回;非 None 时不重读直接返回。
158
- def load_index(self) -> dict:
159
- if self._index is None:
160
- idx = {}
161
- try:
162
- with open(os.path.join(self.archive, self.index_name),
163
- "r", encoding="utf-8") as f:
164
- data = json.load(f)
165
- if isinstance(data, dict):
166
- idx = {k: v for k, v in data.items() if isinstance(v, dict)}
167
- except (OSError, ValueError):
168
- idx = {} # 索引缺失/损坏 → 空起步,自愈补数
169
- self._index = idx
170
- return self._index
171
-
172
- # 生效条件:idx 传入时尝试创建 archive 并把 idx 的 JSON(ensure_ascii=False, indent=1, sort_keys=True)原子写入 index_name;OSError 被吞掉;最后把 self._index 设为 idx。
173
- def save_index(self, idx: dict):
174
- try:
175
- os.makedirs(self.archive, exist_ok=True)
176
- atomic_write(os.path.join(self.archive, self.index_name),
177
- json.dumps(idx, ensure_ascii=False, indent=1,
178
- sort_keys=True))
179
- except OSError:
180
- pass
181
- self._index = idx
182
-
183
- # 生效条件:path 与 size 传入时,size 为 None 则先 getsize(OSError 返回 bytes/events 0 exact True);size <= count_max_bytes 时返回 _counter(path) 精确条数 exact True;超过时返回 _scale_reader(path) 的事件数 exact False。
184
- def count_shard(self, path, size: int = None) -> dict:
185
- """分片条数读数:有界分片给精确值,超大历史分片只给量级。"""
186
- if size is None:
187
- try:
188
- size = os.path.getsize(path)
189
- except OSError:
190
- return {"bytes": 0, "events": 0, "exact": True}
191
- if size <= self.count_max_bytes:
192
- return {"bytes": size,
193
- "events": (self._counter or count_jsonl)(path),
194
- "exact": True}
195
- s = self._scale(path)
196
- return {"bytes": size, "events": s["events"], "exact": False}
197
-
198
- # 生效条件:无入参,按 self._scale_reader(缺省本模块 log_scale,count_max_bytes 透传)读 path 的量级,返回其 dict。
199
- def _scale(self, path):
200
- if self._scale_reader is not None:
201
- return self._scale_reader(path)
202
- return log_scale(path, count_max_bytes=self.count_max_bytes,
203
- counter=self._counter)
204
-
205
- # 生效条件:在已用 root 构造的实例上遍历 shards(),对 "<basename>.<n>.jsonl" 中 int(n) 成功者取最大值 top(解析失败 continue、无可解析项 top=0),返回 top+1。
206
- def next_seq(self) -> int:
207
- top = 0
208
- n_from = len(self.basename) + 1
209
- for n in self.shards():
210
- try:
211
- top = max(top, int(n[n_from:-len(".jsonl")]))
212
- except ValueError:
213
- continue
214
- return top + 1
215
-
216
- # 生效条件:keep_shards > 0 且分片数超过该值时,gone=shards[:-keep] 逐个尝试 os.remove、成功则从索引 pop(OSError continue),有 gone 才保存索引并返回 gone;keep<=0 或 gone 为空时返回 []。
217
- def prune(self):
218
- """保留最近 keep_shards 个分片、淘汰更旧的(≤0 表示不淘汰)。"""
219
- keep = self.keep_shards
220
- if keep <= 0:
221
- return []
222
- shards = self.shards()
223
- gone = shards[:-keep] if len(shards) > keep else []
224
- if not gone:
225
- return []
226
- idx = self.load_index()
227
- # issue #472 站点 3:剪前写**独立对账单**(被剪分片名 + 各自行数 + 汇总)。
228
- # **文件名不得落在 self.shards() 命名空间内**(否则会被同一 prune 剪掉 —— 那正是原病)。
229
- _led = os.path.join(self.archive, PRUNE_LEDGER)
230
- try:
231
- _lines = {}
232
- for _n in gone:
233
- try:
234
- _lines[_n] = sum(1 for _l in io.open( os.path.join(self.archive, _n), encoding="utf-8") if _l.strip())
235
- except OSError:
236
- _lines[_n] = None
237
- _rec = {"t": time.time(), "kept": keep, "pruned": list(gone),
238
- "lines": _lines,
239
- "lines_total": sum(v for v in _lines.values() if v)}
240
- with io.open(_led, "a", encoding="utf-8") as _f:
241
- _f.write(json.dumps(_rec, ensure_ascii=False) + chr(10))
242
- except Exception as _e: # #472 站点 3:**对账单失败不静默**(与站点 2 同精神)
243
- sys.stderr.write("[rotate] prune 对账单写入失败(剪枝继续):%s: %s"
244
- % (type(_e).__name__, _e) + chr(10))
245
- for n in gone:
246
- try:
247
- os.remove(os.path.join(self.archive, n))
248
- except OSError:
249
- continue # 删不掉就留着:不假装已淘汰
250
- idx.pop(n, None)
251
- self.save_index(idx)
252
- return gone
253
-
254
- # 生效条件:rotate_bytes>0 且 self.writes 自增后能被 probe_every 整除,且 active 的 getsize 不小于 rotate_bytes 时调用 rotate() 并返回其结果;rotate_bytes<=0、未到探测间隔、getsize 不足或 OSError 时返回 None。
255
- def maybe_rotate(self):
256
- """写前闸门:活动日志达阈值即切分(把 stat 摊到 1/probe_every)。"""
257
- if self.rotate_bytes <= 0:
258
- return None
259
- self.writes = (self.writes or 0) + 1
260
- if self.writes % self.probe_every:
261
- return None
262
- try:
263
- if os.path.getsize(self.active) < self.rotate_bytes:
264
- return None
265
- except OSError:
266
- return None
267
- return self.rotate()
268
-
269
- # 生效条件:reason(默认 "size")传入时,在 FileLock(lock_path) 下若 active 的 size >= rotate_bytes 则 publish 为 "<basename>.<6位序号>.jsonl" 归档并更新索引/剪枝/自述留痕后返回 {"shard","bytes","events","pruned"};size 不足、getsize OSError 或 publish 失败时返回 None;成功后 writes 归 0。
270
- def rotate(self, reason: str = "size"):
271
- """把活动日志切分为归档分片(publish = rename 原子,不重写一个字节)。"""
272
- with FileLock(self.lock_path, timeout=5.0):
273
- try:
274
- size = os.path.getsize(self.active)
275
- except OSError:
276
- return None
277
- if size < self.rotate_bytes:
278
- return None # 已被并发写者轮转
279
- os.makedirs(self.archive, exist_ok=True)
280
- name = "%s.%06d.jsonl" % (self.basename, self.next_seq())
281
- dst = os.path.join(self.archive, name)
282
- try:
283
- publish(self.active, dst) # Windows 短重试:AV 短锁不误判抢输
284
- except OSError:
285
- return None # 抢输(文件已被移走)→ 让位,不报错
286
- scale = self.count_shard(dst, size) # 分档:有界扫描 / 只读量级
287
- events = scale["events"]
288
- idx = self.load_index()
289
- idx[name] = {"bytes": size, "events": events, "exact": scale["exact"],
290
- "t": round(self._now(), 3), "reason": reason}
291
- self.save_index(idx)
292
- pruned = self.prune()
293
- self.writes = 0
294
- try: # 自述留痕:轮转本身可审计
295
- mark = (self._mark(name, size, events, pruned, reason)
296
- if self._mark is not None else None)
297
- if mark is not None:
298
- # #326-④:append_jsonl 不再上抛、以 False 报写失败——本处**仍容忍**(best-effort
299
- # 簿记),但失败不再静默:源头已记 fsutil.APPEND_FAILURES + 有界 stderr 告警
300
- # (B 面单点)。显式消费返回值:不留裸调用(守卫 AST 钉子会抓)。
301
- _ok = append_jsonl(self.active, mark)
302
- except OSError:
303
- pass
304
- return {"shard": name, "bytes": size, "events": events,
305
- "pruned": pruned}
306
-
307
- # 生效条件:以 _scale(active) 取活动读数为 active,对磁盘现有分片 present 中未登记者按 count_shard 采纳进 idx(OSError continue,有采纳才写回索引),返回 active 并附 shards/shard_bytes/shard_events(均只统计 k in present)/oversized(非 exact 分片数)/total_bytes/total_events(active["events"] 为 None 时取 None)/total_exact/rotate_bytes/keep_shards。
308
- def scale(self) -> dict:
309
- """日志面量级读数(O(1) 稳态):活动文件实时读数 + 归档分片索引缓存。"""
310
- active = self._scale(self.active)
311
- present = self.shards()
312
- idx = self.load_index()
313
- healed = False
314
- for n in present:
315
- if n in idx:
316
- continue
317
- p = os.path.join(self.archive, n)
318
- try:
319
- sc = self.count_shard(p) # 分档:有界扫描 / 只读量级
320
- idx[n] = {"bytes": sc["bytes"], "events": sc["events"],
321
- "exact": sc["exact"], "t": round(self._now(), 3),
322
- "reason": "adopted"}
323
- healed = True
324
- except OSError:
325
- continue
326
- if healed:
327
- self.save_index(idx)
328
- shard_bytes = sum(int(v.get("bytes") or 0)
329
- for k, v in idx.items() if k in present)
330
- shard_events = sum(int(v.get("events") or 0)
331
- for k, v in idx.items() if k in present)
332
- oversized = sum(1 for n in present
333
- if not idx.get(n, {}).get("exact", True))
334
- out = dict(active)
335
- out.update({
336
- "shards": len(present),
337
- "shard_bytes": shard_bytes,
338
- "shard_events": shard_events,
339
- "oversized": oversized, # 非精确分片数(历史遗留超大文件)
340
- "total_bytes": active["bytes"] + shard_bytes,
341
- "total_events": (active["events"] + shard_events
342
- if active["events"] is not None else None),
343
- "total_exact": bool(active["exact"]) and oversized == 0,
344
- "rotate_bytes": self.rotate_bytes,
345
- "keep_shards": self.keep_shards,
346
- })
1
+ # -*- coding: utf-8 -*-
2
+ """分片日志轮转·参数化共享件(有界性的来源)。
3
+
4
+ 背景(第 4 条取证,2026-09-16):审计日志无上界增长——单文件形态让任何「读它」
5
+ 的操作随运行时长线性劣化。上一轮把体检读数降为 O(1) 元数据口径(`_log_scale`),
6
+ 只让「指标与代价错配」不再显形;**有界性的来源是轮转**:单文件 ≤ rotate_bytes、
7
+ 分片数 ≤ keep_shards ⇒ 单片读取代价与总量都有上界。
8
+
9
+ 本模块是**审计面**轮转的参数化共享件(A2,2026-10-10 设计者裁定):`mdcos` 的
10
+ 审计轮转(`_audit.jsonl`)与 `crypto.audit`(`_crypto.jsonl`)此前各自有无一份,
11
+ 现统一改调 `Rotator`。参数化面:目录 + 基名 + 归档目录 + 索引名 + 阈值 + 保留数 +
12
+ 探测节流 + 计数口径(`counter` / `scale_reader`)+ 自述记录形状(`mark_factory`)
13
+ + 时钟(`clock`,取证/守卫用)。
14
+
15
+ **范围订正(2026-10-10,DSH 端独立复核;原注释「唯一实现点」不实)**:本模块覆盖
16
+ **审计(`_audit.jsonl`)与密文(`_crypto.jsonl`)两路**;**心跳台账另有一份独立轮转**
17
+ ——`md_cg/sustain.py::rotate_heartbeat`(分片名 `_heartbeat.%06d.jsonl`,自带
18
+ `_hb_next_seq` / `_hb_load_index` / `_hb_prune_shards`),它**不**经 `Rotator`。
19
+ 本注释此前写「轮转机制唯一实现点」与该事实不符,现订正为「审计/密文两路共用此实现;
20
+ 心跳另有一份独立轮转」。守卫 `test_audit_rotate_parity` 的 ③d 已能扫出「他处仍有
21
+ 独立实现」并点名到文件,防止再新增第三份而不被发现。
22
+
23
+ 设计边界(诚实面)
24
+ ------------------
25
+ · 分片内容与轮转前**逐行一致**(`publish` = rename,不动一个字节);
26
+ · 并发由 `FileLock` + 「rename 前复检大小 / 失败即返回 None」兜住:抢输者不重复
27
+ 切分,也不丢记录;
28
+ · 保留策略只淘汰**分片**,淘汰名单写进自述记录(不静默丢证据);
29
+ · 超大历史分片在轮转/体检路径上**只读量级**(`scale_reader`),不付 O(n) 全量解析
30
+ ——「一次调用堵死通道」不在轮转路径上复现。
31
+ """
32
+ from __future__ import annotations
33
+ import sys
34
+ import io
35
+
36
+ import json
37
+ import os
38
+ import time
39
+
40
+ from .fsutil import (FileLock, append_jsonl, atomic_write, count_jsonl,
41
+ publish, read_jsonl)
42
+
43
+
44
+ # 生效条件:path 为日志路径、count_max_bytes 为规模阈值、est_sample 为尾部采样字节数、counter 为行数计数器(缺省 count_jsonl)时,先 os.stat;OSError 返回 {'bytes':0,'mtime':None,'events':0,'exact':True};size 不超过 count_max_bytes 时返回 counter(path) 的精确条数与 exact=True;否则读末尾 est_sample 字节估算 events、exact=False 并附 note(不触发任何全量计数)。
45
+ def log_scale(path, count_max_bytes=64 << 20, est_sample=256 << 10,
46
+ counter=None):
47
+ """日志量级读数(O(1)):以元数据为主,条数只在与规模相称时才精确。"""
48
+ counter = counter or count_jsonl
49
+ try:
50
+ st = os.stat(path)
51
+ except OSError:
52
+ return {"bytes": 0, "mtime": None, "events": 0, "exact": True}
53
+ size = st.st_size
54
+ out = {"bytes": size, "mtime": st.st_mtime}
55
+ if size <= count_max_bytes:
56
+ out.update({"events": counter(path), "exact": True})
57
+ return out
58
+ est = None
59
+ try:
60
+ with open(path, "rb") as f:
61
+ f.seek(max(0, size - est_sample))
62
+ tail = f.read()
63
+ lines = [ln for ln in tail.split(b"\n") if ln.strip()]
64
+ if lines:
65
+ est = int(size / (len(tail) / float(len(lines))))
66
+ except OSError:
67
+ pass
68
+ out.update({
69
+ "events": est, "exact": False,
70
+ "note": ("超过 %.0f MB 不给全量精确计数(O(n) 磁盘 IO 与量级体检不匹配);"
71
+ "events 为尾部 %d KB 采样的估算值"
72
+ % (count_max_bytes / 1048576.0, est_sample >> 10))})
73
+ return out
74
+
75
+
76
+ # 生效条件:注入的 clock 为 None 时用 lambda 延迟解析本模块 time(守卫冻结时钟可 patch);注入时原样使用。
77
+ def _default_clock():
78
+ return lambda: time.time()
79
+
80
+
81
+ # 生效条件:以 root/basename/archive_name 及可选阈值与回调构造实例;active/archive/lock 路径由 root 与基名拼接,_index 缓存与 writes 计数初值为 None/0;不读盘、不建目录、不抛异常。
82
+ PRUNE_LEDGER = "_prune_ledger.jsonl" #: #472:剪枝对账单(**不在 shards() 命名空间内**)
83
+
84
+ class Rotator:
85
+ """参数化分片轮转器:一次实现,审计面与 crypto 面共用。
86
+
87
+ · `basename` —— 活动文件基名(`_audit` / `_crypto`)⇒ 活动文件
88
+ `<root>/<basename>.jsonl`、分片 `<archive>/<basename>.<6位序号>.jsonl`。
89
+ · `archive_name` —— 归档目录名(`_audit_archive` / `_crypto_archive`)。
90
+ · `counter` —— 精确行数计数(默认 `fsutil.count_jsonl`,流式不物化)。
91
+ · `scale_reader` —— 超大分片的量级读数(默认本模块 `log_scale`);mdcos
92
+ 传自身 `_log_scale` 以保持既有读数口径逐位不变。
93
+ · `mark_factory(name, size, events, pruned, reason)` —— 轮转自述记录
94
+ (形状随消费方协议;返回 None 则不留痕,调用方自担)。
95
+ · `clock` —— 时间源(默认本模块 `time.time`)。
96
+ """
97
+
98
+ # 生效条件:root、basename、archive_name 为必填;index_name/lock_name 缺省 "_index.json" 与 basename+".rotate.lock";rotate_bytes/keep_shards/probe_every/count_max_bytes 缺省 64<<20/8/32/64<<20;counter/scale_reader/mark_factory/clock 缺省见类文档;随后把路径与回调存入实例,_index=None、writes=0。
99
+ def __init__(self, root, basename, archive_name, index_name="_index.json",
100
+ lock_name=None, rotate_bytes=64 << 20, keep_shards=8,
101
+ probe_every=32, count_max_bytes=64 << 20,
102
+ counter=None, scale_reader=None, mark_factory=None,
103
+ clock=None):
104
+ self.root = root
105
+ self.basename = basename
106
+ self.active = os.path.join(root, basename + ".jsonl")
107
+ self.archive = os.path.join(root, archive_name)
108
+ self.index_name = index_name
109
+ # issue #521:`FileLock` 自己会补 ".lock"(fsutil.py:124)⇒ 此处**只给不含 .lock 的基名**,
110
+ # 否则 `basename.rotate.lock` + `.lock` = `…rotate.lock.lock`(后缀拼接职责重叠)。
111
+ # ★ 形态(本条正文自带):「不在清单里 ≠ 不存在;做差集前必须先判『清单是否穷举』」。
112
+ self.lock_path = os.path.join(root, lock_name or
113
+ (basename + ".rotate"))
114
+ self.rotate_bytes = rotate_bytes
115
+ self.keep_shards = keep_shards
116
+ self.probe_every = probe_every
117
+ self.count_max_bytes = count_max_bytes
118
+ self._counter = counter
119
+ self._scale_reader = scale_reader
120
+ self._mark = mark_factory
121
+ self._clock = clock if clock is not None else _default_clock()
122
+ self._index = None
123
+ self.writes = 0
124
+
125
+ # 生效条件:无入参,返回 self._clock() 的结果(默认 time.time())。
126
+ def _now(self):
127
+ return self._clock()
128
+
129
+ # 生效条件:archive 可被 os.listdir 列出时返回其中以 "<basename>." 开头且以 ".jsonl" 结尾的名字升序列表;listdir 抛 OSError 时返回 []。
130
+ def shards(self):
131
+ """归档分片名,序号零填充 ⇒ 字典序 == 时间序。"""
132
+ try:
133
+ names = os.listdir(self.archive)
134
+ except OSError:
135
+ return []
136
+ prefix = self.basename + "."
137
+ return sorted(n for n in names
138
+ if n.startswith(prefix) and n.endswith(".jsonl"))
139
+
140
+ # 生效条件:无必需形参;按 shards() 的字典序(== 时间序)逐个读归档分片、最后读活动
141
+ # 文件,返回记录列表;limit 为真时只取尾部 limit 条;单个文件不可读时按空计。
142
+ def read_records(self, limit=0):
143
+ """活动文件 + 全部分片的记录(**归档感知**读面,#212 缺口 4)。
144
+
145
+ 为什么放在 Rotator 而不是各账本各写一份:shards() 已把「序号零填充 ⇒
146
+ 字典序 == 时间序」这条口径钉在这里,读面必须与它同源,否则又会长出
147
+ 「写侧按分片序号、读侧按 mtime」的第二口径。
148
+ """
149
+ recs = []
150
+ for name in self.shards():
151
+ recs.extend(read_jsonl(os.path.join(self.archive, name)))
152
+ recs.extend(read_jsonl(self.active))
153
+ if limit:
154
+ recs = recs[-int(limit):]
155
+ return recs
156
+
157
+ # 生效条件:self._index 为 None 时尝试读取 archive 中 index_name 并过滤 value 为 dict,读取失败或数据非 dict 时为空字典,随后缓存并返回;非 None 时不重读直接返回。
158
+ def load_index(self) -> dict:
159
+ if self._index is None:
160
+ idx = {}
161
+ try:
162
+ with open(os.path.join(self.archive, self.index_name),
163
+ "r", encoding="utf-8") as f:
164
+ data = json.load(f)
165
+ if isinstance(data, dict):
166
+ idx = {k: v for k, v in data.items() if isinstance(v, dict)}
167
+ except (OSError, ValueError):
168
+ idx = {} # 索引缺失/损坏 → 空起步,自愈补数
169
+ self._index = idx
170
+ return self._index
171
+
172
+ # 生效条件:idx 传入时尝试创建 archive 并把 idx 的 JSON(ensure_ascii=False, indent=1, sort_keys=True)原子写入 index_name;OSError 被吞掉;最后把 self._index 设为 idx。
173
+ def save_index(self, idx: dict):
174
+ try:
175
+ os.makedirs(self.archive, exist_ok=True)
176
+ atomic_write(os.path.join(self.archive, self.index_name),
177
+ json.dumps(idx, ensure_ascii=False, indent=1,
178
+ sort_keys=True))
179
+ except OSError:
180
+ pass
181
+ self._index = idx
182
+
183
+ # 生效条件:path 与 size 传入时,size 为 None 则先 getsize(OSError 返回 bytes/events 0 exact True);size <= count_max_bytes 时返回 _counter(path) 精确条数 exact True;超过时返回 _scale_reader(path) 的事件数 exact False。
184
+ def count_shard(self, path, size: int = None) -> dict:
185
+ """分片条数读数:有界分片给精确值,超大历史分片只给量级。"""
186
+ if size is None:
187
+ try:
188
+ size = os.path.getsize(path)
189
+ except OSError:
190
+ return {"bytes": 0, "events": 0, "exact": True}
191
+ if size <= self.count_max_bytes:
192
+ return {"bytes": size,
193
+ "events": (self._counter or count_jsonl)(path),
194
+ "exact": True}
195
+ s = self._scale(path)
196
+ return {"bytes": size, "events": s["events"], "exact": False}
197
+
198
+ # 生效条件:无入参,按 self._scale_reader(缺省本模块 log_scale,count_max_bytes 透传)读 path 的量级,返回其 dict。
199
+ def _scale(self, path):
200
+ if self._scale_reader is not None:
201
+ return self._scale_reader(path)
202
+ return log_scale(path, count_max_bytes=self.count_max_bytes,
203
+ counter=self._counter)
204
+
205
+ # 生效条件:在已用 root 构造的实例上遍历 shards(),对 "<basename>.<n>.jsonl" 中 int(n) 成功者取最大值 top(解析失败 continue、无可解析项 top=0),返回 top+1。
206
+ def next_seq(self) -> int:
207
+ top = 0
208
+ n_from = len(self.basename) + 1
209
+ for n in self.shards():
210
+ try:
211
+ top = max(top, int(n[n_from:-len(".jsonl")]))
212
+ except ValueError:
213
+ continue
214
+ return top + 1
215
+
216
+ # 生效条件:keep_shards > 0 且分片数超过该值时,gone=shards[:-keep] 逐个尝试 os.remove、成功则从索引 pop(OSError continue),有 gone 才保存索引并返回 gone;keep<=0 或 gone 为空时返回 []。
217
+ def prune(self):
218
+ """保留最近 keep_shards 个分片、淘汰更旧的(≤0 表示不淘汰)。"""
219
+ keep = self.keep_shards
220
+ if keep <= 0:
221
+ return []
222
+ shards = self.shards()
223
+ gone = shards[:-keep] if len(shards) > keep else []
224
+ if not gone:
225
+ return []
226
+ idx = self.load_index()
227
+ # issue #472 站点 3:剪前写**独立对账单**(被剪分片名 + 各自行数 + 汇总)。
228
+ # **文件名不得落在 self.shards() 命名空间内**(否则会被同一 prune 剪掉 —— 那正是原病)。
229
+ _led = os.path.join(self.archive, PRUNE_LEDGER)
230
+ try:
231
+ _lines = {}
232
+ for _n in gone:
233
+ try:
234
+ _lines[_n] = sum(1 for _l in io.open( os.path.join(self.archive, _n), encoding="utf-8") if _l.strip())
235
+ except OSError:
236
+ _lines[_n] = None
237
+ _rec = {"t": time.time(), "kept": keep, "pruned": list(gone),
238
+ "lines": _lines,
239
+ "lines_total": sum(v for v in _lines.values() if v)}
240
+ with io.open(_led, "a", encoding="utf-8") as _f:
241
+ _f.write(json.dumps(_rec, ensure_ascii=False) + chr(10))
242
+ except Exception as _e: # #472 站点 3:**对账单失败不静默**(与站点 2 同精神)
243
+ sys.stderr.write("[rotate] prune 对账单写入失败(剪枝继续):%s: %s"
244
+ % (type(_e).__name__, _e) + chr(10))
245
+ for n in gone:
246
+ try:
247
+ os.remove(os.path.join(self.archive, n))
248
+ except OSError:
249
+ continue # 删不掉就留着:不假装已淘汰
250
+ idx.pop(n, None)
251
+ self.save_index(idx)
252
+ return gone
253
+
254
+ # 生效条件:rotate_bytes>0 且 self.writes 自增后能被 probe_every 整除,且 active 的 getsize 不小于 rotate_bytes 时调用 rotate() 并返回其结果;rotate_bytes<=0、未到探测间隔、getsize 不足或 OSError 时返回 None。
255
+ def maybe_rotate(self):
256
+ """写前闸门:活动日志达阈值即切分(把 stat 摊到 1/probe_every)。"""
257
+ if self.rotate_bytes <= 0:
258
+ return None
259
+ self.writes = (self.writes or 0) + 1
260
+ if self.writes % self.probe_every:
261
+ return None
262
+ try:
263
+ if os.path.getsize(self.active) < self.rotate_bytes:
264
+ return None
265
+ except OSError:
266
+ return None
267
+ return self.rotate()
268
+
269
+ # 生效条件:reason(默认 "size")传入时,在 FileLock(lock_path) 下若 active 的 size >= rotate_bytes 则 publish 为 "<basename>.<6位序号>.jsonl" 归档并更新索引/剪枝/自述留痕后返回 {"shard","bytes","events","pruned"};size 不足、getsize OSError 或 publish 失败时返回 None;成功后 writes 归 0。
270
+ def rotate(self, reason: str = "size"):
271
+ """把活动日志切分为归档分片(publish = rename 原子,不重写一个字节)。"""
272
+ with FileLock(self.lock_path, timeout=5.0):
273
+ try:
274
+ size = os.path.getsize(self.active)
275
+ except OSError:
276
+ return None
277
+ if size < self.rotate_bytes:
278
+ return None # 已被并发写者轮转
279
+ os.makedirs(self.archive, exist_ok=True)
280
+ name = "%s.%06d.jsonl" % (self.basename, self.next_seq())
281
+ dst = os.path.join(self.archive, name)
282
+ try:
283
+ publish(self.active, dst) # Windows 短重试:AV 短锁不误判抢输
284
+ except OSError:
285
+ return None # 抢输(文件已被移走)→ 让位,不报错
286
+ scale = self.count_shard(dst, size) # 分档:有界扫描 / 只读量级
287
+ events = scale["events"]
288
+ idx = self.load_index()
289
+ idx[name] = {"bytes": size, "events": events, "exact": scale["exact"],
290
+ "t": round(self._now(), 3), "reason": reason}
291
+ self.save_index(idx)
292
+ pruned = self.prune()
293
+ self.writes = 0
294
+ try: # 自述留痕:轮转本身可审计
295
+ mark = (self._mark(name, size, events, pruned, reason)
296
+ if self._mark is not None else None)
297
+ if mark is not None:
298
+ # #326-④:append_jsonl 不再上抛、以 False 报写失败——本处**仍容忍**(best-effort
299
+ # 簿记),但失败不再静默:源头已记 fsutil.APPEND_FAILURES + 有界 stderr 告警
300
+ # (B 面单点)。显式消费返回值:不留裸调用(守卫 AST 钉子会抓)。
301
+ _ok = append_jsonl(self.active, mark)
302
+ except OSError:
303
+ pass
304
+ return {"shard": name, "bytes": size, "events": events,
305
+ "pruned": pruned}
306
+
307
+ # 生效条件:以 _scale(active) 取活动读数为 active,对磁盘现有分片 present 中未登记者按 count_shard 采纳进 idx(OSError continue,有采纳才写回索引),返回 active 并附 shards/shard_bytes/shard_events(均只统计 k in present)/oversized(非 exact 分片数)/total_bytes/total_events(active["events"] 为 None 时取 None)/total_exact/rotate_bytes/keep_shards。
308
+ def scale(self) -> dict:
309
+ """日志面量级读数(O(1) 稳态):活动文件实时读数 + 归档分片索引缓存。"""
310
+ active = self._scale(self.active)
311
+ present = self.shards()
312
+ idx = self.load_index()
313
+ healed = False
314
+ for n in present:
315
+ if n in idx:
316
+ continue
317
+ p = os.path.join(self.archive, n)
318
+ try:
319
+ sc = self.count_shard(p) # 分档:有界扫描 / 只读量级
320
+ idx[n] = {"bytes": sc["bytes"], "events": sc["events"],
321
+ "exact": sc["exact"], "t": round(self._now(), 3),
322
+ "reason": "adopted"}
323
+ healed = True
324
+ except OSError:
325
+ continue
326
+ if healed:
327
+ self.save_index(idx)
328
+ shard_bytes = sum(int(v.get("bytes") or 0)
329
+ for k, v in idx.items() if k in present)
330
+ shard_events = sum(int(v.get("events") or 0)
331
+ for k, v in idx.items() if k in present)
332
+ oversized = sum(1 for n in present
333
+ if not idx.get(n, {}).get("exact", True))
334
+ out = dict(active)
335
+ out.update({
336
+ "shards": len(present),
337
+ "shard_bytes": shard_bytes,
338
+ "shard_events": shard_events,
339
+ "oversized": oversized, # 非精确分片数(历史遗留超大文件)
340
+ "total_bytes": active["bytes"] + shard_bytes,
341
+ "total_events": (active["events"] + shard_events
342
+ if active["events"] is not None else None),
343
+ "total_exact": bool(active["exact"]) and oversized == 0,
344
+ "rotate_bytes": self.rotate_bytes,
345
+ "keep_shards": self.keep_shards,
346
+ })
347
347
  return out