@furongjun1999/dsh-memory 0.9.6 → 0.9.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +150 -662
- package/codebuddy/CODEBUDDY.md +333 -204
- package/data/policy.json +38 -38
- package/docs/README.md +18 -0
- package/docs/discipline/harnesses.yaml +11 -3
- package/docs/discipline/templates/compact.txt.tmpl +20 -18
- package/docs/discipline/templates/full.md.tmpl +63 -61
- package/docs/discipline/templates/rules.mdc.tmpl +5 -3
- package/docs/discipline/templates/skill.md.tmpl +25 -23
- package/docs/discipline/templates/spec.txt.tmpl +2 -2
- package/docs/discipline/templates/zcode-user.md.tmpl +1 -1
- package/dsh/cordis.yml.example +139 -139
- package/lib/hooks.d.ts +19 -0
- package/lib/hooks.js +30 -7
- package/lib/index.js +18 -3
- package/lib/init.js +19 -4
- package/lib/lib/hook_audit.js +64 -2
- package/lib/lib/prompt_safety.js +14 -3
- package/lib/tools.d.ts +7 -0
- package/lib/tools.js +22 -1
- package/md_cg/_guard_kit.py +121 -121
- package/md_cg/anchor.py +114 -114
- package/md_cg/audit.py +15 -2
- package/md_cg/chain.py +3 -2
- package/md_cg/config_registry_guard.py +487 -487
- package/md_cg/conformance.py +747 -747
- package/md_cg/consolidate.py +1793 -1793
- package/md_cg/crypto.py +794 -794
- package/md_cg/hyperedge.py +262 -262
- package/md_cg/interface_schema.py +146 -146
- package/md_cg/lifecycle.py +426 -426
- package/md_cg/mdcos.py +12 -8
- package/md_cg/predict.py +1103 -1103
- package/md_cg/protect.py +654 -654
- package/md_cg/provenance.py +606 -606
- package/md_cg/queue_health.py +130 -130
- package/md_cg/refine.py +613 -613
- package/md_cg/retrieve.py +128 -128
- package/md_cg/revise.py +140 -140
- package/md_cg/rollback.py +657 -657
- package/md_cg/rollback_cli.py +124 -124
- package/md_cg/rotate.py +346 -346
- package/md_cg/routing.py +490 -490
- package/md_cg/self_surface.py +50 -50
- package/md_cg/semantic/canonical.py +144 -144
- package/md_cg/semantic/zh_en_atoms.py +141 -141
- package/md_cg/sources.py +1091 -1091
- package/md_cg/stg.py +3 -2
- package/md_cg/sustain.py +10 -1
- package/md_cg/test_audit_rotate_parity.py +279 -279
- package/md_cg/test_autonomy_modes.py +2013 -2013
- package/md_cg/test_cn_recall_grams.py +206 -206
- package/md_cg/test_code_kind_forbidden.py +101 -0
- package/md_cg/test_condaccept.py +156 -156
- package/md_cg/test_condgate.py +141 -141
- package/md_cg/test_heartbeat_ledger.py +9 -3
- package/md_cg/test_i50e_readside_protection.py +658 -658
- package/md_cg/test_issue111_gate_order.py +76 -76
- package/md_cg/test_issue111_scoring_contract.py +86 -86
- package/md_cg/test_issue121_calibration_bounds.py +293 -293
- package/md_cg/test_issue145_sleep_no_silent_rollback.py +167 -167
- package/md_cg/test_issue147_pack_id_escape.py +378 -378
- package/md_cg/test_issue148_merge_downgrade_leak.py +192 -192
- package/md_cg/test_issue149_accel_default_face.py +248 -248
- package/md_cg/test_issue150_empty_query_terms.py +328 -328
- package/md_cg/test_issue151_153_157_write_cache_index.py +151 -151
- package/md_cg/test_issue152_sealed_postings.py +309 -309
- package/md_cg/test_issue153b_index_generation_invalidation.py +142 -142
- package/md_cg/test_issue157b_index_remainder.py +344 -344
- package/md_cg/test_issue189_conclusion_bidirectional.py +223 -223
- package/md_cg/test_issue191_import_trust.py +382 -382
- package/md_cg/test_issue193_write_scope.py +378 -378
- package/md_cg/test_issue212_writeface_gaps.py +349 -349
- package/md_cg/test_issue214_reject_sensitivity.py +266 -266
- package/md_cg/test_issue215_condition_source_gap.py +272 -272
- package/md_cg/test_issue216_clash_routes.py +214 -214
- package/md_cg/test_issue217_slot_routes.py +248 -248
- package/md_cg/test_issue218_postcondition_value_blind.py +222 -222
- package/md_cg/test_issue258_pack_sensitivity.py +468 -468
- package/md_cg/test_issue265_coldverify_multiproc.py +438 -438
- package/md_cg/test_issue274_read_node_id_none.py +341 -341
- package/md_cg/test_issue275_layer_param.py +250 -250
- package/md_cg/test_issue277_node_id_traversal.py +384 -384
- package/md_cg/test_issue278_neg_separated_tension.py +271 -271
- package/md_cg/test_issue278_neg_window.py +294 -294
- package/md_cg/test_issue278_negative_layering.py +83 -83
- package/md_cg/test_issue284_skip_not_pass.py +291 -291
- package/md_cg/test_issue294_flywheel_sensitivity.py +573 -573
- package/md_cg/test_issue301_baseline_corpus_gate.py +245 -245
- package/md_cg/test_issue303_design_boundaries.py +262 -262
- package/md_cg/test_issue315_merge_branch_sensitivity.py +383 -383
- package/md_cg/test_issue319_tenant_unregistered_bypass.py +256 -256
- package/md_cg/test_issue321_mcp_deep_json.py +304 -304
- package/md_cg/test_issue322_node_write_lock.py +727 -727
- package/md_cg/test_issue324_write_size_fields.py +214 -214
- package/md_cg/test_issue325_audit_poison.py +229 -229
- package/md_cg/test_issue326_ingest_auto_whitelist.py +331 -331
- package/md_cg/test_issue326_key_perms.py +355 -355
- package/md_cg/test_issue326_twin_parser_parity.py +323 -323
- package/md_cg/test_issue326_vm_call_frame.py +569 -569
- package/md_cg/test_issue345_writelimit_effective_sens.py +233 -233
- package/md_cg/test_issue346_overwrite_sensitivity_gate.py +424 -424
- package/md_cg/test_issue348_insight_verify_op.py +199 -199
- package/md_cg/test_issue348_merge_rmw_lock.py +345 -345
- package/md_cg/test_issue350_casefold_collision.py +235 -235
- package/md_cg/test_issue351_ingest_root_fallback.py +201 -201
- package/md_cg/test_issue352_imgaudit.py +418 -418
- package/md_cg/test_issue352_pathsjson.py +537 -537
- package/md_cg/test_issue352_refindex.py +412 -412
- package/md_cg/test_issue352_rollback.py +604 -604
- package/md_cg/test_issue354_p2_writelimit_toctou.py +327 -327
- package/md_cg/test_issue354_rust_index_containment.py +571 -571
- package/md_cg/test_issue354_rust_wal_containment.py +606 -606
- package/md_cg/test_issue358_truncation_observable.py +247 -247
- package/md_cg/test_issue360_tidy_private_excerpt.py +195 -195
- package/md_cg/test_issue361_blindspot_not_silent_drop.py +494 -494
- package/md_cg/test_issue362_neg_condition.py +468 -468
- package/md_cg/test_issue363_lexical_idf.py +275 -275
- package/md_cg/test_issue370_readcache_assembly.py +488 -488
- package/md_cg/test_issue376_heartbeat_failed.py +293 -293
- package/md_cg/test_issue377_layer_protect_stamp.py +324 -324
- package/md_cg/test_issue378_protected_floor.py +310 -310
- package/md_cg/test_issue383_state_event_write_gate.py +462 -462
- package/md_cg/test_issue397_sens_eff_wiring.py +435 -435
- package/md_cg/test_issue398_toolface_op_parity.py +511 -511
- package/md_cg/test_issue399_hive_json_shape.py +317 -317
- package/md_cg/test_issue401_wisdom_codegen.py +295 -295
- package/md_cg/test_issue404_heal_receipt.py +389 -389
- package/md_cg/test_issue409_compact_index_completeness.py +509 -509
- package/md_cg/test_issue410_verify_demote_write.py +366 -366
- package/md_cg/test_issue411_scrub_idempotence.py +575 -575
- package/md_cg/test_issue412_discipline_producer_parity.py +311 -311
- package/md_cg/test_issue413_preimage_failclosed.py +239 -239
- package/md_cg/test_issue413_preimage_twin.py +339 -339
- package/md_cg/test_issue415_deps_gate_probe.py +357 -357
- package/md_cg/test_issue416_entry_key_subset.py +525 -525
- package/md_cg/test_issue417_twophase_merge_landed.py +423 -423
- package/md_cg/test_issue425_ingest_library_path.py +331 -331
- package/md_cg/test_issue426c_enqueue_not_silent.py +346 -346
- package/md_cg/test_issue426e_restore_overwrite_live.py +310 -310
- package/md_cg/test_issue426f_paths_gbk_failclosed.py +301 -301
- package/md_cg/test_issue427_serve_status_shape.py +191 -191
- package/md_cg/test_issue431_432_468_protect_single_point.py +322 -322
- package/md_cg/test_issue433_outlook_threshold_single_point.py +278 -278
- package/md_cg/test_issue434_435_float_gate.py +90 -90
- package/md_cg/test_issue439_sibling_sensitivity_inherit.py +448 -448
- package/md_cg/test_issue43_default_policy.py +874 -874
- package/md_cg/test_issue444_scrub_evidence_plaintext.py +281 -281
- package/md_cg/test_issue445_metacognition_ledger_gate.py +454 -454
- package/md_cg/test_issue446_453_403_verified_evidence.py +382 -382
- package/md_cg/test_issue450_sensitivity_fallback_reason.py +51 -51
- package/md_cg/test_issue450_writegate_unknown_sens.py +203 -203
- package/md_cg/test_issue451_audit_actor.py +313 -313
- package/md_cg/test_issue452_ingest_root_sensitivity.py +299 -299
- package/md_cg/test_issue463_ghost_prune_guard.py +259 -259
- package/md_cg/test_issue466_basis_registry.py +469 -469
- package/md_cg/test_issue467_identity_sensitivity_parity.py +360 -360
- package/md_cg/test_issue469_scrub_override_passthrough.py +126 -126
- package/md_cg/test_issue470_wal_event_durable.py +427 -427
- package/md_cg/test_issue471_identity_readable.py +136 -136
- package/md_cg/test_issue472_audit_chain.py +90 -90
- package/md_cg/test_issue472_prune_ledger.py +105 -105
- package/md_cg/test_issue472_rollback_degraded.py +81 -81
- package/md_cg/test_issue491_compose_sandbox.py +460 -460
- package/md_cg/test_issue498_merge_trace_stale_index.py +280 -280
- package/md_cg/test_issue511_scoreboard_bad_files.py +256 -256
- package/md_cg/test_issue521_rotate_lock_suffix.py +110 -110
- package/md_cg/test_issue524_orphan_noise_no_demote.py +88 -88
- package/md_cg/test_issue525_read_null_semantics.py +52 -52
- package/md_cg/test_issue547_append_jsonl_whitelist.py +114 -114
- package/md_cg/test_issue547_audit_durable.py +96 -96
- package/md_cg/test_issue557_r2_write_entries_size_gate.py +83 -83
- package/md_cg/test_issue559_refine_prefix.py +69 -69
- package/md_cg/test_issue562_retired_faces.py +150 -0
- package/md_cg/test_issue71_queue_backlog.py +376 -376
- package/md_cg/test_issue72_coldverify_controlled_drain.py +420 -420
- package/md_cg/test_issue82_credential_rules.py +271 -271
- package/md_cg/test_issue91_verifier_sandbox.py +314 -314
- package/md_cg/test_issue93_remember_redaction.py +282 -282
- package/md_cg/test_logref.py +1141 -1141
- package/md_cg/test_md_access_parity.py +330 -330
- package/md_cg/test_md_cg_no_hive_dep.py +139 -139
- package/md_cg/test_mutation_residue.py +119 -119
- package/md_cg/test_mutation_rollback.py +835 -835
- package/md_cg/test_n123_derive_expiry_chain.py +210 -210
- package/md_cg/test_n184_keys_concurrent_provision.py +1 -1
- package/md_cg/test_n209_verify_write_face_gates.py +466 -466
- package/md_cg/test_n62_tenant_bind_failclosed.py +200 -200
- package/md_cg/test_neg_tail_honesty.py +733 -733
- package/md_cg/test_p21_tokens.py +294 -294
- package/md_cg/test_p27_docindex.py +792 -792
- package/md_cg/test_p28_refcheck.py +1053 -1053
- package/md_cg/test_p2_six_elements.py +476 -476
- package/md_cg/test_p33_ccg_wiring.py +300 -300
- package/md_cg/test_p3_legacy_closure.py +438 -438
- package/md_cg/test_p40_refine_worklist.py +244 -244
- package/md_cg/test_p4_freshness.py +781 -781
- package/md_cg/test_p9c_dedup_hints.py +288 -288
- package/md_cg/test_policy_required_ccg.py +444 -444
- package/md_cg/test_read_scope_b27.py +2 -2
- package/md_cg/test_retr_s5.py +312 -312
- package/md_cg/test_s1_ast_judge.py +109 -109
- package/md_cg/test_security_audit_v21.py +249 -249
- package/md_cg/test_state_event_op.py +521 -521
- package/md_cg/test_stgidx_index_parity.py +1024 -1024
- package/md_cg/test_sustain_bad_stamp.py +178 -0
- package/md_cg/test_sustain_bounded.py +652 -652
- package/md_cg/test_trust_edge_reachability.py +405 -405
- package/md_cg/test_w7_redlines.py +659 -659
- package/md_cg/test_write_return_shape.py +919 -919
- package/md_cg/test_writepipe.py +228 -228
- package/md_cg/trust.py +1114 -1114
- package/md_cg/twophase.py +303 -303
- package/md_cg/whitebox.py +536 -536
- package/md_cg/whitebox_kb/wisdom/compose_engine.py +1634 -1634
- package/md_cg/whitebox_kb/wisdom/core/condition_vm.py +318 -318
- package/md_cg/whitebox_kb/wisdom/core/parser.py +1082 -1082
- package/md_cg/whitebox_kb/wisdom/md_access.py +490 -490
- package/md_cg/whitebox_kb/wisdom/migrate_wisdom.py +497 -497
- package/md_cg/whitebox_kb/wisdom/mini_python.py +995 -995
- package/md_cg/whitebox_kb/wisdom/net_units.py +2900 -2900
- package/md_cg/whitebox_kb/wisdom/semantic_translate.py +4060 -4060
- package/md_cg/writepipe.py +1337 -1337
- package/package.json +3 -2
- package/scripts/workspace_index.py +328 -0
- package/src/hooks.ts +932 -908
- package/src/index.ts +21 -6
- package/src/init.ts +16 -4
- package/src/lib/hook_audit.ts +48 -2
- package/src/lib/mdcg_client.ts +469 -469
- package/src/lib/prompt_safety.ts +17 -3
- package/src/tools.ts +23 -1
- package/zcode/AGENTS.md +333 -204
package/md_cg/rotate.py
CHANGED
|
@@ -1,347 +1,347 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
|
-
"""分片日志轮转·参数化共享件(有界性的来源)。
|
|
3
|
-
|
|
4
|
-
背景(第 4 条取证,2026-09-16):审计日志无上界增长——单文件形态让任何「读它」
|
|
5
|
-
的操作随运行时长线性劣化。上一轮把体检读数降为 O(1) 元数据口径(`_log_scale`),
|
|
6
|
-
只让「指标与代价错配」不再显形;**有界性的来源是轮转**:单文件 ≤ rotate_bytes、
|
|
7
|
-
分片数 ≤ keep_shards ⇒ 单片读取代价与总量都有上界。
|
|
8
|
-
|
|
9
|
-
本模块是**审计面**轮转的参数化共享件(A2,2026-10-10 设计者裁定):`mdcos` 的
|
|
10
|
-
审计轮转(`_audit.jsonl`)与 `crypto.audit`(`_crypto.jsonl`)此前各自有无一份,
|
|
11
|
-
现统一改调 `Rotator`。参数化面:目录 + 基名 + 归档目录 + 索引名 + 阈值 + 保留数 +
|
|
12
|
-
探测节流 + 计数口径(`counter` / `scale_reader`)+ 自述记录形状(`mark_factory`)
|
|
13
|
-
+ 时钟(`clock`,取证/守卫用)。
|
|
14
|
-
|
|
15
|
-
**范围订正(2026-10-10,DSH 端独立复核;原注释「唯一实现点」不实)**:本模块覆盖
|
|
16
|
-
**审计(`_audit.jsonl`)与密文(`_crypto.jsonl`)两路**;**心跳台账另有一份独立轮转**
|
|
17
|
-
——`md_cg/sustain.py::rotate_heartbeat`(分片名 `_heartbeat.%06d.jsonl`,自带
|
|
18
|
-
`_hb_next_seq` / `_hb_load_index` / `_hb_prune_shards`),它**不**经 `Rotator`。
|
|
19
|
-
本注释此前写「轮转机制唯一实现点」与该事实不符,现订正为「审计/密文两路共用此实现;
|
|
20
|
-
心跳另有一份独立轮转」。守卫 `test_audit_rotate_parity` 的 ③d 已能扫出「他处仍有
|
|
21
|
-
独立实现」并点名到文件,防止再新增第三份而不被发现。
|
|
22
|
-
|
|
23
|
-
设计边界(诚实面)
|
|
24
|
-
------------------
|
|
25
|
-
· 分片内容与轮转前**逐行一致**(`publish` = rename,不动一个字节);
|
|
26
|
-
· 并发由 `FileLock` + 「rename 前复检大小 / 失败即返回 None」兜住:抢输者不重复
|
|
27
|
-
切分,也不丢记录;
|
|
28
|
-
· 保留策略只淘汰**分片**,淘汰名单写进自述记录(不静默丢证据);
|
|
29
|
-
· 超大历史分片在轮转/体检路径上**只读量级**(`scale_reader`),不付 O(n) 全量解析
|
|
30
|
-
——「一次调用堵死通道」不在轮转路径上复现。
|
|
31
|
-
"""
|
|
32
|
-
from __future__ import annotations
|
|
33
|
-
import sys
|
|
34
|
-
import io
|
|
35
|
-
|
|
36
|
-
import json
|
|
37
|
-
import os
|
|
38
|
-
import time
|
|
39
|
-
|
|
40
|
-
from .fsutil import (FileLock, append_jsonl, atomic_write, count_jsonl,
|
|
41
|
-
publish, read_jsonl)
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
# 生效条件:path 为日志路径、count_max_bytes 为规模阈值、est_sample 为尾部采样字节数、counter 为行数计数器(缺省 count_jsonl)时,先 os.stat;OSError 返回 {'bytes':0,'mtime':None,'events':0,'exact':True};size 不超过 count_max_bytes 时返回 counter(path) 的精确条数与 exact=True;否则读末尾 est_sample 字节估算 events、exact=False 并附 note(不触发任何全量计数)。
|
|
45
|
-
def log_scale(path, count_max_bytes=64 << 20, est_sample=256 << 10,
|
|
46
|
-
counter=None):
|
|
47
|
-
"""日志量级读数(O(1)):以元数据为主,条数只在与规模相称时才精确。"""
|
|
48
|
-
counter = counter or count_jsonl
|
|
49
|
-
try:
|
|
50
|
-
st = os.stat(path)
|
|
51
|
-
except OSError:
|
|
52
|
-
return {"bytes": 0, "mtime": None, "events": 0, "exact": True}
|
|
53
|
-
size = st.st_size
|
|
54
|
-
out = {"bytes": size, "mtime": st.st_mtime}
|
|
55
|
-
if size <= count_max_bytes:
|
|
56
|
-
out.update({"events": counter(path), "exact": True})
|
|
57
|
-
return out
|
|
58
|
-
est = None
|
|
59
|
-
try:
|
|
60
|
-
with open(path, "rb") as f:
|
|
61
|
-
f.seek(max(0, size - est_sample))
|
|
62
|
-
tail = f.read()
|
|
63
|
-
lines = [ln for ln in tail.split(b"\n") if ln.strip()]
|
|
64
|
-
if lines:
|
|
65
|
-
est = int(size / (len(tail) / float(len(lines))))
|
|
66
|
-
except OSError:
|
|
67
|
-
pass
|
|
68
|
-
out.update({
|
|
69
|
-
"events": est, "exact": False,
|
|
70
|
-
"note": ("超过 %.0f MB 不给全量精确计数(O(n) 磁盘 IO 与量级体检不匹配);"
|
|
71
|
-
"events 为尾部 %d KB 采样的估算值"
|
|
72
|
-
% (count_max_bytes / 1048576.0, est_sample >> 10))})
|
|
73
|
-
return out
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
# 生效条件:注入的 clock 为 None 时用 lambda 延迟解析本模块 time(守卫冻结时钟可 patch);注入时原样使用。
|
|
77
|
-
def _default_clock():
|
|
78
|
-
return lambda: time.time()
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
# 生效条件:以 root/basename/archive_name 及可选阈值与回调构造实例;active/archive/lock 路径由 root 与基名拼接,_index 缓存与 writes 计数初值为 None/0;不读盘、不建目录、不抛异常。
|
|
82
|
-
PRUNE_LEDGER = "_prune_ledger.jsonl" #: #472:剪枝对账单(**不在 shards() 命名空间内**)
|
|
83
|
-
|
|
84
|
-
class Rotator:
|
|
85
|
-
"""参数化分片轮转器:一次实现,审计面与 crypto 面共用。
|
|
86
|
-
|
|
87
|
-
· `basename` —— 活动文件基名(`_audit` / `_crypto`)⇒ 活动文件
|
|
88
|
-
`<root>/<basename>.jsonl`、分片 `<archive>/<basename>.<6位序号>.jsonl`。
|
|
89
|
-
· `archive_name` —— 归档目录名(`_audit_archive` / `_crypto_archive`)。
|
|
90
|
-
· `counter` —— 精确行数计数(默认 `fsutil.count_jsonl`,流式不物化)。
|
|
91
|
-
· `scale_reader` —— 超大分片的量级读数(默认本模块 `log_scale`);mdcos
|
|
92
|
-
传自身 `_log_scale` 以保持既有读数口径逐位不变。
|
|
93
|
-
· `mark_factory(name, size, events, pruned, reason)` —— 轮转自述记录
|
|
94
|
-
(形状随消费方协议;返回 None 则不留痕,调用方自担)。
|
|
95
|
-
· `clock` —— 时间源(默认本模块 `time.time`)。
|
|
96
|
-
"""
|
|
97
|
-
|
|
98
|
-
# 生效条件:root、basename、archive_name 为必填;index_name/lock_name 缺省 "_index.json" 与 basename+".rotate.lock";rotate_bytes/keep_shards/probe_every/count_max_bytes 缺省 64<<20/8/32/64<<20;counter/scale_reader/mark_factory/clock 缺省见类文档;随后把路径与回调存入实例,_index=None、writes=0。
|
|
99
|
-
def __init__(self, root, basename, archive_name, index_name="_index.json",
|
|
100
|
-
lock_name=None, rotate_bytes=64 << 20, keep_shards=8,
|
|
101
|
-
probe_every=32, count_max_bytes=64 << 20,
|
|
102
|
-
counter=None, scale_reader=None, mark_factory=None,
|
|
103
|
-
clock=None):
|
|
104
|
-
self.root = root
|
|
105
|
-
self.basename = basename
|
|
106
|
-
self.active = os.path.join(root, basename + ".jsonl")
|
|
107
|
-
self.archive = os.path.join(root, archive_name)
|
|
108
|
-
self.index_name = index_name
|
|
109
|
-
# issue #521:`FileLock` 自己会补 ".lock"(fsutil.py:124)⇒ 此处**只给不含 .lock 的基名**,
|
|
110
|
-
# 否则 `basename.rotate.lock` + `.lock` = `…rotate.lock.lock`(后缀拼接职责重叠)。
|
|
111
|
-
# ★ 形态(本条正文自带):「不在清单里 ≠ 不存在;做差集前必须先判『清单是否穷举』」。
|
|
112
|
-
self.lock_path = os.path.join(root, lock_name or
|
|
113
|
-
(basename + ".rotate"))
|
|
114
|
-
self.rotate_bytes = rotate_bytes
|
|
115
|
-
self.keep_shards = keep_shards
|
|
116
|
-
self.probe_every = probe_every
|
|
117
|
-
self.count_max_bytes = count_max_bytes
|
|
118
|
-
self._counter = counter
|
|
119
|
-
self._scale_reader = scale_reader
|
|
120
|
-
self._mark = mark_factory
|
|
121
|
-
self._clock = clock if clock is not None else _default_clock()
|
|
122
|
-
self._index = None
|
|
123
|
-
self.writes = 0
|
|
124
|
-
|
|
125
|
-
# 生效条件:无入参,返回 self._clock() 的结果(默认 time.time())。
|
|
126
|
-
def _now(self):
|
|
127
|
-
return self._clock()
|
|
128
|
-
|
|
129
|
-
# 生效条件:archive 可被 os.listdir 列出时返回其中以 "<basename>." 开头且以 ".jsonl" 结尾的名字升序列表;listdir 抛 OSError 时返回 []。
|
|
130
|
-
def shards(self):
|
|
131
|
-
"""归档分片名,序号零填充 ⇒ 字典序 == 时间序。"""
|
|
132
|
-
try:
|
|
133
|
-
names = os.listdir(self.archive)
|
|
134
|
-
except OSError:
|
|
135
|
-
return []
|
|
136
|
-
prefix = self.basename + "."
|
|
137
|
-
return sorted(n for n in names
|
|
138
|
-
if n.startswith(prefix) and n.endswith(".jsonl"))
|
|
139
|
-
|
|
140
|
-
# 生效条件:无必需形参;按 shards() 的字典序(== 时间序)逐个读归档分片、最后读活动
|
|
141
|
-
# 文件,返回记录列表;limit 为真时只取尾部 limit 条;单个文件不可读时按空计。
|
|
142
|
-
def read_records(self, limit=0):
|
|
143
|
-
"""活动文件 + 全部分片的记录(**归档感知**读面,#212 缺口 4)。
|
|
144
|
-
|
|
145
|
-
为什么放在 Rotator 而不是各账本各写一份:shards() 已把「序号零填充 ⇒
|
|
146
|
-
字典序 == 时间序」这条口径钉在这里,读面必须与它同源,否则又会长出
|
|
147
|
-
「写侧按分片序号、读侧按 mtime」的第二口径。
|
|
148
|
-
"""
|
|
149
|
-
recs = []
|
|
150
|
-
for name in self.shards():
|
|
151
|
-
recs.extend(read_jsonl(os.path.join(self.archive, name)))
|
|
152
|
-
recs.extend(read_jsonl(self.active))
|
|
153
|
-
if limit:
|
|
154
|
-
recs = recs[-int(limit):]
|
|
155
|
-
return recs
|
|
156
|
-
|
|
157
|
-
# 生效条件:self._index 为 None 时尝试读取 archive 中 index_name 并过滤 value 为 dict,读取失败或数据非 dict 时为空字典,随后缓存并返回;非 None 时不重读直接返回。
|
|
158
|
-
def load_index(self) -> dict:
|
|
159
|
-
if self._index is None:
|
|
160
|
-
idx = {}
|
|
161
|
-
try:
|
|
162
|
-
with open(os.path.join(self.archive, self.index_name),
|
|
163
|
-
"r", encoding="utf-8") as f:
|
|
164
|
-
data = json.load(f)
|
|
165
|
-
if isinstance(data, dict):
|
|
166
|
-
idx = {k: v for k, v in data.items() if isinstance(v, dict)}
|
|
167
|
-
except (OSError, ValueError):
|
|
168
|
-
idx = {} # 索引缺失/损坏 → 空起步,自愈补数
|
|
169
|
-
self._index = idx
|
|
170
|
-
return self._index
|
|
171
|
-
|
|
172
|
-
# 生效条件:idx 传入时尝试创建 archive 并把 idx 的 JSON(ensure_ascii=False, indent=1, sort_keys=True)原子写入 index_name;OSError 被吞掉;最后把 self._index 设为 idx。
|
|
173
|
-
def save_index(self, idx: dict):
|
|
174
|
-
try:
|
|
175
|
-
os.makedirs(self.archive, exist_ok=True)
|
|
176
|
-
atomic_write(os.path.join(self.archive, self.index_name),
|
|
177
|
-
json.dumps(idx, ensure_ascii=False, indent=1,
|
|
178
|
-
sort_keys=True))
|
|
179
|
-
except OSError:
|
|
180
|
-
pass
|
|
181
|
-
self._index = idx
|
|
182
|
-
|
|
183
|
-
# 生效条件:path 与 size 传入时,size 为 None 则先 getsize(OSError 返回 bytes/events 0 exact True);size <= count_max_bytes 时返回 _counter(path) 精确条数 exact True;超过时返回 _scale_reader(path) 的事件数 exact False。
|
|
184
|
-
def count_shard(self, path, size: int = None) -> dict:
|
|
185
|
-
"""分片条数读数:有界分片给精确值,超大历史分片只给量级。"""
|
|
186
|
-
if size is None:
|
|
187
|
-
try:
|
|
188
|
-
size = os.path.getsize(path)
|
|
189
|
-
except OSError:
|
|
190
|
-
return {"bytes": 0, "events": 0, "exact": True}
|
|
191
|
-
if size <= self.count_max_bytes:
|
|
192
|
-
return {"bytes": size,
|
|
193
|
-
"events": (self._counter or count_jsonl)(path),
|
|
194
|
-
"exact": True}
|
|
195
|
-
s = self._scale(path)
|
|
196
|
-
return {"bytes": size, "events": s["events"], "exact": False}
|
|
197
|
-
|
|
198
|
-
# 生效条件:无入参,按 self._scale_reader(缺省本模块 log_scale,count_max_bytes 透传)读 path 的量级,返回其 dict。
|
|
199
|
-
def _scale(self, path):
|
|
200
|
-
if self._scale_reader is not None:
|
|
201
|
-
return self._scale_reader(path)
|
|
202
|
-
return log_scale(path, count_max_bytes=self.count_max_bytes,
|
|
203
|
-
counter=self._counter)
|
|
204
|
-
|
|
205
|
-
# 生效条件:在已用 root 构造的实例上遍历 shards(),对 "<basename>.<n>.jsonl" 中 int(n) 成功者取最大值 top(解析失败 continue、无可解析项 top=0),返回 top+1。
|
|
206
|
-
def next_seq(self) -> int:
|
|
207
|
-
top = 0
|
|
208
|
-
n_from = len(self.basename) + 1
|
|
209
|
-
for n in self.shards():
|
|
210
|
-
try:
|
|
211
|
-
top = max(top, int(n[n_from:-len(".jsonl")]))
|
|
212
|
-
except ValueError:
|
|
213
|
-
continue
|
|
214
|
-
return top + 1
|
|
215
|
-
|
|
216
|
-
# 生效条件:keep_shards > 0 且分片数超过该值时,gone=shards[:-keep] 逐个尝试 os.remove、成功则从索引 pop(OSError continue),有 gone 才保存索引并返回 gone;keep<=0 或 gone 为空时返回 []。
|
|
217
|
-
def prune(self):
|
|
218
|
-
"""保留最近 keep_shards 个分片、淘汰更旧的(≤0 表示不淘汰)。"""
|
|
219
|
-
keep = self.keep_shards
|
|
220
|
-
if keep <= 0:
|
|
221
|
-
return []
|
|
222
|
-
shards = self.shards()
|
|
223
|
-
gone = shards[:-keep] if len(shards) > keep else []
|
|
224
|
-
if not gone:
|
|
225
|
-
return []
|
|
226
|
-
idx = self.load_index()
|
|
227
|
-
# issue #472 站点 3:剪前写**独立对账单**(被剪分片名 + 各自行数 + 汇总)。
|
|
228
|
-
# **文件名不得落在 self.shards() 命名空间内**(否则会被同一 prune 剪掉 —— 那正是原病)。
|
|
229
|
-
_led = os.path.join(self.archive, PRUNE_LEDGER)
|
|
230
|
-
try:
|
|
231
|
-
_lines = {}
|
|
232
|
-
for _n in gone:
|
|
233
|
-
try:
|
|
234
|
-
_lines[_n] = sum(1 for _l in io.open( os.path.join(self.archive, _n), encoding="utf-8") if _l.strip())
|
|
235
|
-
except OSError:
|
|
236
|
-
_lines[_n] = None
|
|
237
|
-
_rec = {"t": time.time(), "kept": keep, "pruned": list(gone),
|
|
238
|
-
"lines": _lines,
|
|
239
|
-
"lines_total": sum(v for v in _lines.values() if v)}
|
|
240
|
-
with io.open(_led, "a", encoding="utf-8") as _f:
|
|
241
|
-
_f.write(json.dumps(_rec, ensure_ascii=False) + chr(10))
|
|
242
|
-
except Exception as _e: # #472 站点 3:**对账单失败不静默**(与站点 2 同精神)
|
|
243
|
-
sys.stderr.write("[rotate] prune 对账单写入失败(剪枝继续):%s: %s"
|
|
244
|
-
% (type(_e).__name__, _e) + chr(10))
|
|
245
|
-
for n in gone:
|
|
246
|
-
try:
|
|
247
|
-
os.remove(os.path.join(self.archive, n))
|
|
248
|
-
except OSError:
|
|
249
|
-
continue # 删不掉就留着:不假装已淘汰
|
|
250
|
-
idx.pop(n, None)
|
|
251
|
-
self.save_index(idx)
|
|
252
|
-
return gone
|
|
253
|
-
|
|
254
|
-
# 生效条件:rotate_bytes>0 且 self.writes 自增后能被 probe_every 整除,且 active 的 getsize 不小于 rotate_bytes 时调用 rotate() 并返回其结果;rotate_bytes<=0、未到探测间隔、getsize 不足或 OSError 时返回 None。
|
|
255
|
-
def maybe_rotate(self):
|
|
256
|
-
"""写前闸门:活动日志达阈值即切分(把 stat 摊到 1/probe_every)。"""
|
|
257
|
-
if self.rotate_bytes <= 0:
|
|
258
|
-
return None
|
|
259
|
-
self.writes = (self.writes or 0) + 1
|
|
260
|
-
if self.writes % self.probe_every:
|
|
261
|
-
return None
|
|
262
|
-
try:
|
|
263
|
-
if os.path.getsize(self.active) < self.rotate_bytes:
|
|
264
|
-
return None
|
|
265
|
-
except OSError:
|
|
266
|
-
return None
|
|
267
|
-
return self.rotate()
|
|
268
|
-
|
|
269
|
-
# 生效条件:reason(默认 "size")传入时,在 FileLock(lock_path) 下若 active 的 size >= rotate_bytes 则 publish 为 "<basename>.<6位序号>.jsonl" 归档并更新索引/剪枝/自述留痕后返回 {"shard","bytes","events","pruned"};size 不足、getsize OSError 或 publish 失败时返回 None;成功后 writes 归 0。
|
|
270
|
-
def rotate(self, reason: str = "size"):
|
|
271
|
-
"""把活动日志切分为归档分片(publish = rename 原子,不重写一个字节)。"""
|
|
272
|
-
with FileLock(self.lock_path, timeout=5.0):
|
|
273
|
-
try:
|
|
274
|
-
size = os.path.getsize(self.active)
|
|
275
|
-
except OSError:
|
|
276
|
-
return None
|
|
277
|
-
if size < self.rotate_bytes:
|
|
278
|
-
return None # 已被并发写者轮转
|
|
279
|
-
os.makedirs(self.archive, exist_ok=True)
|
|
280
|
-
name = "%s.%06d.jsonl" % (self.basename, self.next_seq())
|
|
281
|
-
dst = os.path.join(self.archive, name)
|
|
282
|
-
try:
|
|
283
|
-
publish(self.active, dst) # Windows 短重试:AV 短锁不误判抢输
|
|
284
|
-
except OSError:
|
|
285
|
-
return None # 抢输(文件已被移走)→ 让位,不报错
|
|
286
|
-
scale = self.count_shard(dst, size) # 分档:有界扫描 / 只读量级
|
|
287
|
-
events = scale["events"]
|
|
288
|
-
idx = self.load_index()
|
|
289
|
-
idx[name] = {"bytes": size, "events": events, "exact": scale["exact"],
|
|
290
|
-
"t": round(self._now(), 3), "reason": reason}
|
|
291
|
-
self.save_index(idx)
|
|
292
|
-
pruned = self.prune()
|
|
293
|
-
self.writes = 0
|
|
294
|
-
try: # 自述留痕:轮转本身可审计
|
|
295
|
-
mark = (self._mark(name, size, events, pruned, reason)
|
|
296
|
-
if self._mark is not None else None)
|
|
297
|
-
if mark is not None:
|
|
298
|
-
# #326-④:append_jsonl 不再上抛、以 False 报写失败——本处**仍容忍**(best-effort
|
|
299
|
-
# 簿记),但失败不再静默:源头已记 fsutil.APPEND_FAILURES + 有界 stderr 告警
|
|
300
|
-
# (B 面单点)。显式消费返回值:不留裸调用(守卫 AST 钉子会抓)。
|
|
301
|
-
_ok = append_jsonl(self.active, mark)
|
|
302
|
-
except OSError:
|
|
303
|
-
pass
|
|
304
|
-
return {"shard": name, "bytes": size, "events": events,
|
|
305
|
-
"pruned": pruned}
|
|
306
|
-
|
|
307
|
-
# 生效条件:以 _scale(active) 取活动读数为 active,对磁盘现有分片 present 中未登记者按 count_shard 采纳进 idx(OSError continue,有采纳才写回索引),返回 active 并附 shards/shard_bytes/shard_events(均只统计 k in present)/oversized(非 exact 分片数)/total_bytes/total_events(active["events"] 为 None 时取 None)/total_exact/rotate_bytes/keep_shards。
|
|
308
|
-
def scale(self) -> dict:
|
|
309
|
-
"""日志面量级读数(O(1) 稳态):活动文件实时读数 + 归档分片索引缓存。"""
|
|
310
|
-
active = self._scale(self.active)
|
|
311
|
-
present = self.shards()
|
|
312
|
-
idx = self.load_index()
|
|
313
|
-
healed = False
|
|
314
|
-
for n in present:
|
|
315
|
-
if n in idx:
|
|
316
|
-
continue
|
|
317
|
-
p = os.path.join(self.archive, n)
|
|
318
|
-
try:
|
|
319
|
-
sc = self.count_shard(p) # 分档:有界扫描 / 只读量级
|
|
320
|
-
idx[n] = {"bytes": sc["bytes"], "events": sc["events"],
|
|
321
|
-
"exact": sc["exact"], "t": round(self._now(), 3),
|
|
322
|
-
"reason": "adopted"}
|
|
323
|
-
healed = True
|
|
324
|
-
except OSError:
|
|
325
|
-
continue
|
|
326
|
-
if healed:
|
|
327
|
-
self.save_index(idx)
|
|
328
|
-
shard_bytes = sum(int(v.get("bytes") or 0)
|
|
329
|
-
for k, v in idx.items() if k in present)
|
|
330
|
-
shard_events = sum(int(v.get("events") or 0)
|
|
331
|
-
for k, v in idx.items() if k in present)
|
|
332
|
-
oversized = sum(1 for n in present
|
|
333
|
-
if not idx.get(n, {}).get("exact", True))
|
|
334
|
-
out = dict(active)
|
|
335
|
-
out.update({
|
|
336
|
-
"shards": len(present),
|
|
337
|
-
"shard_bytes": shard_bytes,
|
|
338
|
-
"shard_events": shard_events,
|
|
339
|
-
"oversized": oversized, # 非精确分片数(历史遗留超大文件)
|
|
340
|
-
"total_bytes": active["bytes"] + shard_bytes,
|
|
341
|
-
"total_events": (active["events"] + shard_events
|
|
342
|
-
if active["events"] is not None else None),
|
|
343
|
-
"total_exact": bool(active["exact"]) and oversized == 0,
|
|
344
|
-
"rotate_bytes": self.rotate_bytes,
|
|
345
|
-
"keep_shards": self.keep_shards,
|
|
346
|
-
})
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""分片日志轮转·参数化共享件(有界性的来源)。
|
|
3
|
+
|
|
4
|
+
背景(第 4 条取证,2026-09-16):审计日志无上界增长——单文件形态让任何「读它」
|
|
5
|
+
的操作随运行时长线性劣化。上一轮把体检读数降为 O(1) 元数据口径(`_log_scale`),
|
|
6
|
+
只让「指标与代价错配」不再显形;**有界性的来源是轮转**:单文件 ≤ rotate_bytes、
|
|
7
|
+
分片数 ≤ keep_shards ⇒ 单片读取代价与总量都有上界。
|
|
8
|
+
|
|
9
|
+
本模块是**审计面**轮转的参数化共享件(A2,2026-10-10 设计者裁定):`mdcos` 的
|
|
10
|
+
审计轮转(`_audit.jsonl`)与 `crypto.audit`(`_crypto.jsonl`)此前各自有无一份,
|
|
11
|
+
现统一改调 `Rotator`。参数化面:目录 + 基名 + 归档目录 + 索引名 + 阈值 + 保留数 +
|
|
12
|
+
探测节流 + 计数口径(`counter` / `scale_reader`)+ 自述记录形状(`mark_factory`)
|
|
13
|
+
+ 时钟(`clock`,取证/守卫用)。
|
|
14
|
+
|
|
15
|
+
**范围订正(2026-10-10,DSH 端独立复核;原注释「唯一实现点」不实)**:本模块覆盖
|
|
16
|
+
**审计(`_audit.jsonl`)与密文(`_crypto.jsonl`)两路**;**心跳台账另有一份独立轮转**
|
|
17
|
+
——`md_cg/sustain.py::rotate_heartbeat`(分片名 `_heartbeat.%06d.jsonl`,自带
|
|
18
|
+
`_hb_next_seq` / `_hb_load_index` / `_hb_prune_shards`),它**不**经 `Rotator`。
|
|
19
|
+
本注释此前写「轮转机制唯一实现点」与该事实不符,现订正为「审计/密文两路共用此实现;
|
|
20
|
+
心跳另有一份独立轮转」。守卫 `test_audit_rotate_parity` 的 ③d 已能扫出「他处仍有
|
|
21
|
+
独立实现」并点名到文件,防止再新增第三份而不被发现。
|
|
22
|
+
|
|
23
|
+
设计边界(诚实面)
|
|
24
|
+
------------------
|
|
25
|
+
· 分片内容与轮转前**逐行一致**(`publish` = rename,不动一个字节);
|
|
26
|
+
· 并发由 `FileLock` + 「rename 前复检大小 / 失败即返回 None」兜住:抢输者不重复
|
|
27
|
+
切分,也不丢记录;
|
|
28
|
+
· 保留策略只淘汰**分片**,淘汰名单写进自述记录(不静默丢证据);
|
|
29
|
+
· 超大历史分片在轮转/体检路径上**只读量级**(`scale_reader`),不付 O(n) 全量解析
|
|
30
|
+
——「一次调用堵死通道」不在轮转路径上复现。
|
|
31
|
+
"""
|
|
32
|
+
from __future__ import annotations
|
|
33
|
+
import sys
|
|
34
|
+
import io
|
|
35
|
+
|
|
36
|
+
import json
|
|
37
|
+
import os
|
|
38
|
+
import time
|
|
39
|
+
|
|
40
|
+
from .fsutil import (FileLock, append_jsonl, atomic_write, count_jsonl,
|
|
41
|
+
publish, read_jsonl)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
# 生效条件:path 为日志路径、count_max_bytes 为规模阈值、est_sample 为尾部采样字节数、counter 为行数计数器(缺省 count_jsonl)时,先 os.stat;OSError 返回 {'bytes':0,'mtime':None,'events':0,'exact':True};size 不超过 count_max_bytes 时返回 counter(path) 的精确条数与 exact=True;否则读末尾 est_sample 字节估算 events、exact=False 并附 note(不触发任何全量计数)。
|
|
45
|
+
def log_scale(path, count_max_bytes=64 << 20, est_sample=256 << 10,
|
|
46
|
+
counter=None):
|
|
47
|
+
"""日志量级读数(O(1)):以元数据为主,条数只在与规模相称时才精确。"""
|
|
48
|
+
counter = counter or count_jsonl
|
|
49
|
+
try:
|
|
50
|
+
st = os.stat(path)
|
|
51
|
+
except OSError:
|
|
52
|
+
return {"bytes": 0, "mtime": None, "events": 0, "exact": True}
|
|
53
|
+
size = st.st_size
|
|
54
|
+
out = {"bytes": size, "mtime": st.st_mtime}
|
|
55
|
+
if size <= count_max_bytes:
|
|
56
|
+
out.update({"events": counter(path), "exact": True})
|
|
57
|
+
return out
|
|
58
|
+
est = None
|
|
59
|
+
try:
|
|
60
|
+
with open(path, "rb") as f:
|
|
61
|
+
f.seek(max(0, size - est_sample))
|
|
62
|
+
tail = f.read()
|
|
63
|
+
lines = [ln for ln in tail.split(b"\n") if ln.strip()]
|
|
64
|
+
if lines:
|
|
65
|
+
est = int(size / (len(tail) / float(len(lines))))
|
|
66
|
+
except OSError:
|
|
67
|
+
pass
|
|
68
|
+
out.update({
|
|
69
|
+
"events": est, "exact": False,
|
|
70
|
+
"note": ("超过 %.0f MB 不给全量精确计数(O(n) 磁盘 IO 与量级体检不匹配);"
|
|
71
|
+
"events 为尾部 %d KB 采样的估算值"
|
|
72
|
+
% (count_max_bytes / 1048576.0, est_sample >> 10))})
|
|
73
|
+
return out
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
# 生效条件:注入的 clock 为 None 时用 lambda 延迟解析本模块 time(守卫冻结时钟可 patch);注入时原样使用。
|
|
77
|
+
def _default_clock():
|
|
78
|
+
return lambda: time.time()
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
# 生效条件:以 root/basename/archive_name 及可选阈值与回调构造实例;active/archive/lock 路径由 root 与基名拼接,_index 缓存与 writes 计数初值为 None/0;不读盘、不建目录、不抛异常。
|
|
82
|
+
PRUNE_LEDGER = "_prune_ledger.jsonl" #: #472:剪枝对账单(**不在 shards() 命名空间内**)
|
|
83
|
+
|
|
84
|
+
class Rotator:
|
|
85
|
+
"""参数化分片轮转器:一次实现,审计面与 crypto 面共用。
|
|
86
|
+
|
|
87
|
+
· `basename` —— 活动文件基名(`_audit` / `_crypto`)⇒ 活动文件
|
|
88
|
+
`<root>/<basename>.jsonl`、分片 `<archive>/<basename>.<6位序号>.jsonl`。
|
|
89
|
+
· `archive_name` —— 归档目录名(`_audit_archive` / `_crypto_archive`)。
|
|
90
|
+
· `counter` —— 精确行数计数(默认 `fsutil.count_jsonl`,流式不物化)。
|
|
91
|
+
· `scale_reader` —— 超大分片的量级读数(默认本模块 `log_scale`);mdcos
|
|
92
|
+
传自身 `_log_scale` 以保持既有读数口径逐位不变。
|
|
93
|
+
· `mark_factory(name, size, events, pruned, reason)` —— 轮转自述记录
|
|
94
|
+
(形状随消费方协议;返回 None 则不留痕,调用方自担)。
|
|
95
|
+
· `clock` —— 时间源(默认本模块 `time.time`)。
|
|
96
|
+
"""
|
|
97
|
+
|
|
98
|
+
# 生效条件:root、basename、archive_name 为必填;index_name/lock_name 缺省 "_index.json" 与 basename+".rotate.lock";rotate_bytes/keep_shards/probe_every/count_max_bytes 缺省 64<<20/8/32/64<<20;counter/scale_reader/mark_factory/clock 缺省见类文档;随后把路径与回调存入实例,_index=None、writes=0。
|
|
99
|
+
def __init__(self, root, basename, archive_name, index_name="_index.json",
|
|
100
|
+
lock_name=None, rotate_bytes=64 << 20, keep_shards=8,
|
|
101
|
+
probe_every=32, count_max_bytes=64 << 20,
|
|
102
|
+
counter=None, scale_reader=None, mark_factory=None,
|
|
103
|
+
clock=None):
|
|
104
|
+
self.root = root
|
|
105
|
+
self.basename = basename
|
|
106
|
+
self.active = os.path.join(root, basename + ".jsonl")
|
|
107
|
+
self.archive = os.path.join(root, archive_name)
|
|
108
|
+
self.index_name = index_name
|
|
109
|
+
# issue #521:`FileLock` 自己会补 ".lock"(fsutil.py:124)⇒ 此处**只给不含 .lock 的基名**,
|
|
110
|
+
# 否则 `basename.rotate.lock` + `.lock` = `…rotate.lock.lock`(后缀拼接职责重叠)。
|
|
111
|
+
# ★ 形态(本条正文自带):「不在清单里 ≠ 不存在;做差集前必须先判『清单是否穷举』」。
|
|
112
|
+
self.lock_path = os.path.join(root, lock_name or
|
|
113
|
+
(basename + ".rotate"))
|
|
114
|
+
self.rotate_bytes = rotate_bytes
|
|
115
|
+
self.keep_shards = keep_shards
|
|
116
|
+
self.probe_every = probe_every
|
|
117
|
+
self.count_max_bytes = count_max_bytes
|
|
118
|
+
self._counter = counter
|
|
119
|
+
self._scale_reader = scale_reader
|
|
120
|
+
self._mark = mark_factory
|
|
121
|
+
self._clock = clock if clock is not None else _default_clock()
|
|
122
|
+
self._index = None
|
|
123
|
+
self.writes = 0
|
|
124
|
+
|
|
125
|
+
# 生效条件:无入参,返回 self._clock() 的结果(默认 time.time())。
|
|
126
|
+
def _now(self):
|
|
127
|
+
return self._clock()
|
|
128
|
+
|
|
129
|
+
# 生效条件:archive 可被 os.listdir 列出时返回其中以 "<basename>." 开头且以 ".jsonl" 结尾的名字升序列表;listdir 抛 OSError 时返回 []。
|
|
130
|
+
def shards(self):
|
|
131
|
+
"""归档分片名,序号零填充 ⇒ 字典序 == 时间序。"""
|
|
132
|
+
try:
|
|
133
|
+
names = os.listdir(self.archive)
|
|
134
|
+
except OSError:
|
|
135
|
+
return []
|
|
136
|
+
prefix = self.basename + "."
|
|
137
|
+
return sorted(n for n in names
|
|
138
|
+
if n.startswith(prefix) and n.endswith(".jsonl"))
|
|
139
|
+
|
|
140
|
+
# 生效条件:无必需形参;按 shards() 的字典序(== 时间序)逐个读归档分片、最后读活动
|
|
141
|
+
# 文件,返回记录列表;limit 为真时只取尾部 limit 条;单个文件不可读时按空计。
|
|
142
|
+
def read_records(self, limit=0):
|
|
143
|
+
"""活动文件 + 全部分片的记录(**归档感知**读面,#212 缺口 4)。
|
|
144
|
+
|
|
145
|
+
为什么放在 Rotator 而不是各账本各写一份:shards() 已把「序号零填充 ⇒
|
|
146
|
+
字典序 == 时间序」这条口径钉在这里,读面必须与它同源,否则又会长出
|
|
147
|
+
「写侧按分片序号、读侧按 mtime」的第二口径。
|
|
148
|
+
"""
|
|
149
|
+
recs = []
|
|
150
|
+
for name in self.shards():
|
|
151
|
+
recs.extend(read_jsonl(os.path.join(self.archive, name)))
|
|
152
|
+
recs.extend(read_jsonl(self.active))
|
|
153
|
+
if limit:
|
|
154
|
+
recs = recs[-int(limit):]
|
|
155
|
+
return recs
|
|
156
|
+
|
|
157
|
+
# 生效条件:self._index 为 None 时尝试读取 archive 中 index_name 并过滤 value 为 dict,读取失败或数据非 dict 时为空字典,随后缓存并返回;非 None 时不重读直接返回。
|
|
158
|
+
def load_index(self) -> dict:
|
|
159
|
+
if self._index is None:
|
|
160
|
+
idx = {}
|
|
161
|
+
try:
|
|
162
|
+
with open(os.path.join(self.archive, self.index_name),
|
|
163
|
+
"r", encoding="utf-8") as f:
|
|
164
|
+
data = json.load(f)
|
|
165
|
+
if isinstance(data, dict):
|
|
166
|
+
idx = {k: v for k, v in data.items() if isinstance(v, dict)}
|
|
167
|
+
except (OSError, ValueError):
|
|
168
|
+
idx = {} # 索引缺失/损坏 → 空起步,自愈补数
|
|
169
|
+
self._index = idx
|
|
170
|
+
return self._index
|
|
171
|
+
|
|
172
|
+
# 生效条件:idx 传入时尝试创建 archive 并把 idx 的 JSON(ensure_ascii=False, indent=1, sort_keys=True)原子写入 index_name;OSError 被吞掉;最后把 self._index 设为 idx。
|
|
173
|
+
def save_index(self, idx: dict):
|
|
174
|
+
try:
|
|
175
|
+
os.makedirs(self.archive, exist_ok=True)
|
|
176
|
+
atomic_write(os.path.join(self.archive, self.index_name),
|
|
177
|
+
json.dumps(idx, ensure_ascii=False, indent=1,
|
|
178
|
+
sort_keys=True))
|
|
179
|
+
except OSError:
|
|
180
|
+
pass
|
|
181
|
+
self._index = idx
|
|
182
|
+
|
|
183
|
+
# 生效条件:path 与 size 传入时,size 为 None 则先 getsize(OSError 返回 bytes/events 0 exact True);size <= count_max_bytes 时返回 _counter(path) 精确条数 exact True;超过时返回 _scale_reader(path) 的事件数 exact False。
|
|
184
|
+
def count_shard(self, path, size: int = None) -> dict:
|
|
185
|
+
"""分片条数读数:有界分片给精确值,超大历史分片只给量级。"""
|
|
186
|
+
if size is None:
|
|
187
|
+
try:
|
|
188
|
+
size = os.path.getsize(path)
|
|
189
|
+
except OSError:
|
|
190
|
+
return {"bytes": 0, "events": 0, "exact": True}
|
|
191
|
+
if size <= self.count_max_bytes:
|
|
192
|
+
return {"bytes": size,
|
|
193
|
+
"events": (self._counter or count_jsonl)(path),
|
|
194
|
+
"exact": True}
|
|
195
|
+
s = self._scale(path)
|
|
196
|
+
return {"bytes": size, "events": s["events"], "exact": False}
|
|
197
|
+
|
|
198
|
+
# 生效条件:无入参,按 self._scale_reader(缺省本模块 log_scale,count_max_bytes 透传)读 path 的量级,返回其 dict。
|
|
199
|
+
def _scale(self, path):
|
|
200
|
+
if self._scale_reader is not None:
|
|
201
|
+
return self._scale_reader(path)
|
|
202
|
+
return log_scale(path, count_max_bytes=self.count_max_bytes,
|
|
203
|
+
counter=self._counter)
|
|
204
|
+
|
|
205
|
+
# 生效条件:在已用 root 构造的实例上遍历 shards(),对 "<basename>.<n>.jsonl" 中 int(n) 成功者取最大值 top(解析失败 continue、无可解析项 top=0),返回 top+1。
|
|
206
|
+
def next_seq(self) -> int:
|
|
207
|
+
top = 0
|
|
208
|
+
n_from = len(self.basename) + 1
|
|
209
|
+
for n in self.shards():
|
|
210
|
+
try:
|
|
211
|
+
top = max(top, int(n[n_from:-len(".jsonl")]))
|
|
212
|
+
except ValueError:
|
|
213
|
+
continue
|
|
214
|
+
return top + 1
|
|
215
|
+
|
|
216
|
+
# 生效条件:keep_shards > 0 且分片数超过该值时,gone=shards[:-keep] 逐个尝试 os.remove、成功则从索引 pop(OSError continue),有 gone 才保存索引并返回 gone;keep<=0 或 gone 为空时返回 []。
|
|
217
|
+
def prune(self):
|
|
218
|
+
"""保留最近 keep_shards 个分片、淘汰更旧的(≤0 表示不淘汰)。"""
|
|
219
|
+
keep = self.keep_shards
|
|
220
|
+
if keep <= 0:
|
|
221
|
+
return []
|
|
222
|
+
shards = self.shards()
|
|
223
|
+
gone = shards[:-keep] if len(shards) > keep else []
|
|
224
|
+
if not gone:
|
|
225
|
+
return []
|
|
226
|
+
idx = self.load_index()
|
|
227
|
+
# issue #472 站点 3:剪前写**独立对账单**(被剪分片名 + 各自行数 + 汇总)。
|
|
228
|
+
# **文件名不得落在 self.shards() 命名空间内**(否则会被同一 prune 剪掉 —— 那正是原病)。
|
|
229
|
+
_led = os.path.join(self.archive, PRUNE_LEDGER)
|
|
230
|
+
try:
|
|
231
|
+
_lines = {}
|
|
232
|
+
for _n in gone:
|
|
233
|
+
try:
|
|
234
|
+
_lines[_n] = sum(1 for _l in io.open( os.path.join(self.archive, _n), encoding="utf-8") if _l.strip())
|
|
235
|
+
except OSError:
|
|
236
|
+
_lines[_n] = None
|
|
237
|
+
_rec = {"t": time.time(), "kept": keep, "pruned": list(gone),
|
|
238
|
+
"lines": _lines,
|
|
239
|
+
"lines_total": sum(v for v in _lines.values() if v)}
|
|
240
|
+
with io.open(_led, "a", encoding="utf-8") as _f:
|
|
241
|
+
_f.write(json.dumps(_rec, ensure_ascii=False) + chr(10))
|
|
242
|
+
except Exception as _e: # #472 站点 3:**对账单失败不静默**(与站点 2 同精神)
|
|
243
|
+
sys.stderr.write("[rotate] prune 对账单写入失败(剪枝继续):%s: %s"
|
|
244
|
+
% (type(_e).__name__, _e) + chr(10))
|
|
245
|
+
for n in gone:
|
|
246
|
+
try:
|
|
247
|
+
os.remove(os.path.join(self.archive, n))
|
|
248
|
+
except OSError:
|
|
249
|
+
continue # 删不掉就留着:不假装已淘汰
|
|
250
|
+
idx.pop(n, None)
|
|
251
|
+
self.save_index(idx)
|
|
252
|
+
return gone
|
|
253
|
+
|
|
254
|
+
# 生效条件:rotate_bytes>0 且 self.writes 自增后能被 probe_every 整除,且 active 的 getsize 不小于 rotate_bytes 时调用 rotate() 并返回其结果;rotate_bytes<=0、未到探测间隔、getsize 不足或 OSError 时返回 None。
|
|
255
|
+
def maybe_rotate(self):
|
|
256
|
+
"""写前闸门:活动日志达阈值即切分(把 stat 摊到 1/probe_every)。"""
|
|
257
|
+
if self.rotate_bytes <= 0:
|
|
258
|
+
return None
|
|
259
|
+
self.writes = (self.writes or 0) + 1
|
|
260
|
+
if self.writes % self.probe_every:
|
|
261
|
+
return None
|
|
262
|
+
try:
|
|
263
|
+
if os.path.getsize(self.active) < self.rotate_bytes:
|
|
264
|
+
return None
|
|
265
|
+
except OSError:
|
|
266
|
+
return None
|
|
267
|
+
return self.rotate()
|
|
268
|
+
|
|
269
|
+
# 生效条件:reason(默认 "size")传入时,在 FileLock(lock_path) 下若 active 的 size >= rotate_bytes 则 publish 为 "<basename>.<6位序号>.jsonl" 归档并更新索引/剪枝/自述留痕后返回 {"shard","bytes","events","pruned"};size 不足、getsize OSError 或 publish 失败时返回 None;成功后 writes 归 0。
|
|
270
|
+
def rotate(self, reason: str = "size"):
|
|
271
|
+
"""把活动日志切分为归档分片(publish = rename 原子,不重写一个字节)。"""
|
|
272
|
+
with FileLock(self.lock_path, timeout=5.0):
|
|
273
|
+
try:
|
|
274
|
+
size = os.path.getsize(self.active)
|
|
275
|
+
except OSError:
|
|
276
|
+
return None
|
|
277
|
+
if size < self.rotate_bytes:
|
|
278
|
+
return None # 已被并发写者轮转
|
|
279
|
+
os.makedirs(self.archive, exist_ok=True)
|
|
280
|
+
name = "%s.%06d.jsonl" % (self.basename, self.next_seq())
|
|
281
|
+
dst = os.path.join(self.archive, name)
|
|
282
|
+
try:
|
|
283
|
+
publish(self.active, dst) # Windows 短重试:AV 短锁不误判抢输
|
|
284
|
+
except OSError:
|
|
285
|
+
return None # 抢输(文件已被移走)→ 让位,不报错
|
|
286
|
+
scale = self.count_shard(dst, size) # 分档:有界扫描 / 只读量级
|
|
287
|
+
events = scale["events"]
|
|
288
|
+
idx = self.load_index()
|
|
289
|
+
idx[name] = {"bytes": size, "events": events, "exact": scale["exact"],
|
|
290
|
+
"t": round(self._now(), 3), "reason": reason}
|
|
291
|
+
self.save_index(idx)
|
|
292
|
+
pruned = self.prune()
|
|
293
|
+
self.writes = 0
|
|
294
|
+
try: # 自述留痕:轮转本身可审计
|
|
295
|
+
mark = (self._mark(name, size, events, pruned, reason)
|
|
296
|
+
if self._mark is not None else None)
|
|
297
|
+
if mark is not None:
|
|
298
|
+
# #326-④:append_jsonl 不再上抛、以 False 报写失败——本处**仍容忍**(best-effort
|
|
299
|
+
# 簿记),但失败不再静默:源头已记 fsutil.APPEND_FAILURES + 有界 stderr 告警
|
|
300
|
+
# (B 面单点)。显式消费返回值:不留裸调用(守卫 AST 钉子会抓)。
|
|
301
|
+
_ok = append_jsonl(self.active, mark)
|
|
302
|
+
except OSError:
|
|
303
|
+
pass
|
|
304
|
+
return {"shard": name, "bytes": size, "events": events,
|
|
305
|
+
"pruned": pruned}
|
|
306
|
+
|
|
307
|
+
# 生效条件:以 _scale(active) 取活动读数为 active,对磁盘现有分片 present 中未登记者按 count_shard 采纳进 idx(OSError continue,有采纳才写回索引),返回 active 并附 shards/shard_bytes/shard_events(均只统计 k in present)/oversized(非 exact 分片数)/total_bytes/total_events(active["events"] 为 None 时取 None)/total_exact/rotate_bytes/keep_shards。
|
|
308
|
+
def scale(self) -> dict:
|
|
309
|
+
"""日志面量级读数(O(1) 稳态):活动文件实时读数 + 归档分片索引缓存。"""
|
|
310
|
+
active = self._scale(self.active)
|
|
311
|
+
present = self.shards()
|
|
312
|
+
idx = self.load_index()
|
|
313
|
+
healed = False
|
|
314
|
+
for n in present:
|
|
315
|
+
if n in idx:
|
|
316
|
+
continue
|
|
317
|
+
p = os.path.join(self.archive, n)
|
|
318
|
+
try:
|
|
319
|
+
sc = self.count_shard(p) # 分档:有界扫描 / 只读量级
|
|
320
|
+
idx[n] = {"bytes": sc["bytes"], "events": sc["events"],
|
|
321
|
+
"exact": sc["exact"], "t": round(self._now(), 3),
|
|
322
|
+
"reason": "adopted"}
|
|
323
|
+
healed = True
|
|
324
|
+
except OSError:
|
|
325
|
+
continue
|
|
326
|
+
if healed:
|
|
327
|
+
self.save_index(idx)
|
|
328
|
+
shard_bytes = sum(int(v.get("bytes") or 0)
|
|
329
|
+
for k, v in idx.items() if k in present)
|
|
330
|
+
shard_events = sum(int(v.get("events") or 0)
|
|
331
|
+
for k, v in idx.items() if k in present)
|
|
332
|
+
oversized = sum(1 for n in present
|
|
333
|
+
if not idx.get(n, {}).get("exact", True))
|
|
334
|
+
out = dict(active)
|
|
335
|
+
out.update({
|
|
336
|
+
"shards": len(present),
|
|
337
|
+
"shard_bytes": shard_bytes,
|
|
338
|
+
"shard_events": shard_events,
|
|
339
|
+
"oversized": oversized, # 非精确分片数(历史遗留超大文件)
|
|
340
|
+
"total_bytes": active["bytes"] + shard_bytes,
|
|
341
|
+
"total_events": (active["events"] + shard_events
|
|
342
|
+
if active["events"] is not None else None),
|
|
343
|
+
"total_exact": bool(active["exact"]) and oversized == 0,
|
|
344
|
+
"rotate_bytes": self.rotate_bytes,
|
|
345
|
+
"keep_shards": self.keep_shards,
|
|
346
|
+
})
|
|
347
347
|
return out
|