@furongjun1999/dsh-memory 0.9.22 → 0.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.en.md +450 -0
- package/CHANGELOG.md +112 -0
- package/README.en.md +244 -0
- package/README.md +272 -260
- package/codebuddy/CODEBUDDY.md +1 -1
- package/docs/README.md +2 -0
- package/docs/discipline/templates/full.md.tmpl +1 -1
- package/docs/discipline/templates/rules.mdc.tmpl +1 -1
- package/docs/en/README.md +190 -0
- package/docs/en/TRANSLATION.md +78 -0
- package/docs/en/TRANSLATION_MANIFEST.json +170 -0
- package/docs/en/TRANSLATION_REPORT.md +11 -0
- package/docs/en/dev/README.md +10 -0
- package/docs/en/dev/coding-testing.md +72 -0
- package/docs/en/dev/switch-matrix.md +10 -0
- package/docs/en/glossary.md +97 -0
- package/docs/en/harnesses/claude.md +100 -0
- package/docs/en/harnesses/codebuddy.md +93 -0
- package/docs/en/harnesses/codex.md +88 -0
- package/docs/en/harnesses/dsh.md +84 -0
- package/docs/en/harnesses/zcode.md +42 -0
- package/docs/en/mdcg/ccg.md +32 -0
- package/docs/en/mdcg/generated-docs.md +20 -0
- package/docs/en/mdcg/guardrail-charter.md +170 -0
- package/docs/en/mdcg/mcp-tool-names.md +43 -0
- package/docs/en/mdcg/multi-harness.md +43 -0
- package/docs/en/mdcg/recall-metadata.md +30 -0
- package/docs/en/mdcg/world-model-endpoint.md +379 -0
- package/docs/en/mdcg/write-aliases.md +130 -0
- package/docs/en/ops/README.md +11 -0
- package/docs/en/ops/environment-known-issues.md +244 -0
- package/docs/en/ops/install.md +72 -0
- package/docs/en/ops/operations-troubleshooting.md +86 -0
- package/docs/en/ops/permissions-review.md +79 -0
- package/docs/en/ops/release-governance.md +49 -0
- package/docs/en/ops/tool-name-migration.md +30 -0
- package/docs/en/ops/ui-install-troubleshooting.md +53 -0
- package/docs/en/standards/ai-coding-standard-checklist.md +71 -0
- package/docs/en/standards/ai-coding-standard-lingshu.md +47 -0
- package/docs/en/standards/ai-coding-standard.md +185 -0
- package/docs/en/ui/minimal.md +62 -0
- package/docs/mdcg/cogmap_ops.md +16 -15
- package/lib/client.js +29 -3
- package/lib/hooks.d.ts +34 -0
- package/lib/hooks.js +167 -33
- package/lib/index.js +19 -3
- package/lib/lib/console_web.mjs +30 -3
- package/lib/lib/hook_audit.d.ts +3 -2
- package/lib/lib/hook_audit.js +14 -4
- package/lib/lib/mdcg_client.d.ts +5 -2
- package/lib/lib/mdcg_client.js +8 -4
- package/lib/lib/prompt_safety.d.ts +24 -0
- package/lib/lib/prompt_safety.js +77 -1
- package/lib/lib/token_store.js +5 -2
- package/lib/tools.js +5 -3
- package/md_cg/bucketindex.py +231 -0
- package/md_cg/bucketscope.py +339 -0
- package/md_cg/config_registry_bulk.py +19 -10
- package/md_cg/config_validate.py +14 -2
- package/md_cg/detrerank.py +137 -0
- package/md_cg/fsutil.py +215 -25
- package/md_cg/govern/__init__.py +23 -0
- package/md_cg/iface/__init__.py +46 -0
- package/md_cg/iface/_verify.py +609 -0
- package/md_cg/iface/contracts.py +337 -0
- package/md_cg/judgment_manifest.py +14 -0
- package/md_cg/kernel/__init__.py +27 -0
- package/md_cg/mcp_server.py +457 -23
- package/md_cg/mdcg.py +1034 -79
- package/md_cg/mdcos.py +263 -21
- package/md_cg/metacognition.py +66 -13
- package/md_cg/mutate_extract_shipped.py +126 -0
- package/md_cg/mutate_ledger_e2e.py +100 -0
- package/md_cg/nodefile.py +147 -23
- package/md_cg/op_table.py +22 -4
- package/md_cg/probe_c_ledger_integrity.py +258 -0
- package/md_cg/probe_d_retirement_discipline.py +263 -0
- package/md_cg/reconcile.py +9 -11
- package/md_cg/review_cli.py +6 -1
- package/md_cg/rollback.py +16 -1
- package/md_cg/run_tests.py +30 -1
- package/md_cg/scopeselect.py +214 -0
- package/md_cg/scrub.py +160 -23
- package/md_cg/security.py +4 -2
- package/md_cg/state_extract.py +149 -0
- package/md_cg/state_slots.py +58 -10
- package/md_cg/structboost.py +179 -0
- package/md_cg/test_alternatives_historical.py +162 -0
- package/md_cg/test_b2_sensitivity_landing.py +226 -223
- package/md_cg/test_b3_merge_keeps_content.py +608 -583
- package/md_cg/test_bucket_cards.py +85 -0
- package/md_cg/test_bucket_day_order.py +74 -0
- package/md_cg/test_bucket_quota.py +79 -0
- package/md_cg/test_bucket_recency_guards.py +75 -0
- package/md_cg/test_bucket_three_causes.py +83 -0
- package/md_cg/test_bucketindex.py +130 -0
- package/md_cg/test_bucketscope_early.py +71 -0
- package/md_cg/test_cond_term_fold.py +183 -0
- package/md_cg/test_cond_term_fold_gate.py +1210 -0
- package/md_cg/test_cond_term_neg_scope.py +147 -0
- package/md_cg/test_cond_terms_colon.py +348 -0
- package/md_cg/test_detrerank.py +157 -0
- package/md_cg/test_gate_four_kpi.py +136 -0
- package/md_cg/test_generation_guard.py +76 -43
- package/md_cg/test_issue192_move_layer_indices_guard.py +48 -0
- package/md_cg/test_issue362_neg_condition.py +44 -7
- package/md_cg/test_issue39_utf8_stdio.py +572 -572
- package/md_cg/test_issue413_preimage_failclosed.py +147 -9
- package/md_cg/test_issue444_scrub_evidence_plaintext.py +14 -8
- package/md_cg/test_issue677_flush_batch_io.py +139 -0
- package/md_cg/test_issue677_info_fast.py +82 -0
- package/md_cg/test_issue677_remember_dedupe.py +106 -0
- package/md_cg/test_issue712_tokens_sessions_read.py +199 -0
- package/md_cg/test_issue722_strict_conditions.py +96 -0
- package/md_cg/test_issue730_narrowing_audit.py +121 -0
- package/md_cg/test_issue733_verify_override.py +193 -0
- package/md_cg/test_issue733_verify_override_and_overwrite_inherit.py +315 -0
- package/md_cg/test_issue740_filelock_fd_leak.py +218 -0
- package/md_cg/test_issue741_content_hash_same_source.py +158 -0
- package/md_cg/test_issue747_slot_contradiction.py +125 -0
- package/md_cg/test_issue754_nested_root_guard.py +197 -0
- package/md_cg/test_issue757_index_hash_compare.py +85 -0
- package/md_cg/test_issue758_disk_hash_single.py +67 -0
- package/md_cg/test_issue759_twophase_hash.py +67 -0
- package/md_cg/test_issue768_overwrite_time_window_anchor.py +135 -0
- package/md_cg/test_issue789_log_plaintext_probe.py +100 -0
- package/md_cg/test_issue790_verb_negation.py +77 -0
- package/md_cg/test_issue810_root_guard.py +71 -0
- package/md_cg/test_issue812_info_index_stale.py +78 -0
- package/md_cg/test_issue827_weaken_dedup.py +361 -0
- package/md_cg/test_issue829_calibration_positive_gate.py +473 -0
- package/md_cg/test_kernel_iface_contracts.py +18 -0
- package/md_cg/test_md_cg_no_hive_dep.py +238 -17
- package/md_cg/test_n204_n205_n226_n227_n228_n229_exit_gates.py +545 -542
- package/md_cg/test_n209_verify_write_face_gates.py +5 -2
- package/md_cg/test_neg_condition_hits.py +336 -333
- package/md_cg/test_probe_cd_shipped.py +385 -0
- package/md_cg/test_recent_tail.py +93 -0
- package/md_cg/test_run_tests_entry_gates.py +110 -0
- package/md_cg/test_scopeselect.py +70 -0
- package/md_cg/test_scopeselect_clock.py +70 -0
- package/md_cg/test_scrub_source_line_false_positive.py +64 -0
- package/md_cg/test_session_attribution_consistency.py +835 -813
- package/md_cg/test_state_extract_shipped.py +162 -0
- package/md_cg/test_state_ledger_chain.py +207 -0
- package/md_cg/test_state_ledger_e2e.py +145 -0
- package/md_cg/test_structboost.py +102 -0
- package/md_cg/test_timeboost.py +83 -0
- package/md_cg/test_timeboost_relativedate.py +71 -0
- package/md_cg/test_topicagg.py +129 -0
- package/md_cg/test_twophase.py +286 -286
- package/md_cg/timeboost.py +223 -0
- package/md_cg/tokens.py +55 -7
- package/md_cg/topicagg.py +142 -0
- package/md_cg/twophase.py +15 -6
- package/md_cg/whitebox_kb/wisdom/code_compose.py +33 -5
- package/md_cg/whitebox_kb/wisdom/compose_engine.py +5 -1
- package/md_cg/whitebox_kb/wisdom/role_compose.py +271 -268
- package/md_cg/writepipe.py +35 -3
- package/package.json +4 -1
- package/scripts/workspace_index.py +2 -0
- package/src/hooks.ts +182 -34
- package/src/index.ts +19 -3
- package/src/lib/console_web.mjs +30 -3
- package/src/lib/hook_audit.ts +15 -5
- package/src/lib/mdcg_client.ts +8 -4
- package/src/lib/prompt_safety.ts +73 -1
- package/src/lib/token_store.ts +5 -2
- package/src/tools.ts +5 -3
- package/ui/minimal/integrity.json +2 -2
- package/ui/minimal/payload/build-ui.mjs +41 -2
- package/ui/minimal/payload/lib/client.js +29 -3
- package/ui/minimal/payload/lib/lib/console_web.mjs +30 -3
- package/ui/minimal/payload/src/lib/console_web.mjs +30 -3
- package/ui/minimal/payload/web/app.js +3 -1
- package/ui/minimal/payload/web/client.js +30 -3
- package/web/app.js +3 -1
- package/web/client.js +30 -3
- package/zcode/AGENTS.md +1 -1
package/md_cg/fsutil.py
CHANGED
|
@@ -131,44 +131,75 @@ class FileLock:
|
|
|
131
131
|
|
|
132
132
|
# 生效条件:先按 self.path 建父目录并 open(self.path, "a+b"),再在 self.timeout 到期前每 self.poll 秒尝试加锁(IS_WIN 用 msvcrt.locking(LK_NBLCK),否则 fcntl.flock(LOCK_EX|LOCK_NB));成功即置 self.acquired=True 并返回 self;OSError 的 errno 不在 (EACCES, EAGAIN, EDEADLK) 时直接 raise,超时后 self.strict 为真抛 TimeoutError、否则返回 self 放行。
|
|
133
133
|
def __enter__(self):
|
|
134
|
+
# issue #740:**进入前先清掉可能的旧句柄**(重复 enter / 上次未 exit)——
|
|
135
|
+
# 否则旧 fd 静默泄漏;POSIX 下还会与自己的旧 fd 互斥(表现为「卡到超时」)。
|
|
136
|
+
if getattr(self, "_f", None) is not None:
|
|
137
|
+
self._close_fd()
|
|
138
|
+
self.acquired = False
|
|
134
139
|
os.makedirs(os.path.dirname(os.path.abspath(self.path)) or ".", exist_ok=True)
|
|
135
140
|
self._f = open(self.path, "a+b")
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
141
|
+
try:
|
|
142
|
+
# #740:`try` **必须紧贴 open 之后**——`time.time() + self.timeout` 本身就可能
|
|
143
|
+
# 抛(timeout 为 None/字符串时 TypeError),那时 fd 已打开而 `__exit__` 不会
|
|
144
|
+
# 被调用(with 的清理只在 __enter__ 正常返回后才注册)。
|
|
145
|
+
deadline = time.time() + self.timeout
|
|
146
|
+
while True:
|
|
147
|
+
try:
|
|
148
|
+
if IS_WIN:
|
|
149
|
+
self._f.seek(0)
|
|
150
|
+
msvcrt.locking(self._f.fileno(), msvcrt.LK_NBLCK, 1)
|
|
151
|
+
else:
|
|
152
|
+
fcntl.flock(self._f.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
153
|
+
self.acquired = True
|
|
154
|
+
return self
|
|
155
|
+
except OSError as e:
|
|
156
|
+
if e.errno not in (errno.EACCES, errno.EAGAIN, errno.EDEADLK):
|
|
157
|
+
raise
|
|
158
|
+
if time.time() > deadline:
|
|
159
|
+
if self.strict:
|
|
160
|
+
raise TimeoutError(
|
|
161
|
+
f"FileLock 超时未获锁(strict):{self.path}")
|
|
162
|
+
# 非 strict:放行且**显式**标记未持锁(实例复用时靠它区分)。
|
|
163
|
+
self.acquired = False
|
|
164
|
+
return self
|
|
165
|
+
time.sleep(self.poll)
|
|
166
|
+
except BaseException:
|
|
167
|
+
# #740:任何异常路径(含 KeyboardInterrupt / SystemExit / TypeError)都先关 fd
|
|
168
|
+
# 再原样上抛——清理本身绝不抛(否则会遮蔽原始异常)。
|
|
169
|
+
self._close_fd()
|
|
170
|
+
raise
|
|
171
|
+
|
|
172
|
+
def _close_fd(self):
|
|
173
|
+
"""关句柄并复位状态(#740:**单点关闭**,幂等、绝不抛)。
|
|
174
|
+
|
|
175
|
+
- `getattr` 兜底:`__init__` 之前的异常/兼容旧实例时不出现 AttributeError;
|
|
176
|
+
- 吞 `(OSError, ValueError)`:重复 close(ValueError: I/O operation on closed file)
|
|
177
|
+
与平台错误都不该冒泡——清理函数抛异常会遮蔽原始异常。
|
|
178
|
+
"""
|
|
179
|
+
f = getattr(self, "_f", None)
|
|
180
|
+
self._f = None
|
|
181
|
+
self.acquired = False
|
|
182
|
+
if f is None:
|
|
183
|
+
return
|
|
184
|
+
try:
|
|
185
|
+
f.close()
|
|
186
|
+
except (OSError, ValueError):
|
|
187
|
+
pass
|
|
155
188
|
|
|
156
189
|
# 生效条件:self.acquired 为真时按 IS_WIN 用 msvcrt.locking(LK_UNLCK) 或 fcntl.flock(LOCK_UN) 解锁(OSError 被吞掉);finally 中只要 self._f 为真就 close,随后 self._f=None、self.acquired=False。
|
|
157
190
|
def __exit__(self, *exc):
|
|
158
191
|
try:
|
|
159
|
-
|
|
192
|
+
# #740:`_f` 可能已是 None(进入失败/已关)——显式判空,别让 finally 崩。
|
|
193
|
+
if self.acquired and getattr(self, "_f", None) is not None:
|
|
160
194
|
if IS_WIN:
|
|
161
195
|
self._f.seek(0)
|
|
162
196
|
msvcrt.locking(self._f.fileno(), msvcrt.LK_UNLCK, 1)
|
|
163
197
|
else:
|
|
164
198
|
fcntl.flock(self._f.fileno(), fcntl.LOCK_UN)
|
|
165
|
-
except OSError:
|
|
199
|
+
except (OSError, ValueError):
|
|
166
200
|
pass
|
|
167
201
|
finally:
|
|
168
|
-
|
|
169
|
-
self._f.close()
|
|
170
|
-
self._f = None
|
|
171
|
-
self.acquired = False
|
|
202
|
+
self._close_fd() # 单点关闭(幂等,绝不抛)
|
|
172
203
|
|
|
173
204
|
|
|
174
205
|
# 生效条件:作为 `with` 的空上下文——进入返回自身、退出返回 False(不吞异常)。
|
|
@@ -968,6 +999,36 @@ def ensure_shard_dir(directory: str, where: str = "append") -> bool:
|
|
|
968
999
|
|
|
969
1000
|
|
|
970
1001
|
# 生效条件:directory 经 abspath 存入 self.dir 后经 ensure_shard_dir(self.dir, "init") 保证目录存在(首建不记账;失败抛 ShardDirError 而非裸 OSError),实例分片文件名由 os.getpid() 与 uuid.uuid4().hex[:8] 拼成 "{pid}-{hex8}.log",append 时各写者只写自己这一分片,从而不共享写入点。
|
|
1002
|
+
#: 增量游标回看最后一个换行的**有界**扫描窗口,以及总上限。
|
|
1003
|
+
#: 窗口内找不到换行(超长行)时退一个窗口继续;累计到上限仍未找到 ⇒ 记 0
|
|
1004
|
+
#: (安全方向:下次整片重读,重放幂等,不会漏记录)。
|
|
1005
|
+
_CURSOR_CHUNK = 64 * 1024
|
|
1006
|
+
_CURSOR_SCAN_CAP = 8 * 1024 * 1024
|
|
1007
|
+
|
|
1008
|
+
|
|
1009
|
+
# 生效条件:path 存在且 size>0 时,从 size 起向前回看找最后一个 "\n" 并返回其字节后一位(即已提交行边界);回看到 _CURSOR_SCAN_CAP 仍未找到、或读失败时返回 0(全片重读,安全方向)。
|
|
1010
|
+
def _last_newline_boundary(path: str, size: int) -> int:
|
|
1011
|
+
"""分片「已提交行边界」字节水位(只认完整行,与 read_jsonl_tail 同口径)。"""
|
|
1012
|
+
scanned = 0
|
|
1013
|
+
end = size
|
|
1014
|
+
try:
|
|
1015
|
+
with open(path, "rb") as f:
|
|
1016
|
+
while end > 0 and scanned < _CURSOR_SCAN_CAP:
|
|
1017
|
+
start = max(0, end - _CURSOR_CHUNK)
|
|
1018
|
+
f.seek(start)
|
|
1019
|
+
buf = f.read(end - start)
|
|
1020
|
+
if not buf:
|
|
1021
|
+
break
|
|
1022
|
+
i = buf.rfind(b"\n")
|
|
1023
|
+
if i >= 0:
|
|
1024
|
+
return start + i + 1
|
|
1025
|
+
scanned += len(buf)
|
|
1026
|
+
end = start
|
|
1027
|
+
except OSError:
|
|
1028
|
+
return 0
|
|
1029
|
+
return 0
|
|
1030
|
+
|
|
1031
|
+
|
|
971
1032
|
class ShardedLog:
|
|
972
1033
|
"""每写者独占一个分片的 append-only 日志——不能丢记录时用它。
|
|
973
1034
|
|
|
@@ -1006,6 +1067,30 @@ class ShardedLog:
|
|
|
1006
1067
|
separators=(",", ":")) + "\n")
|
|
1007
1068
|
self._fh.flush()
|
|
1008
1069
|
|
|
1070
|
+
# 生效条件:records 为 list[dict](空列表直接返回 0);句柄按需打开,**一次 write + 一次 flush**
|
|
1071
|
+
# (#677 C 根治:把「每条 flush」降为「每批一次」——锁内 IO 次数从 O(N) 降为 O(1),
|
|
1072
|
+
# 而整批仍在调用方的同一个 FileLock 内,与 compact 的 read→clear 互斥语义不变)。
|
|
1073
|
+
def append_many(self, records):
|
|
1074
|
+
recs = list(records or [])
|
|
1075
|
+
if not recs:
|
|
1076
|
+
return 0
|
|
1077
|
+
if self._fh is None:
|
|
1078
|
+
ensure_shard_dir(self.dir, "append")
|
|
1079
|
+
self._fh = open(self.path, "a", encoding="utf-8", newline="\n")
|
|
1080
|
+
# #677 C 复核:上次崩溃可能留下**残尾**(末字节不是换行)。不补一个换行就追加,
|
|
1081
|
+
# 新批第 1 条会与残尾粘成一行——语法完整但解析失败,重放游标照样推进 ⇒
|
|
1082
|
+
# 残尾与那条好记录一起永久丢失(与 append_jsonl 的 ends_mid_line 同一防线)。
|
|
1083
|
+
if ends_mid_line(self.path):
|
|
1084
|
+
self._fh.write("\n")
|
|
1085
|
+
out = []
|
|
1086
|
+
for record in recs:
|
|
1087
|
+
self._seq += 1
|
|
1088
|
+
out.append(json.dumps(dict(record, _t=time.time(), _s=self._seq),
|
|
1089
|
+
ensure_ascii=False, separators=(",", ":")))
|
|
1090
|
+
self._fh.write("\n".join(out) + "\n")
|
|
1091
|
+
self._fh.flush()
|
|
1092
|
+
return len(out)
|
|
1093
|
+
|
|
1009
1094
|
# 生效条件:幂等;self._fh 为真值时 flush 并关闭句柄、再把 self._fh 置 None,为 None 时直接返回不报错;
|
|
1010
1095
|
def close(self):
|
|
1011
1096
|
if self._fh:
|
|
@@ -1087,6 +1172,111 @@ class ShardedLog:
|
|
|
1087
1172
|
recs.sort(key=lambda r: (_norm(r.get("_t", 0)), _norm(r.get("_s", 0))))
|
|
1088
1173
|
return recs
|
|
1089
1174
|
|
|
1175
|
+
@staticmethod
|
|
1176
|
+
# 生效条件:directory 为目录时返回 dict(分片文件名 -> 该文件当前**已提交行边界**的字节水位);文件不存在/不可读的条目跳过;目录不可列时返回 {}。
|
|
1177
|
+
def committed_cursor(directory: str):
|
|
1178
|
+
"""各分片**已提交行边界**的字节水位(增量重载的基线游标)。
|
|
1179
|
+
|
|
1180
|
+
只需知道「整行到哪里为止」,不必解析内容。做法:从文件尾回看一个
|
|
1181
|
+
有界窗口找最后一个 `\\n`;窗口内找不到(超长行)则退一个窗口继续,
|
|
1182
|
+
最多回看 `_CURSOR_SCAN_CAP` 字节,仍未找到就记 0——**记 0 是安全方向**:
|
|
1183
|
+
下次增量会重读整片,重放幂等,只是慢一点,不会漏记录。
|
|
1184
|
+
|
|
1185
|
+
调用方的使用约束(**安全方向**):本函数返回的水位必须在
|
|
1186
|
+
`read_all()` **之前**取。取在前 ⇒ 水位 ⊆ 实际被消费的记录 ⇒ 下次
|
|
1187
|
+
至多重放已应用过的记录(幂等,last-wins)。取在后则可能把「尚未
|
|
1188
|
+
消费的记录」也算进水位,造成**漏重放**(静默丢写入)。
|
|
1189
|
+
"""
|
|
1190
|
+
out = {}
|
|
1191
|
+
if not os.path.isdir(directory):
|
|
1192
|
+
return out
|
|
1193
|
+
try:
|
|
1194
|
+
names = sorted(os.listdir(directory))
|
|
1195
|
+
except OSError:
|
|
1196
|
+
return out
|
|
1197
|
+
for fn in names:
|
|
1198
|
+
if not fn.endswith(".log"):
|
|
1199
|
+
continue
|
|
1200
|
+
p = os.path.join(directory, fn)
|
|
1201
|
+
try:
|
|
1202
|
+
size = os.stat(p).st_size
|
|
1203
|
+
except OSError:
|
|
1204
|
+
continue
|
|
1205
|
+
if size == 0:
|
|
1206
|
+
out[fn] = 0
|
|
1207
|
+
continue
|
|
1208
|
+
out[fn] = _last_newline_boundary(p, size)
|
|
1209
|
+
return out
|
|
1210
|
+
|
|
1211
|
+
@staticmethod
|
|
1212
|
+
# 生效条件:directory 为目录且 cursor 为 dict(分片文件名 -> 已消费字节水位)时,逐分片读**新增尾部**并按 (_t,_s) 排序返回 (记录列表, 新游标);游标里的分片已消失(被 compact 清走)、或某分片当前大小小于其水位(被截断/重建)时返回 (None, None) 表示**不可增量**,调用方须回退全量重放;directory 不是目录时返回 ([], {})。
|
|
1213
|
+
def read_tails(directory: str, cursor: dict):
|
|
1214
|
+
"""读分片日志的**新增尾部**(issue #766 增量化:W3 读侧重载成本)。
|
|
1215
|
+
|
|
1216
|
+
为什么必须有它:`flush()` 只 append 分片、**从不清**;只有
|
|
1217
|
+
`close()`→`compact_index` 才 `ShardedLog.clear`。故长期存活写方
|
|
1218
|
+
(MCP serve 本体即 `autoflush=1 只 flush 不 close`)的分片日志
|
|
1219
|
+
**只增不减**,而 `read_all` 每次都把全部记录重读重解析一遍 ⇒
|
|
1220
|
+
单次重载成本随「该库写入总量」无界增长(实测 300 节点、日志
|
|
1221
|
+
2000→12000 条,单次 `_load_index` 65→258 ms,斜率约 18ms/千条)。
|
|
1222
|
+
增量后该成本降为 O(新增)。
|
|
1223
|
+
|
|
1224
|
+
**正确性边界(诚实声明)**:本函数只负责「读到并排序新增记录」,
|
|
1225
|
+
**不判**这些记录能否安全叠加上已有物化态——排序水位判定、快照
|
|
1226
|
+
是否变过、盘面是否被非合作写者改过,全在调用方(`MdCG._load_index`)。
|
|
1227
|
+
返回 `(None, None)` 的三类情形即「无法证明增量正确」,一律回退。
|
|
1228
|
+
|
|
1229
|
+
`read_jsonl_tail` 只认**完整行**(末行无换行属未提交,留在水位内
|
|
1230
|
+
下次再读),故并发写者正写到一半时不会读到半截 JSON,也不会把
|
|
1231
|
+
水位推过它——与 `append_jsonl` 的 ends_mid_line 同一防线。
|
|
1232
|
+
"""
|
|
1233
|
+
if not os.path.isdir(directory):
|
|
1234
|
+
return [], {}
|
|
1235
|
+
|
|
1236
|
+
def _norm(v):
|
|
1237
|
+
return v if (isinstance(v, (int, float)) and v == v) else 0
|
|
1238
|
+
|
|
1239
|
+
try:
|
|
1240
|
+
names = sorted(os.listdir(directory))
|
|
1241
|
+
except OSError:
|
|
1242
|
+
return None, None
|
|
1243
|
+
|
|
1244
|
+
present = set()
|
|
1245
|
+
recs, new_cursor = [], {}
|
|
1246
|
+
for fn in names:
|
|
1247
|
+
if not fn.endswith(".log"):
|
|
1248
|
+
continue
|
|
1249
|
+
present.add(fn)
|
|
1250
|
+
p = os.path.join(directory, fn)
|
|
1251
|
+
try:
|
|
1252
|
+
size = os.stat(p).st_size
|
|
1253
|
+
except OSError:
|
|
1254
|
+
continue # delete-pending:视同消失,交由下面的校验判
|
|
1255
|
+
off = cursor.get(fn, 0)
|
|
1256
|
+
if size < off:
|
|
1257
|
+
# 分片被截断/重建:水位之后的内容已不可信,全量重放
|
|
1258
|
+
return None, None
|
|
1259
|
+
if size == off:
|
|
1260
|
+
new_cursor[fn] = off
|
|
1261
|
+
continue
|
|
1262
|
+
batch, nxt = read_jsonl_tail(p, off)
|
|
1263
|
+
if nxt is None:
|
|
1264
|
+
return None, None
|
|
1265
|
+
new_cursor[fn] = nxt
|
|
1266
|
+
recs.extend(batch)
|
|
1267
|
+
|
|
1268
|
+
# 游标里记着、盘上已没有的分片:只可能是 compact 清走了(记录已并入
|
|
1269
|
+
# 快照)⇒ 快照必已重写、物化态作废,不可增量。
|
|
1270
|
+
if set(cursor) - present:
|
|
1271
|
+
return None, None
|
|
1272
|
+
|
|
1273
|
+
bad = [r for r in recs if not isinstance(r, dict)]
|
|
1274
|
+
if bad:
|
|
1275
|
+
note_nonobject_rows([(directory, 0, r) for r in bad])
|
|
1276
|
+
recs = [r for r in recs if isinstance(r, dict)]
|
|
1277
|
+
recs.sort(key=lambda r: (_norm(r.get("_t", 0)), _norm(r.get("_s", 0))))
|
|
1278
|
+
return recs, new_cursor
|
|
1279
|
+
|
|
1090
1280
|
@staticmethod
|
|
1091
1281
|
# 生效条件:directory 是目录时,遍历 os.listdir(directory),对以 ".log" 结尾且不满足「keep 为真值且 os.path.abspath(p) == keep」的条目调用 os.remove(keep 为 None/空串等假值时该排除条件恒不成立,所有 ".log" 条目都会被删),删除时的 OSError 被忽略;directory 不是目录时直接返回。
|
|
1092
1282
|
def clear(directory: str, keep: str = None):
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""灵枢 1.0 治理层(govern)——W1 骨架,无实现。
|
|
3
|
+
|
|
4
|
+
真源:docs/architecture/灵枢1.0_建设路线_v0.1.md §2 / §5 M4。
|
|
5
|
+
|
|
6
|
+
治理层职责(路线稿 §2):
|
|
7
|
+
生命周期四档 · 审计台账 · 回滚/2PC · 资格判定(DEFER/BLINDSPOT)
|
|
8
|
+
|
|
9
|
+
**本包为 W1 骨架(M1)**:只放类型与协议。M4 才把 revise/retire/rollback
|
|
10
|
+
真正切换过来;届时 `rollback` 必须**覆盖正文**——该能力已由
|
|
11
|
+
`md_cg/rollback.py` 实现(`method=preimage`,逐字节回前像),M4 的验收是
|
|
12
|
+
**回归保持全绿**,不是重新实现(详见路线稿 §4 注)。
|
|
13
|
+
|
|
14
|
+
与内核层的关系:治理层可依赖内核层;内核层**不得**反向依赖本包。
|
|
15
|
+
打分层不得输出 ``state``(#111 既定契约)——`state` 只由治理层给出。
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
__all__ = ["GOVERN_LAYER", "LIFECYCLE_STATES"]
|
|
19
|
+
|
|
20
|
+
GOVERN_LAYER = "govern"
|
|
21
|
+
|
|
22
|
+
#: 生命周期四档(路线稿 §2 的「生命周期四档」;具体语义在 M4 落定)。
|
|
23
|
+
LIFECYCLE_STATES = ("active", "dormant", "retired", "tombstoned")
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""灵枢 1.0 接口层(iface)——四个契约的唯一出口。
|
|
3
|
+
|
|
4
|
+
真源:docs/architecture/灵枢1.0_建设路线_v0.1.md §3。
|
|
5
|
+
|
|
6
|
+
适配层(dsh 插件 / MCP server / hive worker)**只允许**从本包导入契约类型;
|
|
7
|
+
不得直接 import 内核层或治理层的实现模块(路线稿 §2 禁止项)。
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from .contracts import (
|
|
11
|
+
CONTRACT_NAMES,
|
|
12
|
+
IFACE_LAYER,
|
|
13
|
+
ContractViolation,
|
|
14
|
+
GovernContract,
|
|
15
|
+
ReadContract,
|
|
16
|
+
ReadItem,
|
|
17
|
+
ReadRequest,
|
|
18
|
+
RecordState,
|
|
19
|
+
SelfCheckContract,
|
|
20
|
+
SelfCheckReport,
|
|
21
|
+
WriteContract,
|
|
22
|
+
WriteReceipt,
|
|
23
|
+
WriteRequest,
|
|
24
|
+
reject_state_in_scoring,
|
|
25
|
+
require_provenance,
|
|
26
|
+
require_why,
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
__all__ = [
|
|
30
|
+
"CONTRACT_NAMES",
|
|
31
|
+
"IFACE_LAYER",
|
|
32
|
+
"ContractViolation",
|
|
33
|
+
"WriteContract",
|
|
34
|
+
"ReadContract",
|
|
35
|
+
"GovernContract",
|
|
36
|
+
"SelfCheckContract",
|
|
37
|
+
"WriteRequest",
|
|
38
|
+
"WriteReceipt",
|
|
39
|
+
"ReadRequest",
|
|
40
|
+
"ReadItem",
|
|
41
|
+
"SelfCheckReport",
|
|
42
|
+
"RecordState",
|
|
43
|
+
"require_provenance",
|
|
44
|
+
"require_why",
|
|
45
|
+
"reject_state_in_scoring",
|
|
46
|
+
]
|