cctally 1.115.0 → 1.116.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/bin/_cctally_cache.py +85 -4
- package/bin/_cctally_completion.py +80 -12
- package/bin/_cctally_core.py +106 -9
- package/bin/_cctally_dashboard.py +6 -9
- package/bin/_cctally_dashboard_envelope.py +26 -32
- package/bin/_cctally_db.py +85 -4
- package/bin/_cctally_doctor.py +38 -5
- package/bin/_cctally_five_hour.py +83 -1017
- package/bin/_cctally_journal.py +1307 -180
- package/bin/_cctally_journal_repair.py +9 -7
- package/bin/_cctally_oauth_samples.py +280 -0
- package/bin/_cctally_parser.py +5 -0
- package/bin/_cctally_quarantine.py +14 -4
- package/bin/_cctally_quota.py +70 -8
- package/bin/_cctally_quota_model.py +35 -4
- package/bin/_cctally_record.py +414 -123
- package/bin/_cctally_rederive.py +632 -235
- package/bin/_cctally_refresh.py +29 -0
- package/bin/_cctally_statusline.py +984 -47
- package/bin/_cctally_store.py +972 -58
- package/bin/_cctally_tui.py +62 -7
- package/bin/_cctally_update.py +25 -0
- package/bin/_lib_account_protocol.py +201 -18
- package/bin/_lib_clone_copy.py +46 -11
- package/bin/_lib_dashboard_sources.py +57 -15
- package/bin/_lib_detector_access_registry.py +273 -52
- package/bin/_lib_doctor.py +302 -1
- package/bin/_lib_journal.py +145 -25
- package/bin/_lib_journal_router.py +166 -48
- package/bin/_lib_oauth_samples.py +238 -0
- package/bin/_lib_pricing_check.py +0 -12
- package/bin/_lib_provider_activity.py +770 -0
- package/bin/_lib_quarantine.py +3 -2
- package/bin/_lib_quota_model.py +96 -6
- package/bin/_lib_rebuild_progress.py +146 -0
- package/bin/_lib_record.py +22 -0
- package/bin/_lib_rederive.py +299 -3
- package/bin/_lib_snapshot_cache.py +43 -1
- package/bin/_lib_stale_proof.py +708 -0
- package/bin/_lib_statusline_candidates.py +65 -5
- package/bin/_lib_worker_launches.py +315 -0
- package/bin/cctally +6 -11
- package/dashboard/static/assets/{ConversationsView-Bc8LCOAZ.js → ConversationsView-Dflb_myA.js} +1 -1
- package/dashboard/static/assets/DoctorModal-DFX5CfIs.js +1 -0
- package/dashboard/static/assets/{ModalRoot-DLPgyAM_.js → ModalRoot-HlxoagIa.js} +1 -1
- package/dashboard/static/assets/{ProjectsDrillPanel-z53a4v4M.js → ProjectsDrillPanel-C3PJy1xx.js} +1 -1
- package/dashboard/static/assets/{SourceDetailModal-Bw0T8W-W.js → SourceDetailModal-BczsWKhD.js} +1 -1
- package/dashboard/static/assets/{UpdateModal-D5AKx4Yw.js → UpdateModal-Os-Lo5cs.js} +1 -1
- package/dashboard/static/assets/{index-BXDOzSng.js → index-CYZ3NLqF.js} +3 -3
- package/dashboard/static/assets/{outlineNavigation-JObY7cUY.js → outlineNavigation-k-zIif6W.js} +1 -1
- package/dashboard/static/dashboard.html +1 -1
- package/package.json +7 -2
- package/bin/_lib_merge_gate.py +0 -452
- package/dashboard/static/assets/DoctorModal-D--qVBwe.js +0 -1
package/bin/_cctally_journal.py
CHANGED
|
@@ -40,6 +40,7 @@ import sqlite3
|
|
|
40
40
|
import stat
|
|
41
41
|
import sys
|
|
42
42
|
import time
|
|
43
|
+
from array import array
|
|
43
44
|
from dataclasses import dataclass, field, replace as _dc_replace
|
|
44
45
|
|
|
45
46
|
import _cctally_core
|
|
@@ -50,6 +51,7 @@ import _lib_journal
|
|
|
50
51
|
import _lib_journal_router
|
|
51
52
|
import _lib_legacy_spool
|
|
52
53
|
import _lib_legacy_writer_evidence
|
|
54
|
+
import _lib_rebuild_progress
|
|
53
55
|
import _lib_record
|
|
54
56
|
import _lib_segment_summary
|
|
55
57
|
import _lib_selector_state
|
|
@@ -512,12 +514,15 @@ def _require_account_bound_cutover(now_utc=None) -> None:
|
|
|
512
514
|
boundary = ensure_account_bound_cutover(now_utc=now_utc)
|
|
513
515
|
except AccountBoundCutoverDeferred:
|
|
514
516
|
return
|
|
517
|
+
# #797/#823/#840 (spec 4.6 B2): the memo names the LAST boundary op —
|
|
518
|
+
# the revision op — so a memo hit proves both fences exist.
|
|
519
|
+
last = boundary.revision if boundary.revision is not None else boundary
|
|
515
520
|
try:
|
|
516
|
-
st = os.stat(journal_dir /
|
|
521
|
+
st = os.stat(journal_dir / last.segment)
|
|
517
522
|
except OSError:
|
|
518
523
|
return
|
|
519
524
|
_ACCOUNT_BOUND_CUTOVER_MEMO = (
|
|
520
|
-
str(journal_dir),
|
|
525
|
+
str(journal_dir), last.segment, int(last.end_offset),
|
|
521
526
|
(st.st_dev, st.st_ino))
|
|
522
527
|
|
|
523
528
|
|
|
@@ -1077,6 +1082,136 @@ def plan_segment_elision(segments, high_water):
|
|
|
1077
1082
|
summaries=summaries, covered=covered, verdict=reason)
|
|
1078
1083
|
|
|
1079
1084
|
|
|
1085
|
+
class QuotaLineSourceError(Exception):
|
|
1086
|
+
"""A coordinate no longer names the line the read pass saw (#997 D7)."""
|
|
1087
|
+
|
|
1088
|
+
|
|
1089
|
+
class QuotaLineSource:
|
|
1090
|
+
"""The final rebuild's Codex quota observations, as coordinates (#997 D7).
|
|
1091
|
+
|
|
1092
|
+
The read pass used to retain every observation's raw line until the cache
|
|
1093
|
+
leg ran (1.64 GB on the maintainer's journal, alive across the whole pass).
|
|
1094
|
+
This keeps ``(segment, offset, length)`` instead, in per-segment
|
|
1095
|
+
``array('Q')`` runs, which is 16 bytes per observation, and re-reads exactly
|
|
1096
|
+
the slice the leg asks for. ``_run_bounded_recovery`` therefore keeps its
|
|
1097
|
+
contract: chunk planning by index from ``lengths()`` (no read), a restart
|
|
1098
|
+
at chunk zero re-reads from disk, and only one chunk's bytes are resident.
|
|
1099
|
+
|
|
1100
|
+
``offset`` is the line's start and ``length`` excludes its newline, the
|
|
1101
|
+
shape ``_iter_segment_lines`` yields. Every read stays inside the extent
|
|
1102
|
+
the pass pinned, because it reads only bytes the pass itself saw.
|
|
1103
|
+
"""
|
|
1104
|
+
|
|
1105
|
+
__slots__ = ("_runs", "_count")
|
|
1106
|
+
|
|
1107
|
+
def __init__(self) -> None:
|
|
1108
|
+
#: ``[segment, offsets, lengths]`` per run, in journal order. A run
|
|
1109
|
+
#: holds one segment's consecutive observations; a segment can only
|
|
1110
|
+
#: appear in two runs when the elided refill splices another between.
|
|
1111
|
+
self._runs: list = []
|
|
1112
|
+
self._count = 0
|
|
1113
|
+
|
|
1114
|
+
def append(self, segment: str, offset: int, length: int) -> None:
|
|
1115
|
+
runs = self._runs
|
|
1116
|
+
if not runs or runs[-1][0] != segment:
|
|
1117
|
+
runs.append([segment, array("Q"), array("Q")])
|
|
1118
|
+
runs[-1][1].append(int(offset))
|
|
1119
|
+
runs[-1][2].append(int(length))
|
|
1120
|
+
self._count += 1
|
|
1121
|
+
|
|
1122
|
+
def _extend_run(self, segment, offsets, lengths) -> None:
|
|
1123
|
+
if not offsets:
|
|
1124
|
+
return
|
|
1125
|
+
runs = self._runs
|
|
1126
|
+
if not runs or runs[-1][0] != segment:
|
|
1127
|
+
runs.append([segment, array("Q"), array("Q")])
|
|
1128
|
+
runs[-1][1].extend(offsets)
|
|
1129
|
+
runs[-1][2].extend(lengths)
|
|
1130
|
+
self._count += len(offsets)
|
|
1131
|
+
|
|
1132
|
+
def _copy_range(self, source: "QuotaLineSource", start: int,
|
|
1133
|
+
stop: int) -> None:
|
|
1134
|
+
"""Append ``source``'s observations ``[start, stop)`` to this one."""
|
|
1135
|
+
base = 0
|
|
1136
|
+
for segment, offsets, lengths in source._runs:
|
|
1137
|
+
size = len(offsets)
|
|
1138
|
+
lo = max(start - base, 0)
|
|
1139
|
+
hi = min(stop - base, size)
|
|
1140
|
+
if lo < hi:
|
|
1141
|
+
self._extend_run(segment, offsets[lo:hi], lengths[lo:hi])
|
|
1142
|
+
base += size
|
|
1143
|
+
if base >= stop:
|
|
1144
|
+
break
|
|
1145
|
+
|
|
1146
|
+
def __len__(self) -> int:
|
|
1147
|
+
return self._count
|
|
1148
|
+
|
|
1149
|
+
def lengths(self) -> "array":
|
|
1150
|
+
"""Every observation's line length (without its newline), no read."""
|
|
1151
|
+
out = array("Q")
|
|
1152
|
+
for _segment, _offsets, lengths in self._runs:
|
|
1153
|
+
out.extend(lengths)
|
|
1154
|
+
return out
|
|
1155
|
+
|
|
1156
|
+
def lines(self, start: int, stop: int) -> list:
|
|
1157
|
+
"""Observations ``[start, stop)`` re-read from their segments.
|
|
1158
|
+
|
|
1159
|
+
Raises `QuotaLineSourceError` when a slice is short or no longer framed
|
|
1160
|
+
as one line (a newline before it, unless it starts the segment; one
|
|
1161
|
+
after it; none inside), which is what a truncated or rewritten segment
|
|
1162
|
+
produces. The leg treats that exactly as an unreadable elided segment.
|
|
1163
|
+
"""
|
|
1164
|
+
out: list = []
|
|
1165
|
+
base = 0
|
|
1166
|
+
for segment, offsets, lengths in self._runs:
|
|
1167
|
+
size = len(offsets)
|
|
1168
|
+
lo = max(start - base, 0)
|
|
1169
|
+
hi = min(stop - base, size)
|
|
1170
|
+
if lo < hi:
|
|
1171
|
+
path = _cctally_core.JOURNAL_DIR / segment
|
|
1172
|
+
try:
|
|
1173
|
+
with _open_segment_for_read(path) as fh:
|
|
1174
|
+
for index in range(lo, hi):
|
|
1175
|
+
out.append(_read_framed_line(
|
|
1176
|
+
fh, segment, offsets[index], lengths[index]))
|
|
1177
|
+
except OSError as exc:
|
|
1178
|
+
raise QuotaLineSourceError(
|
|
1179
|
+
f"{segment}: {type(exc).__name__}: {exc}") from exc
|
|
1180
|
+
base += size
|
|
1181
|
+
if base >= stop:
|
|
1182
|
+
break
|
|
1183
|
+
return out
|
|
1184
|
+
|
|
1185
|
+
|
|
1186
|
+
def _read_framed_line(fh, segment, offset, length) -> bytes:
|
|
1187
|
+
"""The ``length`` bytes at ``offset``, verified to still be one line."""
|
|
1188
|
+
lead = 1 if offset > 0 else 0
|
|
1189
|
+
fh.seek(offset - lead)
|
|
1190
|
+
data = fh.read(length + lead + 1)
|
|
1191
|
+
if (len(data) != length + lead + 1
|
|
1192
|
+
or (lead and data[0:1] != b"\n")
|
|
1193
|
+
or data[-1:] != b"\n"
|
|
1194
|
+
or b"\n" in data[lead:lead + length]):
|
|
1195
|
+
raise QuotaLineSourceError(
|
|
1196
|
+
f"{segment}@{offset}: the line the read pass saw is no longer "
|
|
1197
|
+
"there")
|
|
1198
|
+
return data[lead:lead + length]
|
|
1199
|
+
|
|
1200
|
+
|
|
1201
|
+
def _quota_line_sizes(quota_raw):
|
|
1202
|
+
"""Encoded size per observation (line plus newline), for chunk planning."""
|
|
1203
|
+
if isinstance(quota_raw, QuotaLineSource):
|
|
1204
|
+
return array("Q", (length + 1 for length in quota_raw.lengths()))
|
|
1205
|
+
return [len(raw) + 1 for raw in quota_raw]
|
|
1206
|
+
|
|
1207
|
+
|
|
1208
|
+
def _quota_lines(quota_raw, start: int, stop: int) -> list:
|
|
1209
|
+
"""Observations ``[start, stop)`` as raw lines, from either input form."""
|
|
1210
|
+
if isinstance(quota_raw, QuotaLineSource):
|
|
1211
|
+
return quota_raw.lines(start, stop)
|
|
1212
|
+
return quota_raw[start:stop]
|
|
1213
|
+
|
|
1214
|
+
|
|
1080
1215
|
def _refill_elided_quota_raw(quota_raw, gaps, *, failures=None):
|
|
1081
1216
|
"""``(stream, complete)`` — ``quota_raw`` with the elided observations back.
|
|
1082
1217
|
|
|
@@ -1110,20 +1245,26 @@ def _refill_elided_quota_raw(quota_raw, gaps, *, failures=None):
|
|
|
1110
1245
|
a re-read that returns a different number reports the shortfall rather than
|
|
1111
1246
|
leaving the leg to certify a stream it never saw.
|
|
1112
1247
|
"""
|
|
1248
|
+
coordinates = isinstance(quota_raw, QuotaLineSource)
|
|
1113
1249
|
by_index: dict = {}
|
|
1114
1250
|
complete = True
|
|
1115
1251
|
if failures is None:
|
|
1116
1252
|
failures = []
|
|
1117
1253
|
for name, index, extent, expected_lines in gaps:
|
|
1118
|
-
recovered
|
|
1254
|
+
# #997 D7: a coordinate source gets each recovered observation's
|
|
1255
|
+
# coordinates, not its bytes, so the refill keeps nothing either.
|
|
1256
|
+
recovered = QuotaLineSource() if coordinates else []
|
|
1119
1257
|
seen = 0
|
|
1120
1258
|
try:
|
|
1121
|
-
for
|
|
1259
|
+
for seg, off, raw in _iter_segment_lines(
|
|
1122
1260
|
_cctally_core.JOURNAL_DIR / name, 0, int(extent)):
|
|
1123
1261
|
seen += 1
|
|
1124
1262
|
record = _lib_journal.decode_line(raw)
|
|
1125
1263
|
if record is not None and _is_codex_quota_obs(record):
|
|
1126
|
-
|
|
1264
|
+
if coordinates:
|
|
1265
|
+
recovered.append(seg, off, len(raw))
|
|
1266
|
+
else:
|
|
1267
|
+
recovered.append(raw)
|
|
1127
1268
|
except Exception as exc:
|
|
1128
1269
|
# A vanished segment self-heals, because the pinned vector no longer
|
|
1129
1270
|
# matches the journal and the certificate is invalid the moment it
|
|
@@ -1144,13 +1285,25 @@ def _refill_elided_quota_raw(quota_raw, gaps, *, failures=None):
|
|
|
1144
1285
|
continue
|
|
1145
1286
|
if seen != int(expected_lines):
|
|
1146
1287
|
complete = False
|
|
1147
|
-
by_index.setdefault(int(index), []).
|
|
1288
|
+
by_index.setdefault(int(index), []).append(recovered)
|
|
1289
|
+
if coordinates:
|
|
1290
|
+
spliced = QuotaLineSource()
|
|
1291
|
+
cursor = 0
|
|
1292
|
+
for index in sorted(by_index):
|
|
1293
|
+
bounded = min(int(index), len(quota_raw))
|
|
1294
|
+
spliced._copy_range(quota_raw, cursor, bounded)
|
|
1295
|
+
for recovered in by_index[index]:
|
|
1296
|
+
spliced._copy_range(recovered, 0, len(recovered))
|
|
1297
|
+
cursor = max(cursor, bounded)
|
|
1298
|
+
spliced._copy_range(quota_raw, cursor, len(quota_raw))
|
|
1299
|
+
return spliced, complete
|
|
1148
1300
|
out: list = []
|
|
1149
1301
|
cursor = 0
|
|
1150
1302
|
for index in sorted(by_index):
|
|
1151
1303
|
bounded = min(int(index), len(quota_raw))
|
|
1152
1304
|
out.extend(quota_raw[cursor:bounded])
|
|
1153
|
-
|
|
1305
|
+
for recovered in by_index[index]:
|
|
1306
|
+
out.extend(recovered)
|
|
1154
1307
|
cursor = max(cursor, bounded)
|
|
1155
1308
|
out.extend(quota_raw[cursor:])
|
|
1156
1309
|
return out, complete
|
|
@@ -1770,16 +1923,26 @@ def _acquire_ingest_lock(mode: str, timeout_s: float) -> int | None:
|
|
|
1770
1923
|
# busy-timeout branch closes + returns before this handler can fire, and a
|
|
1771
1924
|
# successful acquire returns fd, so the fd is closed exactly once.
|
|
1772
1925
|
try:
|
|
1773
|
-
|
|
1774
|
-
|
|
1775
|
-
|
|
1776
|
-
|
|
1777
|
-
|
|
1778
|
-
|
|
1779
|
-
|
|
1780
|
-
|
|
1781
|
-
|
|
1782
|
-
|
|
1926
|
+
try:
|
|
1927
|
+
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
1928
|
+
return fd
|
|
1929
|
+
except (BlockingIOError, OSError) as exc:
|
|
1930
|
+
if time.monotonic() >= deadline:
|
|
1931
|
+
_note_lock_failure(exc)
|
|
1932
|
+
os.close(fd)
|
|
1933
|
+
return None
|
|
1934
|
+
# #997 D9: contended, so this is a wait a worker reports.
|
|
1935
|
+
with _lib_rebuild_progress.waiting("ingest-lock"):
|
|
1936
|
+
while True:
|
|
1937
|
+
try:
|
|
1938
|
+
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
1939
|
+
return fd
|
|
1940
|
+
except (BlockingIOError, OSError) as exc:
|
|
1941
|
+
if time.monotonic() >= deadline:
|
|
1942
|
+
_note_lock_failure(exc)
|
|
1943
|
+
os.close(fd)
|
|
1944
|
+
return None
|
|
1945
|
+
time.sleep(0.02)
|
|
1783
1946
|
except BaseException:
|
|
1784
1947
|
os.close(fd)
|
|
1785
1948
|
raise
|
|
@@ -1811,17 +1974,28 @@ def _acquire_maintenance_shared(mode: str, timeout_s: float) -> int | None:
|
|
|
1811
1974
|
return None
|
|
1812
1975
|
deadline = time.monotonic() + timeout_s
|
|
1813
1976
|
try:
|
|
1814
|
-
|
|
1815
|
-
|
|
1816
|
-
|
|
1817
|
-
|
|
1818
|
-
|
|
1819
|
-
|
|
1820
|
-
|
|
1821
|
-
|
|
1822
|
-
|
|
1823
|
-
|
|
1824
|
-
|
|
1977
|
+
try:
|
|
1978
|
+
fcntl.flock(fd, fcntl.LOCK_SH | fcntl.LOCK_NB)
|
|
1979
|
+
_cctally_core.note_stats_maintenance_acquired()
|
|
1980
|
+
return fd
|
|
1981
|
+
except (BlockingIOError, OSError) as exc:
|
|
1982
|
+
if time.monotonic() >= deadline:
|
|
1983
|
+
_note_lock_failure(exc)
|
|
1984
|
+
os.close(fd)
|
|
1985
|
+
return None
|
|
1986
|
+
# #997 D9: contended, so this is a wait a worker reports.
|
|
1987
|
+
with _lib_rebuild_progress.waiting("maintenance-lock"):
|
|
1988
|
+
while True:
|
|
1989
|
+
try:
|
|
1990
|
+
fcntl.flock(fd, fcntl.LOCK_SH | fcntl.LOCK_NB)
|
|
1991
|
+
_cctally_core.note_stats_maintenance_acquired()
|
|
1992
|
+
return fd
|
|
1993
|
+
except (BlockingIOError, OSError) as exc:
|
|
1994
|
+
if time.monotonic() >= deadline:
|
|
1995
|
+
_note_lock_failure(exc)
|
|
1996
|
+
os.close(fd)
|
|
1997
|
+
return None
|
|
1998
|
+
time.sleep(0.02)
|
|
1825
1999
|
except BaseException:
|
|
1826
2000
|
os.close(fd)
|
|
1827
2001
|
raise
|
|
@@ -1846,17 +2020,28 @@ def _acquire_maintenance_exclusive(mode: str, timeout_s: float) -> int | None:
|
|
|
1846
2020
|
return None
|
|
1847
2021
|
deadline = time.monotonic() + timeout_s
|
|
1848
2022
|
try:
|
|
1849
|
-
|
|
1850
|
-
|
|
1851
|
-
|
|
1852
|
-
|
|
1853
|
-
|
|
1854
|
-
|
|
1855
|
-
|
|
1856
|
-
|
|
1857
|
-
|
|
1858
|
-
|
|
1859
|
-
|
|
2023
|
+
try:
|
|
2024
|
+
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
2025
|
+
_cctally_core.note_stats_maintenance_acquired()
|
|
2026
|
+
return fd
|
|
2027
|
+
except (BlockingIOError, OSError) as exc:
|
|
2028
|
+
if time.monotonic() >= deadline:
|
|
2029
|
+
_note_lock_failure(exc)
|
|
2030
|
+
os.close(fd)
|
|
2031
|
+
return None
|
|
2032
|
+
# #997 D9: contended, so this is a wait a worker reports.
|
|
2033
|
+
with _lib_rebuild_progress.waiting("maintenance-lock"):
|
|
2034
|
+
while True:
|
|
2035
|
+
try:
|
|
2036
|
+
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
2037
|
+
_cctally_core.note_stats_maintenance_acquired()
|
|
2038
|
+
return fd
|
|
2039
|
+
except (BlockingIOError, OSError) as exc:
|
|
2040
|
+
if time.monotonic() >= deadline:
|
|
2041
|
+
_note_lock_failure(exc)
|
|
2042
|
+
os.close(fd)
|
|
2043
|
+
return None
|
|
2044
|
+
time.sleep(0.02)
|
|
1860
2045
|
except BaseException:
|
|
1861
2046
|
os.close(fd)
|
|
1862
2047
|
raise
|
|
@@ -1957,8 +2142,11 @@ def _write_cursor(conn: sqlite3.Connection, segment: str, offset: int) -> None:
|
|
|
1957
2142
|
#: #825/#832 fix round A: the first stats index epoch that carries
|
|
1958
2143
|
#: `journal_derivation_frontier`. A frontier read from any earlier epoch — in
|
|
1959
2144
|
#: practice the previous release republishing its own index — is never trusted
|
|
1960
|
-
#: as proof that derivation completed (Codex fix-round-1 Q1).
|
|
1961
|
-
|
|
2145
|
+
#: as proof that derivation completed (Codex fix-round-1 Q1). #797/#823/#840
|
|
2146
|
+
#: (spec 4.6 B6): raised to 1019 with the revision-2 fence, so a completion
|
|
2147
|
+
#: proof an epoch-1018 binary published, which never applied the revision-2
|
|
2148
|
+
#: admission, is never trusted by this binary.
|
|
2149
|
+
DERIVATION_FRONTIER_EPOCH = 1019
|
|
1962
2150
|
|
|
1963
2151
|
#: #825/#832 revision 2 (#946, Codex design-fix-round-3 Q5): the generation of
|
|
1964
2152
|
#: the derivation-completeness proof. Generation 1 (stored as NULL) advanced
|
|
@@ -2184,6 +2372,21 @@ def _open_segment_for_read(seg_path):
|
|
|
2184
2372
|
return open(seg_path, "rb")
|
|
2185
2373
|
|
|
2186
2374
|
|
|
2375
|
+
def _open_segment_by_name(name):
|
|
2376
|
+
"""`_open_segment_for_read` for a segment basename, resolved against the
|
|
2377
|
+
journal directory at call time. The prefix hasher's evidence re-read
|
|
2378
|
+
(#997 D12) comes through here, so it stays visible at the one seam."""
|
|
2379
|
+
return _open_segment_for_read(_cctally_core.JOURNAL_DIR / name)
|
|
2380
|
+
|
|
2381
|
+
|
|
2382
|
+
def _new_prefix_hasher():
|
|
2383
|
+
"""A `PrefixHashAccumulator` whose evidence re-reads go through the
|
|
2384
|
+
segment-read seam (#997 D12). Every streaming pass that captures prefix
|
|
2385
|
+
evidence builds its hasher here."""
|
|
2386
|
+
return _lib_journal_router.PrefixHashAccumulator(
|
|
2387
|
+
open_segment=_open_segment_by_name)
|
|
2388
|
+
|
|
2389
|
+
|
|
2187
2390
|
def _iter_segment_lines(seg_path, lo: int, hi: int, *, on_bytes=None):
|
|
2188
2391
|
"""Stream `(basename, absolute-offset, raw-line-without-newline)` for every
|
|
2189
2392
|
complete line in `[lo, hi)`, holding at most one chunk plus one partial line
|
|
@@ -2255,9 +2458,12 @@ def _iter_range_with_segments(cursor, hw, segments, *, on_segment=None,
|
|
|
2255
2458
|
pinned high-water and drives every pass from it, so two passes of the same
|
|
2256
2459
|
rebuild cannot disagree about the journal's shape.
|
|
2257
2460
|
|
|
2258
|
-
`on_segment` is called for EVERY segment in the range, including
|
|
2259
|
-
function then skips because it holds no bytes in range:
|
|
2260
|
-
frames a zero-byte segment, so a hash accumulator has
|
|
2461
|
+
`on_segment(seg, size)` is called for EVERY segment in the range, including
|
|
2462
|
+
one this function then skips because it holds no bytes in range:
|
|
2463
|
+
`journal_prefix_hash` frames a zero-byte segment, so a hash accumulator has
|
|
2464
|
+
to be told it exists. `size` is `max(hi - lo, 0)`, the exact byte count
|
|
2465
|
+
`on_bytes` will receive for the segment unless it shrinks under the pass;
|
|
2466
|
+
the streamed prefix hash frames the segment header with it (#997 D12).
|
|
2261
2467
|
`on_bytes` is forwarded to `_iter_segment_lines`.
|
|
2262
2468
|
|
|
2263
2469
|
`on_extent(seg, lo, hi, stat_result)` is called after this pass has pinned
|
|
@@ -2311,7 +2517,7 @@ def _iter_range_with_segments(cursor, hw, segments, *, on_segment=None,
|
|
|
2311
2517
|
if on_extent is not None:
|
|
2312
2518
|
on_extent(seg, lo, hi, stat_result)
|
|
2313
2519
|
if on_segment is not None:
|
|
2314
|
-
on_segment(seg)
|
|
2520
|
+
on_segment(seg, max(hi - lo, 0))
|
|
2315
2521
|
if lo >= hi:
|
|
2316
2522
|
continue
|
|
2317
2523
|
yield from _iter_segment_lines(seg_path, lo, hi, on_bytes=on_bytes)
|
|
@@ -4296,6 +4502,43 @@ def fold_policy_of(document, key=FOLD_POLICY_KEY) -> int:
|
|
|
4296
4502
|
return FOLD_POLICY_HISTORICAL
|
|
4297
4503
|
|
|
4298
4504
|
|
|
4505
|
+
# ── Detector policy (#797/#823/#840 spec 4.9, D12) ──────────────────────────
|
|
4506
|
+
#
|
|
4507
|
+
#: The raw Claude usage observation's payload key that records the detector
|
|
4508
|
+
#: revision it was captured under. It is separate from `fold_policy` because
|
|
4509
|
+
#: the previous release's `fold_policy_of` maps every value other than 1 to
|
|
4510
|
+
#: its historical fold: new observations keep `fold_policy: 1` and add this
|
|
4511
|
+
#: additive key, which that release reads past with `.get`. Both detector
|
|
4512
|
+
#: changes of spec 4.9 (the five-hour credit generation in place of the pair
|
|
4513
|
+
#: check, and projection fills that do not advance detector state) are
|
|
4514
|
+
#: versioned together under it.
|
|
4515
|
+
DETECTOR_POLICY_KEY = "detector_policy"
|
|
4516
|
+
#: The same revision on a frozen statusline publication plan, frozen beside
|
|
4517
|
+
#: `foldPolicy` before the publication identity is computed. Recovery copies
|
|
4518
|
+
#: the frozen value or preserves its absence, never the running default.
|
|
4519
|
+
PLAN_DETECTOR_POLICY_KEY = "detectorPolicy"
|
|
4520
|
+
#: The revision newly captured observations and newly frozen plans carry.
|
|
4521
|
+
DETECTOR_POLICY = 2
|
|
4522
|
+
|
|
4523
|
+
|
|
4524
|
+
def detector_policy_of(payload) -> str:
|
|
4525
|
+
"""The detector a raw observation payload selects: ``"historical"`` when
|
|
4526
|
+
it records no `detector_policy` (every observation an earlier release
|
|
4527
|
+
journaled: the pair check and projection-fill detection, so a plain
|
|
4528
|
+
rederive of old history adds or removes no credit), ``"v2"`` for exactly
|
|
4529
|
+
the integer 2, and ``"unsupported"`` for any other value — a bool, a
|
|
4530
|
+
float, a string, null or another revision — which selects neither
|
|
4531
|
+
behavior and withholds that observation's detector step (an outcome may
|
|
4532
|
+
always be withheld). Detector policy is semantic data, never transport
|
|
4533
|
+
metadata."""
|
|
4534
|
+
if not isinstance(payload, dict) or DETECTOR_POLICY_KEY not in payload:
|
|
4535
|
+
return "historical"
|
|
4536
|
+
value = payload[DETECTOR_POLICY_KEY]
|
|
4537
|
+
if type(value) is int and value == DETECTOR_POLICY:
|
|
4538
|
+
return "v2"
|
|
4539
|
+
return "unsupported"
|
|
4540
|
+
|
|
4541
|
+
|
|
4299
4542
|
@dataclass(frozen=True)
|
|
4300
4543
|
class AxisFloor:
|
|
4301
4544
|
"""One axis's floor for EXACTLY one account, resolved on the caller's
|
|
@@ -4916,7 +5159,10 @@ _ACCOUNTS_MACHINERY_KINDS = frozenset(
|
|
|
4916
5159
|
# two strict administrative kinds. None carries a target row to stamp.
|
|
4917
5160
|
"account_bound_cutover", "legacy_spool_activity",
|
|
4918
5161
|
"journal_protocol_resolution_v2",
|
|
4919
|
-
"claude_weekly_observation_decision_v2"
|
|
5162
|
+
"claude_weekly_observation_decision_v2",
|
|
5163
|
+
# #797/#823/#840 spec 4.6 B2, B5: the revision-2 boundary op and the
|
|
5164
|
+
# successor of the repair acknowledgement.
|
|
5165
|
+
"account_bound_revision", "journal_protocol_resolution_v3"))
|
|
4920
5166
|
|
|
4921
5167
|
# Legacy-classifier exhaustiveness guard (#341, review finding P2-1). EVERY evt
|
|
4922
5168
|
# kind in `_EVT_SPECS` and every harvest kind in `_HARVEST_SPECS` must carry a
|
|
@@ -5076,10 +5322,22 @@ def _normalize_legacy_account_stamp(record, claude_legacy_account: str) -> None:
|
|
|
5076
5322
|
# Obs/op fold registry (spec §5.3 "fold op"). Keyed by `payload.kind`; the
|
|
5077
5323
|
# built-in `_pipeline_op_fold` pipeline hook dispatches through it. 6b may
|
|
5078
5324
|
# register more op folds; obs no longer fold directly (see the NOTE above).
|
|
5325
|
+
def _apply_op_weekly_observation_decision(conn, record) -> None:
|
|
5326
|
+
"""#797/#823/#840 spec 4.3: project a reviewed weekly decision op into
|
|
5327
|
+
`claude_weekly_observation_decisions`, the read-time weekly eligibility
|
|
5328
|
+
of provider samples. It writes nothing else: the decision's accounting
|
|
5329
|
+
effect is still the scratch replay's (`db rederive`)."""
|
|
5330
|
+
import _cctally_oauth_samples
|
|
5331
|
+
_cctally_oauth_samples.apply_weekly_observation_decision_op(conn, record)
|
|
5332
|
+
|
|
5333
|
+
|
|
5079
5334
|
FOLD_APPLIERS = {
|
|
5080
5335
|
"weekly_credit_floor": _apply_op_weekly_credit_floor,
|
|
5081
5336
|
"account_observe": _apply_op_account_observe,
|
|
5082
5337
|
"account_label": _apply_op_account_label,
|
|
5338
|
+
"claude_weekly_observation_decision": _apply_op_weekly_observation_decision,
|
|
5339
|
+
"claude_weekly_observation_decision_v2": (
|
|
5340
|
+
_apply_op_weekly_observation_decision),
|
|
5083
5341
|
}
|
|
5084
5342
|
|
|
5085
5343
|
|
|
@@ -5877,6 +6135,16 @@ class ScratchRetention:
|
|
|
5877
6135
|
for event_id in event_ids:
|
|
5878
6136
|
self.applied.setdefault(event_id, _PREDECESSOR_HELD)
|
|
5879
6137
|
|
|
6138
|
+
def applied_records(self):
|
|
6139
|
+
"""The decisions the projection holds at the current replay point, as
|
|
6140
|
+
records (#840): what each id materialized, or for an id the
|
|
6141
|
+
predecessor index holds its effective record; a tombstone is none."""
|
|
6142
|
+
for event_id, held in list(self.applied.items()):
|
|
6143
|
+
record = (self.effective.get(event_id)
|
|
6144
|
+
if held is _PREDECESSOR_HELD else held)
|
|
6145
|
+
if record is not None:
|
|
6146
|
+
yield record
|
|
6147
|
+
|
|
5880
6148
|
def _affected(self, evt) -> bool:
|
|
5881
6149
|
if self.reference is None:
|
|
5882
6150
|
return False
|
|
@@ -5916,7 +6184,8 @@ class ScratchRetention:
|
|
|
5916
6184
|
# The predecessor index holds this id as of an earlier
|
|
5917
6185
|
# revision; the retained decision is its effective one.
|
|
5918
6186
|
self.applied[event_id] = target
|
|
5919
|
-
_rematerialize_scratch_row(
|
|
6187
|
+
_rematerialize_scratch_row(
|
|
6188
|
+
conn, event_id, target, suppressors=self.applied_records())
|
|
5920
6189
|
return
|
|
5921
6190
|
self.applied[event_id] = target
|
|
5922
6191
|
if target is not None and (
|
|
@@ -5924,9 +6193,11 @@ class ScratchRetention:
|
|
|
5924
6193
|
_apply_evt(conn, target)
|
|
5925
6194
|
|
|
5926
6195
|
|
|
5927
|
-
def _rematerialize_scratch_row(conn, event_id, target
|
|
6196
|
+
def _rematerialize_scratch_row(conn, event_id, target, *,
|
|
6197
|
+
suppressors=None) -> None:
|
|
5928
6198
|
"""Bring a scratch projection row to ``target`` (None: remove it). An
|
|
5929
|
-
effects-only family has no row to bring, and its effects already ran.
|
|
6199
|
+
effects-only family has no row to bring, and its effects already ran.
|
|
6200
|
+
``suppressors`` are the decisions the replay has applied so far (#840)."""
|
|
5930
6201
|
kind = None if target is None else (target.get("payload") or {}).get("kind")
|
|
5931
6202
|
tables = ({_EVT_SPECS[kind].table} if kind in _EVT_SPECS
|
|
5932
6203
|
else {spec.table for spec in _EVT_SPECS.values()})
|
|
@@ -5940,7 +6211,8 @@ def _rematerialize_scratch_row(conn, event_id, target) -> None:
|
|
|
5940
6211
|
_delete_scratch_row(conn, table, row[0])
|
|
5941
6212
|
else:
|
|
5942
6213
|
_converge_row_from_effective(
|
|
5943
|
-
conn, event_id, table=table, rowid=row[0], record=target
|
|
6214
|
+
conn, event_id, table=table, rowid=row[0], record=target,
|
|
6215
|
+
suppressors=suppressors)
|
|
5944
6216
|
return
|
|
5945
6217
|
|
|
5946
6218
|
|
|
@@ -6251,7 +6523,7 @@ def _emit_harvest_row(ctx, spec, row):
|
|
|
6251
6523
|
!= _lib_journal.event_content_hash(evt)):
|
|
6252
6524
|
_converge_row_from_effective(
|
|
6253
6525
|
conn, chosen["id"], table=spec.table, rowid=row["id"],
|
|
6254
|
-
record=chosen)
|
|
6526
|
+
record=chosen, suppressors=retention.applied_records())
|
|
6255
6527
|
conn.execute(
|
|
6256
6528
|
f"UPDATE {spec.table} SET journal_id = ? WHERE id = ?",
|
|
6257
6529
|
(evt["id"], row["id"]),
|
|
@@ -7643,9 +7915,77 @@ _CONVERGE_OUTCOME_WORDING: dict = {
|
|
|
7643
7915
|
}
|
|
7644
7916
|
|
|
7645
7917
|
|
|
7646
|
-
|
|
7918
|
+
#: The suppression families by the axis their exact snapshot list retires
|
|
7919
|
+
#: (#840, spec section 4.8): a `week_reset`'s and a `weekly_credit_effects`'
|
|
7920
|
+
#: list retire the WEEKLY axis, a `five_hour_credit`'s the FIVE-HOUR axis.
|
|
7921
|
+
_SUPPRESSOR_KINDS_BY_AXIS = {
|
|
7922
|
+
"weekly": frozenset({"week_reset", "weekly_credit_effects"}),
|
|
7923
|
+
"five_hour": frozenset({"five_hour_credit"}),
|
|
7924
|
+
}
|
|
7925
|
+
|
|
7926
|
+
#: The effective, active decisions whose payload names a snapshot id; the
|
|
7927
|
+
#: membership itself is decided on the decoded payload.
|
|
7928
|
+
_SUPPRESSOR_CANDIDATES_SQL = (
|
|
7929
|
+
"SELECT event_id, event_json FROM journal_effective_events "
|
|
7930
|
+
"WHERE status = 'active' AND event_json LIKE ?")
|
|
7931
|
+
|
|
7932
|
+
|
|
7933
|
+
def _suppression_retires(conn, snapshot_id, axis, *, suppressors=None) -> bool:
|
|
7934
|
+
"""Whether an effective suppression decision retires ``axis`` of the
|
|
7935
|
+
snapshot ``snapshot_id`` (#840, spec section 4.8).
|
|
7936
|
+
|
|
7937
|
+
The relation is exact membership: `(suppressor event id and revision,
|
|
7938
|
+
snapshot journal id, retired axis)`. A suppressor counts when it is an
|
|
7939
|
+
admitted, selected, active decision of a family that retires ``axis``,
|
|
7940
|
+
its `suppression` list (on `weekly_usage_snapshots`) names the snapshot's
|
|
7941
|
+
retained id (`sa:<observation-id>` or a bootstrap id, never a rowid), and
|
|
7942
|
+
its effect applies at the current replay point. The decision's EFFECTIVE
|
|
7943
|
+
revision is the one read, so replacing a suppressor updates its
|
|
7944
|
+
memberships and tombstoning it removes them, while another active
|
|
7945
|
+
suppressor can independently keep the retirement. An old event without a
|
|
7946
|
+
suppression list asserts nothing, and no percentage band reconstructs
|
|
7947
|
+
one.
|
|
7948
|
+
|
|
7949
|
+
``suppressors`` is the replay's own set of applied decisions (a
|
|
7950
|
+
derivation-completion scratch passes what its retention has applied so
|
|
7951
|
+
far, so a future credit's suppression never applies early); ``None``
|
|
7952
|
+
reads the live effective metadata, which ingest writes as each decision
|
|
7953
|
+
folds and a rebuild writes in the same transaction as its folds, before
|
|
7954
|
+
any convergence can run. A rebuild applies the same lists directly in
|
|
7955
|
+
those folds."""
|
|
7956
|
+
kinds = _SUPPRESSOR_KINDS_BY_AXIS[axis]
|
|
7957
|
+
if not isinstance(snapshot_id, str):
|
|
7958
|
+
return False
|
|
7959
|
+
if suppressors is None:
|
|
7960
|
+
records = []
|
|
7961
|
+
for _event_id, event_json in conn.execute(
|
|
7962
|
+
_SUPPRESSOR_CANDIDATES_SQL,
|
|
7963
|
+
("%" + json.dumps(snapshot_id) + "%",)).fetchall():
|
|
7964
|
+
if event_json is None:
|
|
7965
|
+
continue
|
|
7966
|
+
record = _lib_journal.decode_line(event_json.encode("utf-8"))
|
|
7967
|
+
if record is not None:
|
|
7968
|
+
records.append(record)
|
|
7969
|
+
else:
|
|
7970
|
+
records = suppressors
|
|
7971
|
+
for record in records:
|
|
7972
|
+
if not isinstance(record, dict) or record.get("t") != "evt":
|
|
7973
|
+
continue
|
|
7974
|
+
payload = record.get("payload") or {}
|
|
7975
|
+
if (payload.get("kind") not in kinds
|
|
7976
|
+
or payload.get("suppression_table", "weekly_usage_snapshots")
|
|
7977
|
+
!= "weekly_usage_snapshots"):
|
|
7978
|
+
continue
|
|
7979
|
+
if snapshot_id in (payload.get("suppression") or ()):
|
|
7980
|
+
return True
|
|
7981
|
+
return False
|
|
7982
|
+
|
|
7983
|
+
|
|
7984
|
+
def _keep_five_hour_retirement(conn, cols: dict, rowid: int, event_id,
|
|
7985
|
+
*, suppressors=None) -> None:
|
|
7647
7986
|
"""Keep an admitted five-hour retirement through a row convergence
|
|
7648
|
-
(#825/#832 spec 4.7 revision 6, fix round K15, review finding K14R-5
|
|
7987
|
+
(#825/#832 spec 4.7 revision 6, fix round K15, review finding K14R-5;
|
|
7988
|
+
exact membership since #840).
|
|
7649
7989
|
|
|
7650
7990
|
A five-hour credit retires an unheld stale replica in place: its
|
|
7651
7991
|
five-hour percent, reset and window key are NULL while its
|
|
@@ -7655,9 +7995,10 @@ def _keep_five_hour_retirement(conn, cols: dict, rowid: int) -> None:
|
|
|
7655
7995
|
values back, so live state would differ from a rebuild, which folds the
|
|
7656
7996
|
event and then the credit. The row is that retired replica when it is
|
|
7657
7997
|
unheld, all three five-hour columns are NULL while the payload carries a
|
|
7658
|
-
five-hour reading, and
|
|
7659
|
-
|
|
7660
|
-
|
|
7998
|
+
five-hour reading, and an effective `five_hour_credit` suppression names
|
|
7999
|
+
its snapshot id (`_suppression_retires`). ``cols`` then converges every
|
|
8000
|
+
other column, keeping its accepted weekly evidence, and keeps the three
|
|
8001
|
+
NULL."""
|
|
7661
8002
|
if cols.get("five_hour_percent") is None or cols.get(
|
|
7662
8003
|
"five_hour_window_key") is None:
|
|
7663
8004
|
return
|
|
@@ -7668,37 +8009,30 @@ def _keep_five_hour_retirement(conn, cols: dict, rowid: int) -> None:
|
|
|
7668
8009
|
if (row is None or row[0] is not None or row[1] is not None
|
|
7669
8010
|
or row[2] is not None or row[3]):
|
|
7670
8011
|
return
|
|
7671
|
-
|
|
7672
|
-
|
|
7673
|
-
"WHERE account_key = ? AND five_hour_window_key = ? "
|
|
7674
|
-
" AND unixepoch(effective_reset_at_utc) <= unixepoch(?) LIMIT 1",
|
|
7675
|
-
(cols.get("account_key") or _lib_accounts.UNATTRIBUTED,
|
|
7676
|
-
int(cols["five_hour_window_key"]), cols.get("captured_at_utc")),
|
|
7677
|
-
).fetchone()
|
|
7678
|
-
if credited is None:
|
|
8012
|
+
if not _suppression_retires(conn, event_id, "five_hour",
|
|
8013
|
+
suppressors=suppressors):
|
|
7679
8014
|
return
|
|
7680
8015
|
cols["five_hour_percent"] = None
|
|
7681
8016
|
cols["five_hour_resets_at"] = None
|
|
7682
8017
|
cols["five_hour_window_key"] = None
|
|
7683
8018
|
|
|
7684
8019
|
|
|
7685
|
-
def _keep_weekly_retirement(conn, cols: dict, rowid: int
|
|
8020
|
+
def _keep_weekly_retirement(conn, cols: dict, rowid: int, event_id,
|
|
8021
|
+
*, suppressors=None) -> None:
|
|
7686
8022
|
"""Keep an admitted weekly retirement through a row convergence (#825/#832
|
|
7687
|
-
spec 4.7 revision 7, fix round K16b), the
|
|
7688
|
-
`_keep_five_hour_retirement`.
|
|
7689
|
-
|
|
7690
|
-
A weekly
|
|
7691
|
-
|
|
7692
|
-
|
|
7693
|
-
|
|
7694
|
-
|
|
7695
|
-
|
|
7696
|
-
|
|
7697
|
-
`
|
|
7698
|
-
|
|
7699
|
-
|
|
7700
|
-
`_weekly_value_is_retired_replica` applies. ``cols`` then converges every
|
|
7701
|
-
other column and keeps the row held."""
|
|
8023
|
+
spec 4.7 revision 7, fix round K16b; exact membership since #840), the
|
|
8024
|
+
weekly twin of `_keep_five_hour_retirement`.
|
|
8025
|
+
|
|
8026
|
+
A weekly suppression keeps a named row that carries a five-hour reading
|
|
8027
|
+
and makes it weekly-held, while its `snapshot_accept` payload still says
|
|
8028
|
+
unheld. Converging the row from that payload would hand the retired
|
|
8029
|
+
weekly value back as accepted weekly evidence. The row is that converted
|
|
8030
|
+
replica when it is weekly-held while the payload is not, carries a
|
|
8031
|
+
five-hour reading, and an effective `week_reset` or
|
|
8032
|
+
`weekly_credit_effects` suppression names its snapshot id
|
|
8033
|
+
(`_suppression_retires`). ``cols`` then converges every other column,
|
|
8034
|
+
keeping its valid five-hour data, and keeps the row held. An explicitly
|
|
8035
|
+
held payload is converged as it is."""
|
|
7702
8036
|
if cols.get("weekly_observation_held") not in (None, 0):
|
|
7703
8037
|
return
|
|
7704
8038
|
row = conn.execute(
|
|
@@ -7707,41 +8041,13 @@ def _keep_weekly_retirement(conn, cols: dict, rowid: int) -> None:
|
|
|
7707
8041
|
"FROM weekly_usage_snapshots WHERE id = ?", (rowid,)).fetchone()
|
|
7708
8042
|
if row is None or not row[0] or row[1] is None or row[2] is None:
|
|
7709
8043
|
return
|
|
7710
|
-
|
|
7711
|
-
|
|
7712
|
-
bands = []
|
|
7713
|
-
try:
|
|
7714
|
-
week_end = _cctally_core._canonicalize_optional_iso(
|
|
7715
|
-
cols.get("week_end_at"), "converge.week_end_at")
|
|
7716
|
-
except (TypeError, ValueError):
|
|
7717
|
-
week_end = None
|
|
7718
|
-
if week_end is not None:
|
|
7719
|
-
for floor_at, retired in conn.execute(
|
|
7720
|
-
"SELECT effective_reset_at_utc, observed_pre_credit_pct "
|
|
7721
|
-
"FROM week_reset_events "
|
|
7722
|
-
"WHERE account_key = ? AND new_week_end_at = ? "
|
|
7723
|
-
" AND observed_pre_credit_pct IS NOT NULL",
|
|
7724
|
-
(account_key, week_end)).fetchall():
|
|
7725
|
-
bands.append((str(floor_at), float(retired), True))
|
|
7726
|
-
for floor_at, retired in conn.execute(
|
|
7727
|
-
"SELECT effective_at_utc, observed_pre_credit_pct "
|
|
7728
|
-
"FROM weekly_credit_floors "
|
|
7729
|
-
"WHERE week_start_date = ? AND account_key = ? "
|
|
7730
|
-
" AND observed_pre_credit_pct IS NOT NULL",
|
|
7731
|
-
(cols.get("week_start_date"), account_key)).fetchall():
|
|
7732
|
-
bands.append((str(floor_at), float(retired), False))
|
|
7733
|
-
try:
|
|
7734
|
-
weekly_percent = float(cols.get("weekly_percent"))
|
|
7735
|
-
except (TypeError, ValueError):
|
|
7736
|
-
return
|
|
7737
|
-
if _cctally_record._weekly_value_is_retired_replica(
|
|
7738
|
-
bands, captured_at_utc=cols.get("captured_at_utc"),
|
|
7739
|
-
weekly_percent=weekly_percent):
|
|
8044
|
+
if _suppression_retires(conn, event_id, "weekly",
|
|
8045
|
+
suppressors=suppressors):
|
|
7740
8046
|
cols["weekly_observation_held"] = 1
|
|
7741
8047
|
|
|
7742
8048
|
|
|
7743
8049
|
def _converge_row_from_effective(conn, event_id, *, table=None, rowid=None,
|
|
7744
|
-
record=None) -> str:
|
|
8050
|
+
record=None, suppressors=None) -> str:
|
|
7745
8051
|
"""Bring the live physical row into agreement with the already-journaled
|
|
7746
8052
|
effective event, and stamp `journal_id` in the SAME operation (#374 §6).
|
|
7747
8053
|
|
|
@@ -7768,7 +8074,9 @@ def _converge_row_from_effective(conn, event_id, *, table=None, rowid=None,
|
|
|
7768
8074
|
|
|
7769
8075
|
``record`` (#825/#832 fix round E) converges to that evt-shaped record
|
|
7770
8076
|
instead of the live effective metadata's: a derivation-completion scratch
|
|
7771
|
-
replay passes the retained effective decision it holds
|
|
8077
|
+
replay passes the retained effective decision it holds, and with it
|
|
8078
|
+
``suppressors``, the decisions it has applied so far, against which a
|
|
8079
|
+
retired snapshot's exact suppression membership is decided (#840).
|
|
7772
8080
|
"""
|
|
7773
8081
|
record = (_effective_event_for_convergence(conn, event_id)
|
|
7774
8082
|
if record is None else record)
|
|
@@ -7809,8 +8117,10 @@ def _converge_row_from_effective(conn, event_id, *, table=None, rowid=None,
|
|
|
7809
8117
|
)
|
|
7810
8118
|
return CONVERGE_DROPPED
|
|
7811
8119
|
if target == "weekly_usage_snapshots":
|
|
7812
|
-
_keep_five_hour_retirement(conn, cols, int(rowid)
|
|
7813
|
-
|
|
8120
|
+
_keep_five_hour_retirement(conn, cols, int(rowid), event_id,
|
|
8121
|
+
suppressors=suppressors)
|
|
8122
|
+
_keep_weekly_retirement(conn, cols, int(rowid), event_id,
|
|
8123
|
+
suppressors=suppressors)
|
|
7814
8124
|
assignments = ", ".join(f"{name} = ?" for name in cols)
|
|
7815
8125
|
conn.execute(
|
|
7816
8126
|
f"UPDATE {target} SET {assignments} WHERE id = ?",
|
|
@@ -9526,6 +9836,9 @@ def run_stats_ingest(
|
|
|
9526
9836
|
the caller owns the set — and surfaces the same signal with the same manual
|
|
9527
9837
|
remedy rather than inventing a second recovery path.
|
|
9528
9838
|
"""
|
|
9839
|
+
# #797/#823/#840 spec 4.6 B2: ingest is an authoritative decision, so
|
|
9840
|
+
# the revision-2 boundary is in the journal before it interprets it.
|
|
9841
|
+
establish_revision_before_decision()
|
|
9529
9842
|
kwargs = {
|
|
9530
9843
|
"mode": mode,
|
|
9531
9844
|
"timeout_s": timeout_s,
|
|
@@ -9818,6 +10131,8 @@ _REBUILD_REQUIRED_TABLES = frozenset(
|
|
|
9818
10131
|
{
|
|
9819
10132
|
"accounts",
|
|
9820
10133
|
"budget_milestones",
|
|
10134
|
+
"claude_oauth_samples",
|
|
10135
|
+
"claude_weekly_observation_decisions",
|
|
9821
10136
|
"five_hour_block_models",
|
|
9822
10137
|
"five_hour_block_projects",
|
|
9823
10138
|
"five_hour_blocks",
|
|
@@ -9870,6 +10185,8 @@ _REBUILD_REQUIRED_INDEXES = frozenset(
|
|
|
9870
10185
|
"idx_budget_milestones_codex_digest",
|
|
9871
10186
|
"idx_budget_milestones_journal_id",
|
|
9872
10187
|
"idx_budget_milestones_journal_id_null",
|
|
10188
|
+
"idx_claude_oauth_samples_five_hour",
|
|
10189
|
+
"idx_claude_oauth_samples_weekly",
|
|
9873
10190
|
"idx_cost_week_start_at_time",
|
|
9874
10191
|
"idx_cost_week_time",
|
|
9875
10192
|
"idx_five_hour_block_models_block",
|
|
@@ -9933,8 +10250,11 @@ _REBUILD_REQUIRED_INDEXES = frozenset(
|
|
|
9933
10250
|
# omitted column, constraint, partial predicate, or index definition. An epoch
|
|
9934
10251
|
# schema change must update this contract alongside STATS_INDEX_EPOCH.
|
|
9935
10252
|
# #825/#832 revision 2 (#946): `journal_derivation_frontier.proof_generation`.
|
|
10253
|
+
# #797/#823/#840 epoch 1019: `claude_oauth_samples` and its two indexes,
|
|
10254
|
+
# `claude_weekly_observation_decisions`, and the four D11 identity columns of
|
|
10255
|
+
# `five_hour_credit_confirmation_state`.
|
|
9936
10256
|
_REBUILD_SCHEMA_FINGERPRINT = (
|
|
9937
|
-
"
|
|
10257
|
+
"6fe811013635e0e23c5c7fc59ed88ec6563664e0a3c5df11b0964c01f253c8fc"
|
|
9938
10258
|
)
|
|
9939
10259
|
|
|
9940
10260
|
|
|
@@ -10126,7 +10446,7 @@ def stats_index_matches_journal_prefix(
|
|
|
10126
10446
|
prior_high_water = None
|
|
10127
10447
|
cutover_captured = _CUTOVER_UNSEEN
|
|
10128
10448
|
marker_coordinates: dict = {}
|
|
10129
|
-
hasher =
|
|
10449
|
+
hasher = _new_prefix_hasher()
|
|
10130
10450
|
# #825/#832: the same admission the rebuild applied.
|
|
10131
10451
|
admission_boundary = _whole_prefix_admission_boundary()
|
|
10132
10452
|
if high_water is not None:
|
|
@@ -10150,8 +10470,8 @@ def stats_index_matches_journal_prefix(
|
|
|
10150
10470
|
|
|
10151
10471
|
for segment, offset, raw in _iter_range_with_segments(
|
|
10152
10472
|
None, high_water, segments,
|
|
10153
|
-
on_segment=lambda name: hasher.begin_segment(
|
|
10154
|
-
name, prior_high_water),
|
|
10473
|
+
on_segment=lambda name, size: hasher.begin_segment(
|
|
10474
|
+
name, prior_high_water, size=size),
|
|
10155
10475
|
on_bytes=hasher.extend,
|
|
10156
10476
|
elide=_elide_segment,
|
|
10157
10477
|
):
|
|
@@ -12002,7 +12322,7 @@ def _resolve_quota_cache_coverage(cache_path, high_water, decoded_end=None):
|
|
|
12002
12322
|
def _rebuild_quota_cache_leg_raw(
|
|
12003
12323
|
quota_raw, decoded, cutover_claude, counters=None, *,
|
|
12004
12324
|
high_water=None, coverage=None, decoded_end=None, snapshot_out=None,
|
|
12005
|
-
quiet=False, elision_gaps=None,
|
|
12325
|
+
quiet=False, elision_gaps=None, hold_stats=None,
|
|
12006
12326
|
) -> float:
|
|
12007
12327
|
"""Re-materialize cache.db `quota_window_snapshots` AND the #416 Codex
|
|
12008
12328
|
attribution map from the journal (spec §5.4 + #416 spec §3.4), fed RAW
|
|
@@ -12035,7 +12355,12 @@ def _rebuild_quota_cache_leg_raw(
|
|
|
12035
12355
|
begins, exactly as the §3.5 precedence rule requires.
|
|
12036
12356
|
|
|
12037
12357
|
Returns the measured flock hold in seconds, which acceptance criterion 7
|
|
12038
|
-
caps.
|
|
12358
|
+
caps. ``hold_stats``, when given, receives ``maxSeconds``: the longest
|
|
12359
|
+
single hold, which #997 D7 adds to the rebuild record.
|
|
12360
|
+
|
|
12361
|
+
``quota_raw`` is a list of raw lines or, from the final rebuild, a
|
|
12362
|
+
`QuotaLineSource` of coordinates that the recovery re-reads one chunk at a
|
|
12363
|
+
time (#997 D7).
|
|
12039
12364
|
"""
|
|
12040
12365
|
file_accounts = [
|
|
12041
12366
|
r for r in decoded if r is not None and _is_codex_file_account_op(r)
|
|
@@ -12136,7 +12461,7 @@ def _rebuild_quota_cache_leg_raw(
|
|
|
12136
12461
|
quota_raw, file_accounts, cutover_claude, counters,
|
|
12137
12462
|
cache_path=cache_path, vector=vector, covered=covered,
|
|
12138
12463
|
high_water=high_water, coverage=coverage, quiet=quiet,
|
|
12139
|
-
attributions=attributions,
|
|
12464
|
+
attributions=attributions, hold_stats=hold_stats,
|
|
12140
12465
|
)
|
|
12141
12466
|
|
|
12142
12467
|
|
|
@@ -12195,7 +12520,7 @@ def _recovery_state(cache):
|
|
|
12195
12520
|
def _run_bounded_recovery(
|
|
12196
12521
|
quota_raw, file_accounts, cutover_claude, counters, *,
|
|
12197
12522
|
cache_path, vector, covered, high_water, coverage, quiet=False,
|
|
12198
|
-
attributions=(),
|
|
12523
|
+
attributions=(), hold_stats=None,
|
|
12199
12524
|
) -> float:
|
|
12200
12525
|
"""Recovery as resumable chunks, each capped by bytes AND record count.
|
|
12201
12526
|
|
|
@@ -12227,7 +12552,16 @@ def _run_bounded_recovery(
|
|
|
12227
12552
|
of one pass, so a single recovery reports at most what a single unchunked
|
|
12228
12553
|
call reports today.
|
|
12229
12554
|
|
|
12230
|
-
Returns the total measured flock hold across every chunk
|
|
12555
|
+
Returns the total measured flock hold across every chunk; ``hold_stats``,
|
|
12556
|
+
when given, receives the longest single one as ``maxSeconds``.
|
|
12557
|
+
|
|
12558
|
+
**A coordinate source that can no longer be re-read** (#997 D7: a short,
|
|
12559
|
+
missing or reframed slice) fails the pass the way an unreadable elided
|
|
12560
|
+
segment does: that chunk's observations are not applied, the covered
|
|
12561
|
+
boundary is dropped so nothing is certified over them, and the remaining
|
|
12562
|
+
chunks still apply, ending in `noCoverageEstablished` with `complete`
|
|
12563
|
+
False. The failures are listed in the coverage block's
|
|
12564
|
+
`lineSourceFailures`.
|
|
12231
12565
|
"""
|
|
12232
12566
|
import _cctally_cache
|
|
12233
12567
|
from _lib_cache_writer_lock import (
|
|
@@ -12238,7 +12572,7 @@ def _run_bounded_recovery(
|
|
|
12238
12572
|
pass_id, started_at = _recovery_pass_identity()
|
|
12239
12573
|
identity_root = _lib_cache_coverage.identity_root(vector)
|
|
12240
12574
|
spans = _lib_cache_coverage.chunk_spans(
|
|
12241
|
-
|
|
12575
|
+
_quota_line_sizes(quota_raw),
|
|
12242
12576
|
byte_cap=_RECOVERY_CHUNK_BYTES, record_cap=_RECOVERY_CHUNK_RECORDS,
|
|
12243
12577
|
)
|
|
12244
12578
|
# NOT cleared on a restart, deliberately. §4.6 requires that one recovery
|
|
@@ -12251,6 +12585,8 @@ def _run_bounded_recovery(
|
|
|
12251
12585
|
# are snapshotted here and restored on each restart.
|
|
12252
12586
|
counter_baseline = dict(counters) if counters is not None else None
|
|
12253
12587
|
hold_seconds = 0.0
|
|
12588
|
+
max_hold_seconds = 0.0
|
|
12589
|
+
source_failures: list = []
|
|
12254
12590
|
restarts = 0
|
|
12255
12591
|
chunk_index = 0
|
|
12256
12592
|
applied = 0
|
|
@@ -12306,14 +12642,33 @@ def _run_bounded_recovery(
|
|
|
12306
12642
|
decoded_chunk = None
|
|
12307
12643
|
if span is not None:
|
|
12308
12644
|
start, stop, _bytes = span
|
|
12309
|
-
|
|
12310
|
-
|
|
12645
|
+
try:
|
|
12646
|
+
decoded_chunk = list(_decoded_quota_stream(
|
|
12647
|
+
_quota_lines(quota_raw, start, stop), cutover_claude,
|
|
12648
|
+
counters))
|
|
12649
|
+
except QuotaLineSourceError as exc:
|
|
12650
|
+
source_failures.append(
|
|
12651
|
+
{"chunk": chunk_index, "error": str(exc)})
|
|
12652
|
+
if not quiet:
|
|
12653
|
+
print(f"[rebuild] quota cache leg re-read failed: {exc}",
|
|
12654
|
+
file=sys.stderr)
|
|
12655
|
+
decoded_chunk = None
|
|
12656
|
+
covered = None
|
|
12311
12657
|
try:
|
|
12658
|
+
# #997 D9: a first non-blocking try, so free flocks cost no
|
|
12659
|
+
# progress write; only a contended acquisition is a reported wait.
|
|
12312
12660
|
held = acquire_cache_writer_flocks(
|
|
12313
12661
|
_cctally_core.CACHE_LOCK_PATH,
|
|
12314
12662
|
_cctally_core.CACHE_LOCK_CODEX_PATH,
|
|
12315
|
-
timeout=
|
|
12663
|
+
timeout=None,
|
|
12316
12664
|
)
|
|
12665
|
+
if held is None:
|
|
12666
|
+
with _lib_rebuild_progress.waiting("quota-flocks"):
|
|
12667
|
+
held = acquire_cache_writer_flocks(
|
|
12668
|
+
_cctally_core.CACHE_LOCK_PATH,
|
|
12669
|
+
_cctally_core.CACHE_LOCK_CODEX_PATH,
|
|
12670
|
+
timeout=15.0,
|
|
12671
|
+
)
|
|
12317
12672
|
except OSError as exc:
|
|
12318
12673
|
if not quiet:
|
|
12319
12674
|
print(f"[rebuild] quota cache leg lock failed: {exc}",
|
|
@@ -12490,13 +12845,22 @@ def _run_bounded_recovery(
|
|
|
12490
12845
|
if cache is not None:
|
|
12491
12846
|
cache.close()
|
|
12492
12847
|
finally:
|
|
12493
|
-
|
|
12848
|
+
held_for = time.monotonic() - held_from
|
|
12849
|
+
hold_seconds += held_for
|
|
12850
|
+
max_hold_seconds = max(max_hold_seconds, held_for)
|
|
12494
12851
|
release_cache_writer_flocks(held)
|
|
12495
12852
|
decoded_chunk = None
|
|
12853
|
+
_lib_rebuild_progress.tick(
|
|
12854
|
+
chunks=committed_chunks, plannedChunks=len(plan),
|
|
12855
|
+
observations=applied)
|
|
12496
12856
|
if _RECOVERY_BETWEEN_CHUNKS is not None:
|
|
12497
12857
|
_RECOVERY_BETWEEN_CHUNKS(committed_chunks)
|
|
12498
12858
|
|
|
12499
12859
|
_record(outcome)
|
|
12860
|
+
if source_failures and coverage is not None:
|
|
12861
|
+
coverage["lineSourceFailures"] = source_failures
|
|
12862
|
+
if hold_stats is not None:
|
|
12863
|
+
hold_stats["maxSeconds"] = max_hold_seconds
|
|
12500
12864
|
return hold_seconds
|
|
12501
12865
|
|
|
12502
12866
|
|
|
@@ -12731,6 +13095,12 @@ def rebuild_stats_index(
|
|
|
12731
13095
|
raise ValueError(
|
|
12732
13096
|
"complete_derivation applies to the live destination only")
|
|
12733
13097
|
import _cctally_completion
|
|
13098
|
+
if high_water is None:
|
|
13099
|
+
# #797/#823/#840 spec 4.6 B2: recovery and maintenance rebuilds
|
|
13100
|
+
# are authoritative decisions; the revision-2 boundary is in the
|
|
13101
|
+
# journal before completion reads its prefix, so completion's own
|
|
13102
|
+
# append never races the boundary's.
|
|
13103
|
+
establish_revision_before_decision(scan=True)
|
|
12734
13104
|
return _cctally_completion.complete_and_rebuild(
|
|
12735
13105
|
context=context,
|
|
12736
13106
|
high_water=high_water,
|
|
@@ -12952,6 +13322,7 @@ def _rebuild_stats_index_locked(
|
|
|
12952
13322
|
"quota_replay")
|
|
12953
13323
|
}
|
|
12954
13324
|
quota_lock_hold = 0.0
|
|
13325
|
+
quota_hold_stats: dict = {}
|
|
12955
13326
|
#: Whether this generation's quota projection was materialized from a cache
|
|
12956
13327
|
#: whose recovery left an uncovered remainder. Declared out here because the
|
|
12957
13328
|
#: record and the result are assembled outside the try that sets it.
|
|
@@ -12980,9 +13351,11 @@ def _rebuild_stats_index_locked(
|
|
|
12980
13351
|
# consumer actually needs: the decision records decoded (5.08% of a
|
|
12981
13352
|
# production journal) and the Codex quota observations as RAW BYTES
|
|
12982
13353
|
# (1.64 GB against roughly six gigabytes of dicts). Everything else is
|
|
12983
|
-
# dropped as soon as it has contributed (#496 S4 §4).
|
|
13354
|
+
# dropped as soon as it has contributed (#496 S4 §4). #997 D7: the
|
|
13355
|
+
# quota observations are now retained as COORDINATES (16 bytes each)
|
|
13356
|
+
# and the cache leg re-reads them one chunk at a time.
|
|
12984
13357
|
decoded: list = []
|
|
12985
|
-
quota_raw
|
|
13358
|
+
quota_raw = QuotaLineSource()
|
|
12986
13359
|
protocol_evidence = []
|
|
12987
13360
|
prior_high_water = None
|
|
12988
13361
|
cutover_captured = _CUTOVER_UNSEEN
|
|
@@ -12993,15 +13366,25 @@ def _rebuild_stats_index_locked(
|
|
|
12993
13366
|
# 64,248 correction records carry far fewer markers than lines.
|
|
12994
13367
|
marker_coordinates: dict = {}
|
|
12995
13368
|
last_seen = _lib_journal_router.LastSeenAccumulator()
|
|
13369
|
+
# #797/#823/#840 spec 4.3 D8: this rebuild is apply-only and never runs
|
|
13370
|
+
# the pipeline that projects provider samples live, so the pass offers
|
|
13371
|
+
# every ADMITTED record to the same projection, in bounded batches on
|
|
13372
|
+
# the scratch index.
|
|
13373
|
+
import _cctally_oauth_samples
|
|
13374
|
+
samples = _cctally_oauth_samples.RebuildProjector(conn)
|
|
12996
13375
|
summaries = _SegmentSummaryCollector()
|
|
12997
13376
|
# F12 (#496 S5b §5). Planned before the first segment is reached, from
|
|
12998
13377
|
# the sidecar this pass's predecessor wrote and the coverage certificate
|
|
12999
13378
|
# Stage 3 mints. A plan that elides nothing is exactly today's pass.
|
|
13000
13379
|
elision = plan_segment_elision(segments, hw)
|
|
13001
|
-
hasher =
|
|
13380
|
+
hasher = _new_prefix_hasher()
|
|
13002
13381
|
evidence_seconds = 0.0
|
|
13003
13382
|
prefix = traversal["stats_prefix"]
|
|
13004
13383
|
read_started = time.monotonic()
|
|
13384
|
+
# #997 D9: progress for a detached worker's state file (a no-op
|
|
13385
|
+
# unless one installed a sink); ticks are throttled to one write per
|
|
13386
|
+
# ten seconds.
|
|
13387
|
+
_lib_rebuild_progress.phase("rebuild:read", lines=0, bytes=0)
|
|
13005
13388
|
|
|
13006
13389
|
def _elide_segment(name, lo, hi, stat_result) -> bool:
|
|
13007
13390
|
"""Contribute an elided segment's share instead of reading it."""
|
|
@@ -13044,8 +13427,8 @@ def _rebuild_stats_index_locked(
|
|
|
13044
13427
|
if hw is not None:
|
|
13045
13428
|
for segment, offset, raw in _iter_range_with_segments(
|
|
13046
13429
|
None, hw, segments,
|
|
13047
|
-
on_segment=lambda name: hasher.begin_segment(
|
|
13048
|
-
name, prior_high_water),
|
|
13430
|
+
on_segment=lambda name, size: hasher.begin_segment(
|
|
13431
|
+
name, prior_high_water, size=size),
|
|
13049
13432
|
on_bytes=hasher.extend,
|
|
13050
13433
|
on_extent=lambda name, lo, hi, st: summaries.begin(
|
|
13051
13434
|
name, lo, hi, st, last_seen),
|
|
@@ -13053,8 +13436,16 @@ def _rebuild_stats_index_locked(
|
|
|
13053
13436
|
):
|
|
13054
13437
|
prefix["lines"] += 1
|
|
13055
13438
|
prefix["bytes"] += len(raw) + 1
|
|
13439
|
+
if not prefix["lines"] & 4095:
|
|
13440
|
+
_lib_rebuild_progress.tick(
|
|
13441
|
+
lines=prefix["lines"], bytes=prefix["bytes"])
|
|
13056
13442
|
summaries.line(raw, offset + len(raw) + 1)
|
|
13057
13443
|
rec = _lib_journal.decode_line(raw)
|
|
13444
|
+
if (rec is not None and rec.get("t")
|
|
13445
|
+
in _lib_journal_router.RETAINED_RECORD_TYPES):
|
|
13446
|
+
# #997 D15: a record this pass may retain is compacted
|
|
13447
|
+
# before anything holds a reference to it.
|
|
13448
|
+
rec = _lib_journal.compact_retained(rec)
|
|
13058
13449
|
if rec is None:
|
|
13059
13450
|
malformed += 1
|
|
13060
13451
|
summaries.malformed_line()
|
|
@@ -13088,6 +13479,7 @@ def _rebuild_stats_index_locked(
|
|
|
13088
13479
|
continue
|
|
13089
13480
|
rebuild_quarantine.on_record(
|
|
13090
13481
|
rec, segment=segment, offset=offset)
|
|
13482
|
+
samples.offer(rec)
|
|
13091
13483
|
# `_capture_protocol_prefix_evidence` returns immediately for
|
|
13092
13484
|
# anything that is not an op, so this guard changes nothing it
|
|
13093
13485
|
# does — it moves the phase attribution's two clock reads from
|
|
@@ -13146,7 +13538,7 @@ def _rebuild_stats_index_locked(
|
|
|
13146
13538
|
decoded.append(rec)
|
|
13147
13539
|
else:
|
|
13148
13540
|
if update_quota_cache and _is_codex_quota_obs(rec):
|
|
13149
|
-
quota_raw.append(raw)
|
|
13541
|
+
quota_raw.append(segment, offset, len(raw))
|
|
13150
13542
|
# A PLACEHOLDER, not a dropped element. `resolve_effective_events`
|
|
13151
13543
|
# numbers candidates with `enumerate(records)`, and three of the
|
|
13152
13544
|
# seven structural violation kinds put that number inside
|
|
@@ -13165,6 +13557,8 @@ def _rebuild_stats_index_locked(
|
|
|
13165
13557
|
offset + len(raw) + 1,
|
|
13166
13558
|
)
|
|
13167
13559
|
summaries.close(last_seen)
|
|
13560
|
+
samples.flush()
|
|
13561
|
+
samples = None
|
|
13168
13562
|
# §6.3 "recorded, not silent": every elision refusal is a SILENT
|
|
13169
13563
|
# fallback, and this block is the only surface that says which one.
|
|
13170
13564
|
traversal["elision"] = elision.counters()
|
|
@@ -13193,6 +13587,7 @@ def _rebuild_stats_index_locked(
|
|
|
13193
13587
|
# streamed prefix contained it, otherwise from the unvisited suffix alone
|
|
13194
13588
|
# (falls back to `unattributed` when neither has it), so a fresh
|
|
13195
13589
|
# single-account rebuild is byte-neutral.
|
|
13590
|
+
_lib_rebuild_progress.phase("rebuild:selection")
|
|
13196
13591
|
cutover_started = time.monotonic()
|
|
13197
13592
|
cutover_claude = _resolve_cutover_for_rebuild(
|
|
13198
13593
|
cutover_captured, hw, all_segments, traversal["cutover_suffix"],
|
|
@@ -13238,13 +13633,14 @@ def _rebuild_stats_index_locked(
|
|
|
13238
13633
|
# Codex quota obs -> cache.db quota_window_snapshots. The retained bytes
|
|
13239
13634
|
# are decoded inside the leg's existing transaction and freed here, so
|
|
13240
13635
|
# the SQLite fold never runs on top of them.
|
|
13636
|
+
_lib_rebuild_progress.phase("rebuild:quota-leg")
|
|
13241
13637
|
leg_started = time.monotonic()
|
|
13242
13638
|
if update_quota_cache:
|
|
13243
13639
|
quota_lock_hold = _rebuild_quota_cache_leg_raw(
|
|
13244
13640
|
quota_raw, decoded, cutover_claude, traversal["quota_replay"],
|
|
13245
13641
|
high_water=hw, coverage=quota_coverage,
|
|
13246
13642
|
decoded_end=prior_high_water, snapshot_out=quota_snapshot,
|
|
13247
|
-
elision_gaps=elision.quota_gaps)
|
|
13643
|
+
elision_gaps=elision.quota_gaps, hold_stats=quota_hold_stats)
|
|
13248
13644
|
quota_raw = []
|
|
13249
13645
|
phase_seconds["quota_cache_leg"] = round(
|
|
13250
13646
|
time.monotonic() - leg_started, 6)
|
|
@@ -13268,6 +13664,7 @@ def _rebuild_stats_index_locked(
|
|
|
13268
13664
|
|
|
13269
13665
|
_stats_rebuild_test_pause("rebuild_fold_started")
|
|
13270
13666
|
fold_started = time.monotonic()
|
|
13667
|
+
_lib_rebuild_progress.phase("rebuild:fold", rows=0)
|
|
13271
13668
|
|
|
13272
13669
|
# Phase 1 (txn A) — structural folds: op floors, snapshot_accept, cost
|
|
13273
13670
|
# snapshots, resets+suppression, block_close, arming, credit effects.
|
|
@@ -13280,6 +13677,8 @@ def _rebuild_stats_index_locked(
|
|
|
13280
13677
|
else:
|
|
13281
13678
|
_apply_evt(conn, rec)
|
|
13282
13679
|
lines_folded += 1
|
|
13680
|
+
if not lines_folded & 1023:
|
|
13681
|
+
_lib_rebuild_progress.tick(rows=lines_folded)
|
|
13283
13682
|
conn.commit()
|
|
13284
13683
|
except BaseException:
|
|
13285
13684
|
try:
|
|
@@ -13318,12 +13717,34 @@ def _rebuild_stats_index_locked(
|
|
|
13318
13717
|
# later (see the note on `structural_fold` above). It cannot move any
|
|
13319
13718
|
# earlier without holding the bundle's 254 MiB of observations (measured
|
|
13320
13719
|
# on the same store, 232,466 rows) across the structural fold as well.
|
|
13720
|
+
#
|
|
13721
|
+
# #997 D14: the read-pass state is released FIRST, so the bundle's
|
|
13722
|
+
# observations (and, without a snapshot, the projection's own load in
|
|
13723
|
+
# transaction B) reuse its memory instead of stacking on it. The
|
|
13724
|
+
# open-block projection reads that state only through
|
|
13725
|
+
# `journaled_block_first_observations`, so its compact result is
|
|
13726
|
+
# computed here, with the three `effective` fields the result reads
|
|
13727
|
+
# later. `selector_accumulators` goes too: its `SelectorFold`
|
|
13728
|
+
# candidates hold decoded event records. `selector_rows` (materialized
|
|
13729
|
+
# above, for `_validate_selector_state`) and the records `tail` retains
|
|
13730
|
+
# for transaction B stay. The pin on the cache snapshot grows only by
|
|
13731
|
+
# this compact computation; the bundle is still read before phase 2a.
|
|
13732
|
+
# `rec` is the folds' loop variable, still naming the last structural
|
|
13733
|
+
# event.
|
|
13734
|
+
journaled_first = journaled_block_first_observations(
|
|
13735
|
+
decoded, effective.by_id)
|
|
13736
|
+
conflicts = effective.conflicts
|
|
13737
|
+
protocol_violations = effective.protocol_violations
|
|
13738
|
+
acknowledged = effective.acknowledged_protocol_violations
|
|
13739
|
+
decoded = effective = stream = structural = rec = None
|
|
13740
|
+
selector_accumulators = None
|
|
13321
13741
|
quota_bundle = _read_quota_projection_bundle(quota_snapshot)
|
|
13322
13742
|
|
|
13323
13743
|
# Phase 2a — OPEN 5h block projection (own txn; block-only). Closed blocks
|
|
13324
13744
|
# came from block_close evts; this materializes the never-closed window(s)
|
|
13325
13745
|
# so the five_hour_milestone block_id derived_fk resolves. Best-effort
|
|
13326
13746
|
# (the open block is a projection, §5.3).
|
|
13747
|
+
_lib_rebuild_progress.phase("rebuild:open-blocks")
|
|
13327
13748
|
projection_started = time.monotonic()
|
|
13328
13749
|
try:
|
|
13329
13750
|
cctally = sys.modules.get("cctally")
|
|
@@ -13334,15 +13755,15 @@ def _rebuild_stats_index_locked(
|
|
|
13334
13755
|
# `snapshot_accept` in journal order, which a credit's
|
|
13335
13756
|
# stale-replica removal cannot delete and an earlier capture
|
|
13336
13757
|
# appended later cannot displace.
|
|
13337
|
-
bf(conn, only_missing=True,
|
|
13338
|
-
journaled_first=journaled_block_first_observations(
|
|
13339
|
-
decoded, effective.by_id))
|
|
13758
|
+
bf(conn, only_missing=True, journaled_first=journaled_first)
|
|
13340
13759
|
except Exception as exc: # pragma: no cover — projection is best-effort
|
|
13341
13760
|
print(f"[rebuild] open 5h block re-materialization failed: {exc}",
|
|
13342
13761
|
file=sys.stderr)
|
|
13343
13762
|
phase_seconds["open_block_projection"] = round(
|
|
13344
13763
|
time.monotonic() - projection_started, 6)
|
|
13764
|
+
journaled_first = None
|
|
13345
13765
|
|
|
13766
|
+
_lib_rebuild_progress.phase("rebuild:stats-fold", rows=lines_folded)
|
|
13346
13767
|
# Phase 2b + 3 (txn B) — quota projection re-materialization (after the
|
|
13347
13768
|
# order-45 arming folds) + milestone/budget folds + cursor advance.
|
|
13348
13769
|
conn.execute("BEGIN IMMEDIATE")
|
|
@@ -13372,6 +13793,8 @@ def _rebuild_stats_index_locked(
|
|
|
13372
13793
|
for _order, _seq, _kind, rec in tail:
|
|
13373
13794
|
_apply_evt(conn, rec)
|
|
13374
13795
|
lines_folded += 1
|
|
13796
|
+
if not lines_folded & 1023:
|
|
13797
|
+
_lib_rebuild_progress.tick(rows=lines_folded)
|
|
13375
13798
|
# Fold-time `last_seen_utc` derivation (#341): re-derive each
|
|
13376
13799
|
# account's last-seen from the whole journal. The map was
|
|
13377
13800
|
# accumulated during the single read pass — `decoded` no longer
|
|
@@ -13396,6 +13819,7 @@ def _rebuild_stats_index_locked(
|
|
|
13396
13819
|
raise
|
|
13397
13820
|
phase_seconds["stats_fold"] = round(time.monotonic() - fold_started, 6)
|
|
13398
13821
|
|
|
13822
|
+
_lib_rebuild_progress.phase("rebuild:validate")
|
|
13399
13823
|
validate_started = time.monotonic()
|
|
13400
13824
|
rows_by_table = {}
|
|
13401
13825
|
for tbl in _REBUILD_COUNT_TABLES:
|
|
@@ -13442,15 +13866,15 @@ def _rebuild_stats_index_locked(
|
|
|
13442
13866
|
# database-sized rollback journal while the scratch still exists, so the
|
|
13443
13867
|
# multi-gigabyte replay peak must not still be resident on top of it.
|
|
13444
13868
|
segments_read = len(segments)
|
|
13445
|
-
conflicts
|
|
13446
|
-
|
|
13447
|
-
|
|
13869
|
+
# `conflicts`, `protocol_violations` and `acknowledged` were captured
|
|
13870
|
+
# before the quota projection bundle loaded, where the read-pass state was
|
|
13871
|
+
# released (#997 D14).
|
|
13448
13872
|
# The pre-publication window is what F9's memory acceptance is measured
|
|
13449
13873
|
# over: everything after this point is publication, whose rollback-journal
|
|
13450
13874
|
# cost S3 already accounts for through projected headroom.
|
|
13451
13875
|
peak_heap_bytes = (
|
|
13452
13876
|
tracemalloc.get_traced_memory()[1] if tracing else 0)
|
|
13453
|
-
|
|
13877
|
+
tail = None
|
|
13454
13878
|
segments = protocol_evidence = last_seen = None
|
|
13455
13879
|
|
|
13456
13880
|
# First fresh-connection validation (#496 S1 F1). A failure here raises
|
|
@@ -13465,6 +13889,7 @@ def _rebuild_stats_index_locked(
|
|
|
13465
13889
|
scratch, phase="fresh pre-publication validation"
|
|
13466
13890
|
)
|
|
13467
13891
|
|
|
13892
|
+
_lib_rebuild_progress.phase("rebuild:publish")
|
|
13468
13893
|
publication_started = time.monotonic()
|
|
13469
13894
|
incident = _publish_rebuilt_stats_index(
|
|
13470
13895
|
scratch=scratch,
|
|
@@ -13500,7 +13925,14 @@ def _rebuild_stats_index_locked(
|
|
|
13500
13925
|
name: dict(counts) for name, counts in traversal.items()
|
|
13501
13926
|
},
|
|
13502
13927
|
"peakHeapBytes": peak_heap_bytes,
|
|
13928
|
+
# #997 D11: the process-lifetime resident peak, which for the
|
|
13929
|
+
# detached worker covers completion and this rebuild.
|
|
13930
|
+
"peakResidentBytes": _lib_rebuild_progress.peak_resident_bytes(),
|
|
13503
13931
|
"quotaLockHoldSeconds": round(quota_lock_hold, 6),
|
|
13932
|
+
# #997 D7: the longest single hold; the field above stays the
|
|
13933
|
+
# cumulative hold across chunks and retries.
|
|
13934
|
+
"quotaLockMaxHoldSeconds": round(
|
|
13935
|
+
quota_hold_stats.get("maxSeconds", 0.0), 6),
|
|
13504
13936
|
"selectorDesynchronized": selector_desync,
|
|
13505
13937
|
"quotaCacheCoverage": quota_coverage,
|
|
13506
13938
|
# §4.7: distinct from `quotaCacheCoverage`, which describes the
|
|
@@ -14277,10 +14709,19 @@ def _account_bound_seam(step: str) -> None:
|
|
|
14277
14709
|
seam(step)
|
|
14278
14710
|
|
|
14279
14711
|
|
|
14280
|
-
def _legacy_spool_paths() -> list:
|
|
14281
|
-
"""The previous
|
|
14712
|
+
def _legacy_spool_paths(*, include_v1: bool = True) -> list:
|
|
14713
|
+
"""The previous releases' statusline coordination paths (spec section 7).
|
|
14714
|
+
|
|
14715
|
+
#797/#823/#840 (spec 4.6 B4): the retired v1 account-bound root's
|
|
14716
|
+
coordination state — its ``accounts/`` tree and ``transport.last`` —
|
|
14717
|
+
joins the pre-#825 paths. The v1 root's own caches, baseline and
|
|
14718
|
+
diagnostics carry no accounting input and stay out, so an older process
|
|
14719
|
+
refreshing a diagnostic file raises no trigger. ``include_v1=False`` is
|
|
14720
|
+
the ORIGINAL inventory the cutover op and its anchors describe; the
|
|
14721
|
+
revision boundary records the expanded one, and every rescan after it
|
|
14722
|
+
compares the expanded inventory."""
|
|
14282
14723
|
core = _cctally_core
|
|
14283
|
-
|
|
14724
|
+
paths = [
|
|
14284
14725
|
core.STATUSLINE_OBSERVE_MARKER_PATH,
|
|
14285
14726
|
core.STATUSLINE_PERSIST_LOCK_PATH,
|
|
14286
14727
|
core.STATUSLINE_CANDIDATE_DIR,
|
|
@@ -14289,6 +14730,12 @@ def _legacy_spool_paths() -> list:
|
|
|
14289
14730
|
core.STATUSLINE_AUTHORITATIVE_7D_PATH,
|
|
14290
14731
|
core.STATUSLINE_AUTHORITATIVE_5H_PATH,
|
|
14291
14732
|
]
|
|
14733
|
+
if include_v1:
|
|
14734
|
+
paths.extend([
|
|
14735
|
+
core.STATUSLINE_ACCOUNT_BOUND_V1_DIR / "accounts",
|
|
14736
|
+
core.STATUSLINE_ACCOUNT_BOUND_V1_DIR / "transport.last",
|
|
14737
|
+
])
|
|
14738
|
+
return paths
|
|
14292
14739
|
|
|
14293
14740
|
|
|
14294
14741
|
def _legacy_relative_name(path) -> str:
|
|
@@ -14341,9 +14788,9 @@ def _legacy_file_entry(path, rel: str):
|
|
|
14341
14788
|
_lib_legacy_spool.content_digest(data))
|
|
14342
14789
|
|
|
14343
14790
|
|
|
14344
|
-
def capture_legacy_spool_state():
|
|
14791
|
+
def capture_legacy_spool_state(*, include_v1: bool = True):
|
|
14345
14792
|
"""Observe the legacy statusline spool. Content is read only to digest it;
|
|
14346
|
-
it is never accounting input.
|
|
14793
|
+
it is never accounting input. ``include_v1`` as in `_legacy_spool_paths`.
|
|
14347
14794
|
|
|
14348
14795
|
Complete or raising (review cycle 7 825-832-CD7-J01): a spool path that
|
|
14349
14796
|
is absent, or an entry that vanished since its directory was listed, is
|
|
@@ -14361,7 +14808,7 @@ def capture_legacy_spool_state():
|
|
|
14361
14808
|
raise _legacy_inventory_incomplete(
|
|
14362
14809
|
getattr(exc, "filename", None) or walking[0], exc, "list") from exc
|
|
14363
14810
|
|
|
14364
|
-
for path in _legacy_spool_paths():
|
|
14811
|
+
for path in _legacy_spool_paths(include_v1=include_v1):
|
|
14365
14812
|
path = pathlib.Path(path)
|
|
14366
14813
|
walking[0] = path
|
|
14367
14814
|
try:
|
|
@@ -14393,8 +14840,9 @@ def _decode_anchor_record(raw: bytes):
|
|
|
14393
14840
|
|
|
14394
14841
|
|
|
14395
14842
|
def _anchor_record(rec):
|
|
14396
|
-
"""``rec`` when it is a stamped cutover or legacy-spool-activity
|
|
14397
|
-
None.
|
|
14843
|
+
"""``rec`` when it is a stamped cutover, revision or legacy-spool-activity
|
|
14844
|
+
op, else None. Position-free: after the revision boundary a reader also
|
|
14845
|
+
requires ``wpr == 2`` (`_post_revision_without_revision`)."""
|
|
14398
14846
|
if (not isinstance(rec, dict) or rec.get("t") != "op"
|
|
14399
14847
|
or not _lib_account_protocol.is_stamped(rec)):
|
|
14400
14848
|
return None
|
|
@@ -14402,6 +14850,8 @@ def _anchor_record(rec):
|
|
|
14402
14850
|
if not isinstance(payload, dict):
|
|
14403
14851
|
return None
|
|
14404
14852
|
kind = payload.get("kind")
|
|
14853
|
+
if kind == _lib_account_protocol.REVISION_OP_KIND:
|
|
14854
|
+
return _revision_record(rec)
|
|
14405
14855
|
if kind == _lib_account_protocol.CUTOVER_OP_KIND:
|
|
14406
14856
|
if rec.get("id") != _lib_account_protocol.CUTOVER_OP_ID:
|
|
14407
14857
|
return None
|
|
@@ -14437,7 +14887,9 @@ def _inventory_uncertainty_record(rec):
|
|
|
14437
14887
|
return rec
|
|
14438
14888
|
return None
|
|
14439
14889
|
anchor = _anchor_record(rec)
|
|
14440
|
-
if (anchor is not None
|
|
14890
|
+
if (anchor is not None
|
|
14891
|
+
and kind in (_lib_account_protocol.CUTOVER_OP_KIND,
|
|
14892
|
+
_lib_account_protocol.REVISION_OP_KIND)
|
|
14441
14893
|
and payload.get("legacySpoolInventoryFailure") is not None):
|
|
14442
14894
|
return rec
|
|
14443
14895
|
return None
|
|
@@ -14453,11 +14905,26 @@ def _cutover_uncertainty(found):
|
|
|
14453
14905
|
|
|
14454
14906
|
def _anchor_digest(rec) -> str:
|
|
14455
14907
|
payload = rec["payload"]
|
|
14456
|
-
if payload["kind"]
|
|
14908
|
+
if payload["kind"] in (_lib_account_protocol.CUTOVER_OP_KIND,
|
|
14909
|
+
_lib_account_protocol.REVISION_OP_KIND):
|
|
14457
14910
|
return payload["legacySpoolBaselineDigest"]
|
|
14458
14911
|
return payload["digest"]
|
|
14459
14912
|
|
|
14460
14913
|
|
|
14914
|
+
def _post_revision_without_revision(rec, segment, offset, revision) -> bool:
|
|
14915
|
+
"""Whether ``rec`` at ``(segment, offset)`` lies after the revision-2
|
|
14916
|
+
boundary without carrying ``wpr == 2`` (spec 4.6 B3): such a record is
|
|
14917
|
+
excluded input, so an older process's anchoring or uncertainty op there
|
|
14918
|
+
never advances or opens anything ("old post-boundary anchoring ops never
|
|
14919
|
+
advance the revision-2 baseline")."""
|
|
14920
|
+
if revision is None:
|
|
14921
|
+
return False
|
|
14922
|
+
if _lib_account_protocol.is_historical(
|
|
14923
|
+
_lib_account_protocol.Coordinate(segment, offset), revision):
|
|
14924
|
+
return False
|
|
14925
|
+
return not _lib_account_protocol.has_current_revision(rec)
|
|
14926
|
+
|
|
14927
|
+
|
|
14461
14928
|
def _read_line_at(segment: str, offset: int):
|
|
14462
14929
|
"""The raw complete line starting at ``(segment, offset)``, or None."""
|
|
14463
14930
|
if not isinstance(segment, str) or type(offset) is not int or offset < 0:
|
|
@@ -14483,9 +14950,11 @@ def _read_line_at(segment: str, offset: int):
|
|
|
14483
14950
|
|
|
14484
14951
|
|
|
14485
14952
|
_CUTOVER_ID_BYTES = _lib_account_protocol.CUTOVER_OP_ID.encode("ascii")
|
|
14953
|
+
_REVISION_ID_BYTES = _lib_account_protocol.REVISION_OP_ID.encode("ascii")
|
|
14486
14954
|
_ANCHOR_KIND_BYTES = (
|
|
14487
14955
|
_lib_account_protocol.CUTOVER_OP_KIND.encode("ascii"),
|
|
14488
14956
|
_lib_account_protocol.LEGACY_SPOOL_ACTIVITY_KIND.encode("ascii"),
|
|
14957
|
+
_lib_account_protocol.REVISION_OP_KIND.encode("ascii"),
|
|
14489
14958
|
)
|
|
14490
14959
|
|
|
14491
14960
|
|
|
@@ -14941,7 +15410,8 @@ def _whole_prefix_admission_boundary():
|
|
|
14941
15410
|
completion abort before they publish instead of disabling admission."""
|
|
14942
15411
|
found = _verified_cutover_from_cache(_read_account_bound_cache())
|
|
14943
15412
|
if found is not None:
|
|
14944
|
-
return
|
|
15413
|
+
return _attach_revision(
|
|
15414
|
+
_lib_account_protocol.boundary_from_op(found[2]))
|
|
14945
15415
|
summaries = read_segment_summaries()
|
|
14946
15416
|
ranges = {}
|
|
14947
15417
|
for seg in _lookup_segments():
|
|
@@ -14960,7 +15430,8 @@ def _whole_prefix_admission_boundary():
|
|
|
14960
15430
|
ranges[seg] = (0, st.st_size)
|
|
14961
15431
|
found = _scan_for_cutover(ranges)
|
|
14962
15432
|
return (None if found is None
|
|
14963
|
-
else
|
|
15433
|
+
else _attach_revision(
|
|
15434
|
+
_lib_account_protocol.boundary_from_op(found[2])))
|
|
14964
15435
|
|
|
14965
15436
|
|
|
14966
15437
|
def find_account_bound_cutover(*, publish=False):
|
|
@@ -14973,7 +15444,9 @@ def find_account_bound_cutover(*, publish=False):
|
|
|
14973
15444
|
found = _locate_account_bound_cutover(publish=publish)
|
|
14974
15445
|
if found is None:
|
|
14975
15446
|
return None
|
|
14976
|
-
|
|
15447
|
+
# #797/#823/#840 (spec 4.6 B3): the revision-2 boundary rides along.
|
|
15448
|
+
return _attach_revision(
|
|
15449
|
+
_lib_account_protocol.boundary_from_op(found[2]), publish=publish)
|
|
14977
15450
|
|
|
14978
15451
|
|
|
14979
15452
|
def _stats_journal_cutover_pending() -> bool:
|
|
@@ -15044,7 +15517,118 @@ def ensure_account_bound_cutover(*, now_utc=None):
|
|
|
15044
15517
|
"""Return the account-bound cutover boundary, establishing it if absent
|
|
15045
15518
|
(spec 5.5). Never establishes a second boundary; a producer that loses the
|
|
15046
15519
|
locked race installs nothing. Raises `AccountBoundCutoverDeferred` while
|
|
15047
|
-
the one-time journal cutover is still pending.
|
|
15520
|
+
the one-time journal cutover is still pending.
|
|
15521
|
+
|
|
15522
|
+
#797/#823/#840 (spec 4.6 B2): every caller is an authoritative producer
|
|
15523
|
+
— the append chokepoint (ingest emission, refresh, manual recording, the
|
|
15524
|
+
dashboard), maintenance planning, recovery and the statusline's detached
|
|
15525
|
+
child, never render — so this also establishes the revision-2 boundary
|
|
15526
|
+
right after the original cutover, and the returned boundary carries it.
|
|
15527
|
+
On a fresh install the revision op follows the cutover op immediately."""
|
|
15528
|
+
boundary = _ensure_original_cutover(now_utc=now_utc)
|
|
15529
|
+
return _establish_account_bound_revision(boundary, now_utc=now_utc)
|
|
15530
|
+
|
|
15531
|
+
|
|
15532
|
+
def establish_revision_before_decision(*, scan: bool = False) -> None:
|
|
15533
|
+
"""Establish the revision-2 boundary before an authoritative decision
|
|
15534
|
+
that does not itself append (#797/#823/#840 spec 4.6 B2): an ingest or
|
|
15535
|
+
rebuild interpreting a store whose original cutover exists but whose
|
|
15536
|
+
revision op does not yet. A store without the original cutover is left
|
|
15537
|
+
exactly as #825 leaves it: its first stamped append establishes both.
|
|
15538
|
+
|
|
15539
|
+
By default (the ingest tick) the original cutover is looked up WITHOUT
|
|
15540
|
+
a journal scan. Proving it ABSENT takes a scan of the whole journal
|
|
15541
|
+
prefix, and doing that on every tick re-read every segment the index has
|
|
15542
|
+
already consumed, which #496 S5b removed: a tick opens only the segments
|
|
15543
|
+
holding its unread records. So the tick asks its verified coordinate
|
|
15544
|
+
cache, then the boundary its own index already holds
|
|
15545
|
+
(`_index_cutover_boundary`; review RV-1: after an upgrade the v2
|
|
15546
|
+
directory starts empty, and deferring every tick until the first stamped
|
|
15547
|
+
append admitted an old-release write in that gap as history). Neither
|
|
15548
|
+
answering, no cutover is known without a scan and the establishment
|
|
15549
|
+
defers like any other unavailable lookup. ``scan=True`` (the completing
|
|
15550
|
+
rebuild, which reads the whole prefix anyway) finds the cutover by a
|
|
15551
|
+
scan when the cache cannot answer.
|
|
15552
|
+
|
|
15553
|
+
A revision op found by a scan is published to the revision cache (review
|
|
15554
|
+
RV-2), so a missing revision cache costs one scan, never one per tick.
|
|
15555
|
+
|
|
15556
|
+
Best-effort: a deferred, unreadable, uncached or contended establishment
|
|
15557
|
+
leaves the decision to run on the prefix it has. Admission is a function
|
|
15558
|
+
of the journal alone, so that prefix is still interpreted consistently;
|
|
15559
|
+
the next producer establishes the boundary (every stamped append runs
|
|
15560
|
+
`ensure_account_bound_cutover`, which also republishes the cache)."""
|
|
15561
|
+
if not account_bound_cutover_enforced():
|
|
15562
|
+
return
|
|
15563
|
+
try:
|
|
15564
|
+
if scan:
|
|
15565
|
+
cutover = find_account_bound_cutover()
|
|
15566
|
+
if cutover is None or cutover.revision is not None:
|
|
15567
|
+
return
|
|
15568
|
+
ensure_account_bound_cutover()
|
|
15569
|
+
return
|
|
15570
|
+
found = _verified_cutover_from_cache(_read_account_bound_cache())
|
|
15571
|
+
original = (_lib_account_protocol.boundary_from_op(found[2])
|
|
15572
|
+
if found is not None else _index_cutover_boundary())
|
|
15573
|
+
if original is None:
|
|
15574
|
+
return
|
|
15575
|
+
cutover = _attach_revision(original, publish=True)
|
|
15576
|
+
if cutover.revision is not None:
|
|
15577
|
+
return
|
|
15578
|
+
# Established from the known cutover directly: the original
|
|
15579
|
+
# cutover's own lookup would scan for it on a cache miss.
|
|
15580
|
+
_establish_account_bound_revision(original)
|
|
15581
|
+
except (AccountBoundCutoverDeferred, AccountBoundCutoverUnavailable,
|
|
15582
|
+
OSError):
|
|
15583
|
+
return
|
|
15584
|
+
|
|
15585
|
+
|
|
15586
|
+
def _index_cutover_boundary():
|
|
15587
|
+
"""The original cutover boundary the live stats index recorded in its
|
|
15588
|
+
admission summary, confirmed against the journal by one line read at its
|
|
15589
|
+
coordinate, or None (no index, no summary boundary, an unproven WAL
|
|
15590
|
+
family, or a coordinate that no longer holds that cutover op).
|
|
15591
|
+
|
|
15592
|
+
The index derived the boundary from the journal itself — it is the one
|
|
15593
|
+
every admission decision of that index applied — so it names the first
|
|
15594
|
+
cutover op without a scan. Read only: the summary is read through a
|
|
15595
|
+
`mode=ro` connection only when the index's WAL family is raw-classified
|
|
15596
|
+
safe (as `read_trusted_derivation_state` reads it)."""
|
|
15597
|
+
import _lib_stats_wal
|
|
15598
|
+
|
|
15599
|
+
path = pathlib.Path(_cctally_core.DB_PATH)
|
|
15600
|
+
if not path.exists():
|
|
15601
|
+
return None
|
|
15602
|
+
wal_index = _lib_stats_wal.inspect_wal_index_family(path)
|
|
15603
|
+
if wal_index.get("verdict") not in {"coherent", "wal_absent", "wal_empty"}:
|
|
15604
|
+
return None
|
|
15605
|
+
try:
|
|
15606
|
+
conn = sqlite3.connect(f"file:{path}?mode=ro", uri=True, timeout=5.0)
|
|
15607
|
+
except sqlite3.Error:
|
|
15608
|
+
return None
|
|
15609
|
+
try:
|
|
15610
|
+
summary = _read_admission_summary(conn)
|
|
15611
|
+
except (sqlite3.Error, TypeError, ValueError, KeyError):
|
|
15612
|
+
return None
|
|
15613
|
+
finally:
|
|
15614
|
+
conn.close()
|
|
15615
|
+
if summary is None or summary.boundary is None:
|
|
15616
|
+
return None
|
|
15617
|
+
original = _lib_account_protocol.with_revision(summary.boundary, None)
|
|
15618
|
+
raw = _read_line_at(original.segment, original.end_offset)
|
|
15619
|
+
rec = None if raw is None else _decode_anchor_record(raw)
|
|
15620
|
+
if rec is None or rec["payload"]["kind"] != (
|
|
15621
|
+
_lib_account_protocol.CUTOVER_OP_KIND):
|
|
15622
|
+
return None
|
|
15623
|
+
try:
|
|
15624
|
+
recorded = _lib_account_protocol.boundary_from_op(rec)
|
|
15625
|
+
except ValueError:
|
|
15626
|
+
return None
|
|
15627
|
+
return recorded if recorded == original else None
|
|
15628
|
+
|
|
15629
|
+
|
|
15630
|
+
def _ensure_original_cutover(*, now_utc=None):
|
|
15631
|
+
"""The #825 establishment of the original cutover (spec 5.5)."""
|
|
15048
15632
|
doc = _read_account_bound_cache()
|
|
15049
15633
|
found = _verified_cutover_from_cache(doc)
|
|
15050
15634
|
if found is not None:
|
|
@@ -15114,7 +15698,9 @@ def ensure_account_bound_cutover(*, now_utc=None):
|
|
|
15114
15698
|
# the cutover op — no digest, no baseline file — so no crash can
|
|
15115
15699
|
# leave a boundary without it.
|
|
15116
15700
|
try:
|
|
15117
|
-
|
|
15701
|
+
# The ORIGINAL inventory: the revision boundary that follows
|
|
15702
|
+
# records the expanded one (#797/#823/#840 spec 4.6 B4).
|
|
15703
|
+
state = capture_legacy_spool_state(include_v1=False)
|
|
15118
15704
|
failure = None
|
|
15119
15705
|
except LegacySpoolInventoryIncomplete as exc:
|
|
15120
15706
|
state = None
|
|
@@ -15208,9 +15794,427 @@ def _frozen_extents(journal_dir, seg_name, seg_created, start) -> dict:
|
|
|
15208
15794
|
return extents
|
|
15209
15795
|
|
|
15210
15796
|
|
|
15211
|
-
|
|
15797
|
+
# --------------------------------------------------------------------------
|
|
15798
|
+
# #797/#823/#840 revision 2 of the account-bound fence (spec 4.6 B1-B4)
|
|
15799
|
+
# --------------------------------------------------------------------------
|
|
15800
|
+
#
|
|
15801
|
+
# The revision boundary is a second durable op, `account_bound_revision` with
|
|
15802
|
+
# the stable id `account-bound-revision-2`, recording with the cutover's codec
|
|
15803
|
+
# the physical position at which it was appended and the EXPANDED legacy spool
|
|
15804
|
+
# baseline digest (the pre-#825 paths plus the retired v1 root's coordination
|
|
15805
|
+
# state). It always follows the original cutover in canonical order: like
|
|
15806
|
+
# every anchor it is appended under `journal.lock` to the canonically-last
|
|
15807
|
+
# segment, so a forward scan from the cutover op is complete. It is located
|
|
15808
|
+
# through its own coordinate cache (`revision-coordinate.json`), which only
|
|
15809
|
+
# accelerates the lookup and never confers authority, and it is established
|
|
15810
|
+
# with #825's 5.5 durability sequence before any authoritative decision —
|
|
15811
|
+
# through `ensure_account_bound_cutover`, never from render.
|
|
15812
|
+
#
|
|
15813
|
+
# Establishment preserves the original boundary, reconciles the existing
|
|
15814
|
+
# legacy baseline (the ORIGINAL inventory against the latest pre-revision
|
|
15815
|
+
# anchor, whose baseline file an older binary may have installed in the v1
|
|
15816
|
+
# root) and journals any difference as an ordinary `legacy_spool_activity`
|
|
15817
|
+
# trigger in the same locked write, carries every unresolved spool marker
|
|
15818
|
+
# forward (markers are journal facts and nothing here clears one), and only
|
|
15819
|
+
# then installs the expanded baseline under the revision op.
|
|
15820
|
+
|
|
15821
|
+
_ACCOUNT_BOUND_REVISION_SRC = "account-bound-revision"
|
|
15822
|
+
_REVISION_CACHE_SCHEMA = 1
|
|
15823
|
+
|
|
15824
|
+
|
|
15825
|
+
def _revision_record(rec):
|
|
15826
|
+
"""``rec`` when it is a stamped revision-2 ``account_bound_revision`` op
|
|
15827
|
+
under its stable id whose payload decodes, else None. An op without
|
|
15828
|
+
``wpr == 2`` is never a revision boundary (spec 4.6 B1)."""
|
|
15829
|
+
if (not isinstance(rec, dict) or rec.get("t") != "op"
|
|
15830
|
+
or not _lib_account_protocol.is_stamped(rec)
|
|
15831
|
+
or not _lib_account_protocol.has_current_revision(rec)
|
|
15832
|
+
or rec.get("id") != _lib_account_protocol.REVISION_OP_ID):
|
|
15833
|
+
return None
|
|
15834
|
+
payload = rec.get("payload")
|
|
15835
|
+
if (not isinstance(payload, dict)
|
|
15836
|
+
or payload.get("kind") != _lib_account_protocol.REVISION_OP_KIND):
|
|
15837
|
+
return None
|
|
15838
|
+
try:
|
|
15839
|
+
_lib_account_protocol.revision_boundary_from_op(rec)
|
|
15840
|
+
except ValueError:
|
|
15841
|
+
return None
|
|
15842
|
+
return rec
|
|
15843
|
+
|
|
15844
|
+
|
|
15845
|
+
def _revision_found_for(found, cutover):
|
|
15846
|
+
"""``found`` (``(segment, offset, record)``) when its revision op records
|
|
15847
|
+
exactly that coordinate as its own and names ``cutover`` (the cutover
|
|
15848
|
+
op's ``(segment, offset)``), else None."""
|
|
15849
|
+
if found is None:
|
|
15850
|
+
return None
|
|
15851
|
+
rec = _revision_record(found[2])
|
|
15852
|
+
if rec is None:
|
|
15853
|
+
return None
|
|
15854
|
+
revision = _lib_account_protocol.revision_boundary_from_op(rec)
|
|
15855
|
+
if ((revision.segment, revision.end_offset) != (found[0], found[1])
|
|
15856
|
+
or (revision.cutover_segment, revision.cutover_offset)
|
|
15857
|
+
!= tuple(cutover)):
|
|
15858
|
+
return None
|
|
15859
|
+
return found
|
|
15860
|
+
|
|
15861
|
+
|
|
15862
|
+
def _scan_for_revision_traversed(cutover, start=None):
|
|
15863
|
+
"""The first revision op after ``cutover`` in canonical order, plus the
|
|
15864
|
+
traversal endpoints (``{segment: complete-line end read}``).
|
|
15865
|
+
|
|
15866
|
+
``start`` maps segments to an offset a previous traversal proved holds no
|
|
15867
|
+
revision op before it; every other segment at or after the cutover's is
|
|
15868
|
+
read from its start (the cutover's own from the cutover op). A segment
|
|
15869
|
+
sorting before the cutover's cannot hold the op and is not read. A read
|
|
15870
|
+
failure raises `AccountBoundCutoverUnavailable` (absence unproven)."""
|
|
15871
|
+
seg0, off0 = cutover
|
|
15872
|
+
floor = _lib_journal.segment_sort_key(seg0)
|
|
15873
|
+
start = start or {}
|
|
15874
|
+
traversed = {}
|
|
15875
|
+
sizes = _segment_sizes()
|
|
15876
|
+
for seg in sorted(sizes, key=_lib_journal.segment_sort_key):
|
|
15877
|
+
if _lib_journal.segment_sort_key(seg) < floor:
|
|
15878
|
+
continue
|
|
15879
|
+
lo = start.get(seg, off0 if seg == seg0 else 0)
|
|
15880
|
+
if seg == seg0:
|
|
15881
|
+
lo = max(lo, off0)
|
|
15882
|
+
hi = sizes[seg]
|
|
15883
|
+
traversed[seg] = lo
|
|
15884
|
+
if hi <= lo:
|
|
15885
|
+
continue
|
|
15886
|
+
try:
|
|
15887
|
+
for _name, off, raw in _iter_segment_lines(
|
|
15888
|
+
_cctally_core.JOURNAL_DIR / seg, lo, hi):
|
|
15889
|
+
if _REVISION_ID_BYTES in raw:
|
|
15890
|
+
found = _revision_found_for(
|
|
15891
|
+
(seg, off, _lib_journal.decode_line(raw)), cutover)
|
|
15892
|
+
if found is not None:
|
|
15893
|
+
traversed[seg] = off
|
|
15894
|
+
return found, traversed
|
|
15895
|
+
traversed[seg] = off + len(raw) + 1
|
|
15896
|
+
except FileNotFoundError:
|
|
15897
|
+
continue
|
|
15898
|
+
except OSError as exc:
|
|
15899
|
+
raise _cutover_lookup_incomplete(seg, exc) from exc
|
|
15900
|
+
return None, traversed
|
|
15901
|
+
|
|
15902
|
+
|
|
15903
|
+
def _read_revision_cache(cutover):
|
|
15904
|
+
"""The revision cache document when it names ``cutover``, else None."""
|
|
15905
|
+
try:
|
|
15906
|
+
doc = json.loads(
|
|
15907
|
+
_cctally_core.ACCOUNT_BOUND_REVISION_CACHE_PATH.read_bytes())
|
|
15908
|
+
except (OSError, ValueError):
|
|
15909
|
+
return None
|
|
15910
|
+
if (not isinstance(doc, dict)
|
|
15911
|
+
or doc.get("schemaVersion") != _REVISION_CACHE_SCHEMA):
|
|
15912
|
+
return None
|
|
15913
|
+
named = _cached_coordinate(doc, "cutover")
|
|
15914
|
+
if named is None or (named["segment"], named["offset"]) != tuple(cutover):
|
|
15915
|
+
return None
|
|
15916
|
+
return doc
|
|
15917
|
+
|
|
15918
|
+
|
|
15919
|
+
def _verified_revision_from_cache(doc, cutover):
|
|
15920
|
+
"""``(segment, offset, record)`` when the cache names a coordinate that
|
|
15921
|
+
holds this cutover's revision op, else None."""
|
|
15922
|
+
coord = _cached_coordinate(doc, "revision")
|
|
15923
|
+
if coord is None:
|
|
15924
|
+
return None
|
|
15925
|
+
raw = _read_line_at(coord["segment"], coord["offset"])
|
|
15926
|
+
if raw is None:
|
|
15927
|
+
return None
|
|
15928
|
+
return _revision_found_for(
|
|
15929
|
+
(coord["segment"], coord["offset"], _lib_journal.decode_line(raw)),
|
|
15930
|
+
cutover)
|
|
15931
|
+
|
|
15932
|
+
|
|
15933
|
+
def _cached_revision_absence(doc):
|
|
15934
|
+
"""The cache's ``{segment: offset}`` absence prefix, or None."""
|
|
15935
|
+
if doc is None or doc.get("revision") is not None:
|
|
15936
|
+
return None
|
|
15937
|
+
absent = doc.get("absentThrough")
|
|
15938
|
+
if not isinstance(absent, dict):
|
|
15939
|
+
return None
|
|
15940
|
+
sizes = None
|
|
15941
|
+
for seg, off in absent.items():
|
|
15942
|
+
if not isinstance(seg, str) or type(off) is not int or off < 0:
|
|
15943
|
+
return None
|
|
15944
|
+
if sizes is None:
|
|
15945
|
+
try:
|
|
15946
|
+
sizes = _segment_sizes()
|
|
15947
|
+
except OSError:
|
|
15948
|
+
return None
|
|
15949
|
+
if seg not in sizes or off > sizes[seg]:
|
|
15950
|
+
return None
|
|
15951
|
+
return absent
|
|
15952
|
+
|
|
15953
|
+
|
|
15954
|
+
def _publish_revision_cache(cutover, revision=None, absent_through=None):
|
|
15955
|
+
"""Best-effort accelerator; a failure only costs the next lookup a scan.
|
|
15956
|
+
Sound for an absence prefix for the same reason the cutover's is: the
|
|
15957
|
+
revision op is only ever appended after such a traversal, past it."""
|
|
15958
|
+
doc = {
|
|
15959
|
+
"schemaVersion": _REVISION_CACHE_SCHEMA,
|
|
15960
|
+
"cutover": {"segment": cutover[0], "offset": cutover[1]},
|
|
15961
|
+
"revision": (None if revision is None else
|
|
15962
|
+
{"segment": revision[0], "offset": revision[1]}),
|
|
15963
|
+
}
|
|
15964
|
+
if revision is None:
|
|
15965
|
+
doc["absentThrough"] = dict(sorted((absent_through or {}).items()))
|
|
15966
|
+
try:
|
|
15967
|
+
_atomic_write_json(_cctally_core.ACCOUNT_BOUND_REVISION_CACHE_PATH, doc)
|
|
15968
|
+
except OSError:
|
|
15969
|
+
pass
|
|
15970
|
+
|
|
15971
|
+
|
|
15972
|
+
def _locate_revision_traversed(cutover, *, publish=False):
|
|
15973
|
+
"""``(found, traversed)`` for the revision op of ``cutover`` (its
|
|
15974
|
+
``(segment, offset)``): through the verified cache, else one scan from
|
|
15975
|
+
the cutover op (only the suffix past a cached absence prefix).
|
|
15976
|
+
``traversed`` is empty when the cache answered."""
|
|
15977
|
+
doc = _read_revision_cache(cutover)
|
|
15978
|
+
found = _verified_revision_from_cache(doc, cutover)
|
|
15979
|
+
if found is not None:
|
|
15980
|
+
return found, {}
|
|
15981
|
+
found, traversed = _scan_for_revision_traversed(
|
|
15982
|
+
cutover, _cached_revision_absence(doc))
|
|
15983
|
+
if publish:
|
|
15984
|
+
if found is not None:
|
|
15985
|
+
try:
|
|
15986
|
+
_make_cutover_durable(found[0])
|
|
15987
|
+
except OSError:
|
|
15988
|
+
return found, traversed
|
|
15989
|
+
_publish_revision_cache(cutover, found[:2])
|
|
15990
|
+
else:
|
|
15991
|
+
_publish_revision_cache(cutover, absent_through=traversed)
|
|
15992
|
+
return found, traversed
|
|
15993
|
+
|
|
15994
|
+
|
|
15995
|
+
def _locate_revision(cutover, *, publish=False):
|
|
15996
|
+
return _locate_revision_traversed(cutover, publish=publish)[0]
|
|
15997
|
+
|
|
15998
|
+
|
|
15999
|
+
def _revision_boundary_of(found):
|
|
16000
|
+
return (None if found is None
|
|
16001
|
+
else _lib_account_protocol.revision_boundary_from_op(found[2]))
|
|
16002
|
+
|
|
16003
|
+
|
|
16004
|
+
def _attach_revision(boundary, *, publish=False):
|
|
16005
|
+
"""``boundary`` (a `CutoverBoundary` or None) carrying the revision-2
|
|
16006
|
+
boundary the journal records after it (spec 4.6 B3)."""
|
|
16007
|
+
if boundary is None:
|
|
16008
|
+
return None
|
|
16009
|
+
cutover = (boundary.segment, boundary.end_offset)
|
|
16010
|
+
return _lib_account_protocol.with_revision(
|
|
16011
|
+
boundary, _revision_boundary_of(_locate_revision(
|
|
16012
|
+
cutover, publish=publish)))
|
|
16013
|
+
|
|
16014
|
+
|
|
16015
|
+
def find_account_bound_revision(*, publish=False):
|
|
16016
|
+
"""The recorded `RevisionBoundary`, or None (no cutover, or no revision
|
|
16017
|
+
yet). A function of the journal alone; see `find_account_bound_cutover`."""
|
|
16018
|
+
boundary = find_account_bound_cutover(publish=publish)
|
|
16019
|
+
return None if boundary is None else boundary.revision
|
|
16020
|
+
|
|
16021
|
+
|
|
16022
|
+
def _read_baseline_document(path, digest):
|
|
16023
|
+
"""The legacy spool state at ``path`` when its digest is ``digest``."""
|
|
16024
|
+
try:
|
|
16025
|
+
state = _lib_legacy_spool.from_document(
|
|
16026
|
+
json.loads(pathlib.Path(path).read_bytes()))
|
|
16027
|
+
except (OSError, ValueError):
|
|
16028
|
+
return None
|
|
16029
|
+
return state if state.digest() == digest else None
|
|
16030
|
+
|
|
16031
|
+
|
|
16032
|
+
def _pre_revision_baseline(digest):
|
|
16033
|
+
"""The installed baseline whose digest is the latest pre-revision anchor's
|
|
16034
|
+
(spec 4.6 B2 "reconciles the existing legacy baseline"): the current
|
|
16035
|
+
root's file (a cutover this binary established) or the retired v1 root's
|
|
16036
|
+
(one the previous release established and advanced). Read only."""
|
|
16037
|
+
if not isinstance(digest, str):
|
|
16038
|
+
return None
|
|
16039
|
+
for path in (_cctally_core.LEGACY_SPOOL_BASELINE_PATH,
|
|
16040
|
+
_cctally_core.STATUSLINE_ACCOUNT_BOUND_V1_DIR
|
|
16041
|
+
/ "legacy-spool-baseline.json"):
|
|
16042
|
+
state = _read_baseline_document(path, digest)
|
|
16043
|
+
if state is not None:
|
|
16044
|
+
return state
|
|
16045
|
+
return None
|
|
16046
|
+
|
|
16047
|
+
|
|
16048
|
+
def _reconciliation_record(cutover_found):
|
|
16049
|
+
"""The `legacy_spool_activity` op (current-stamped, prepared) that
|
|
16050
|
+
reconciles the ORIGINAL legacy inventory with the latest pre-revision
|
|
16051
|
+
anchor, or None when nothing differs, no baseline was ever anchored (a
|
|
16052
|
+
cutover that recorded uncertainty and nothing after it), or the original
|
|
16053
|
+
inventory cannot complete (the revision op then records the failure).
|
|
16054
|
+
Called under `journal.lock`, before the revision op is appended."""
|
|
16055
|
+
(_seg, _off, digest, _sseg, _soff, _uncertain) = _latest_anchor(
|
|
16056
|
+
cutover_found)
|
|
16057
|
+
if digest is None:
|
|
16058
|
+
return None
|
|
16059
|
+
try:
|
|
16060
|
+
original = capture_legacy_spool_state(include_v1=False)
|
|
16061
|
+
except LegacySpoolInventoryIncomplete:
|
|
16062
|
+
return None
|
|
16063
|
+
baseline = _pre_revision_baseline(digest)
|
|
16064
|
+
if baseline is None:
|
|
16065
|
+
reason = "baseline-unavailable"
|
|
16066
|
+
changed = tuple(sorted(entry[0] for entry in original.entries))
|
|
16067
|
+
elif baseline.digest() == original.digest():
|
|
16068
|
+
return None
|
|
16069
|
+
else:
|
|
16070
|
+
reason = "changed"
|
|
16071
|
+
changed = tuple(sorted(set(_lib_legacy_spool.diff(baseline, original))))
|
|
16072
|
+
return _lib_journal.prepare_record(_lib_journal.make_op(
|
|
16073
|
+
at=_now_iso(), src=_LEGACY_SPOOL_ACTIVITY_SRC,
|
|
16074
|
+
payload={
|
|
16075
|
+
"kind": _lib_account_protocol.LEGACY_SPOOL_ACTIVITY_KIND,
|
|
16076
|
+
"reason": reason,
|
|
16077
|
+
"changed": list(changed),
|
|
16078
|
+
"evidence": _lib_legacy_spool.change_evidence(changed, original),
|
|
16079
|
+
"digest": original.digest(),
|
|
16080
|
+
}))
|
|
16081
|
+
|
|
16082
|
+
|
|
16083
|
+
def _establish_account_bound_revision(cutover_boundary, *, now_utc=None):
|
|
16084
|
+
"""Return ``cutover_boundary`` carrying its revision-2 boundary,
|
|
16085
|
+
establishing the revision op when absent (spec 4.6 B2; #825 spec 5.5's
|
|
16086
|
+
durability sequence, unchanged). Never establishes a second revision; a
|
|
16087
|
+
producer that loses the locked race installs nothing."""
|
|
16088
|
+
cutover = (cutover_boundary.segment, cutover_boundary.end_offset)
|
|
16089
|
+
# (1) The verified cache; (2) else a scan of a pinned prefix outside the
|
|
16090
|
+
# lock, whose traversal the locked recheck resumes from.
|
|
16091
|
+
found, traversed = _locate_revision_traversed(cutover, publish=False)
|
|
16092
|
+
if found is not None:
|
|
16093
|
+
if traversed:
|
|
16094
|
+
# Found by a scan, not the verified cache: visible is not
|
|
16095
|
+
# durable (fix round K13), and the cache is worth publishing.
|
|
16096
|
+
# A cache hit is the steady state and writes nothing (#901).
|
|
16097
|
+
_make_cutover_durable(found[0])
|
|
16098
|
+
_publish_revision_cache(cutover, found[:2])
|
|
16099
|
+
return _lib_account_protocol.with_revision(
|
|
16100
|
+
cutover_boundary, _revision_boundary_of(found))
|
|
16101
|
+
_account_bound_seam("revision-after-scan")
|
|
16102
|
+
|
|
16103
|
+
explicit_now = now_utc is not None
|
|
16104
|
+
if now_utc is None:
|
|
16105
|
+
now_utc = _utc_now()
|
|
16106
|
+
journal_dir, dir_created = _journal_dir_for_append()
|
|
16107
|
+
lock_fd = _acquire_leaf_lock()
|
|
16108
|
+
try:
|
|
16109
|
+
# (3) Recheck the suffix the unlocked scan did not traverse.
|
|
16110
|
+
winner, _rechecked = _scan_for_revision_traversed(cutover, traversed)
|
|
16111
|
+
if winner is not None:
|
|
16112
|
+
_make_cutover_durable(winner[0])
|
|
16113
|
+
_publish_revision_cache(cutover, winner[:2])
|
|
16114
|
+
return _lib_account_protocol.with_revision(
|
|
16115
|
+
cutover_boundary, _revision_boundary_of(winner))
|
|
16116
|
+
cutover_found = (cutover[0], cutover[1], _cutover_record_at(cutover))
|
|
16117
|
+
reconciliation = _reconciliation_record(cutover_found)
|
|
16118
|
+
try:
|
|
16119
|
+
state = capture_legacy_spool_state(include_v1=True)
|
|
16120
|
+
failure = None
|
|
16121
|
+
except LegacySpoolInventoryIncomplete as exc:
|
|
16122
|
+
state = None
|
|
16123
|
+
failure = exc.evidence()
|
|
16124
|
+
continued = None
|
|
16125
|
+
if failure is not None and reconciliation is None:
|
|
16126
|
+
# One trigger per failure episode (#825 spec revision 5): an
|
|
16127
|
+
# episode still open before the boundary already has its
|
|
16128
|
+
# uncertainty record, which this failure continues.
|
|
16129
|
+
prior = _latest_anchor(cutover_found)[5]
|
|
16130
|
+
if prior is not None and not _inventory_episode_closed(prior):
|
|
16131
|
+
continued = prior
|
|
16132
|
+
_account_bound_seam("revision-after-baseline-capture")
|
|
16133
|
+
|
|
16134
|
+
seg_name, _seg_path, seg_created, fd = _open_append_target(
|
|
16135
|
+
now_utc, explicit_now)
|
|
16136
|
+
try:
|
|
16137
|
+
if seg_created:
|
|
16138
|
+
try:
|
|
16139
|
+
os.fchmod(fd, 0o600)
|
|
16140
|
+
except OSError:
|
|
16141
|
+
pass
|
|
16142
|
+
_repair_torn_tail(fd)
|
|
16143
|
+
first = os.fstat(fd).st_size
|
|
16144
|
+
head = (b"" if reconciliation is None
|
|
16145
|
+
else _lib_journal.encode_line(reconciliation))
|
|
16146
|
+
start = first + len(head)
|
|
16147
|
+
extents = _frozen_extents(journal_dir, seg_name,
|
|
16148
|
+
seg_created and not head, start)
|
|
16149
|
+
revision = _lib_account_protocol.RevisionBoundary(
|
|
16150
|
+
segment=seg_name,
|
|
16151
|
+
end_offset=start,
|
|
16152
|
+
frozen_extents=extents,
|
|
16153
|
+
legacy_spool_baseline_digest=(
|
|
16154
|
+
None if state is None else state.digest()),
|
|
16155
|
+
cutover_segment=cutover[0],
|
|
16156
|
+
cutover_offset=cutover[1],
|
|
16157
|
+
legacy_spool_inventory_failure=failure,
|
|
16158
|
+
continued_inventory_episode=continued,
|
|
16159
|
+
)
|
|
16160
|
+
record = _lib_journal.make_op(
|
|
16161
|
+
at=_now_iso(), src=_ACCOUNT_BOUND_REVISION_SRC,
|
|
16162
|
+
payload={"kind": _lib_account_protocol.REVISION_OP_KIND,
|
|
16163
|
+
**_lib_account_protocol.revision_boundary_payload(
|
|
16164
|
+
revision)})
|
|
16165
|
+
# A stable semantic id, recognisable without its timestamp.
|
|
16166
|
+
record["id"] = _lib_account_protocol.REVISION_OP_ID
|
|
16167
|
+
line = _lib_journal.encode_line(_lib_journal.prepare_record(record))
|
|
16168
|
+
# The reconciliation trigger and the boundary are one write: no
|
|
16169
|
+
# crash leaves the trigger without the boundary it precedes.
|
|
16170
|
+
_write_all(fd, head + line)
|
|
16171
|
+
_account_bound_seam("revision-after-append")
|
|
16172
|
+
os.fsync(fd)
|
|
16173
|
+
finally:
|
|
16174
|
+
os.close(fd)
|
|
16175
|
+
if seg_created:
|
|
16176
|
+
_fsync_dir(journal_dir)
|
|
16177
|
+
if dir_created:
|
|
16178
|
+
_fsync_dir(journal_dir.parent)
|
|
16179
|
+
_account_bound_seam("revision-after-fsync")
|
|
16180
|
+
if state is not None:
|
|
16181
|
+
_atomic_write_json(_cctally_core.LEGACY_SPOOL_BASELINE_PATH,
|
|
16182
|
+
_lib_legacy_spool.to_document(state))
|
|
16183
|
+
_account_bound_seam("revision-after-baseline-install")
|
|
16184
|
+
finally:
|
|
16185
|
+
_release_leaf_lock(lock_fd)
|
|
16186
|
+
_publish_revision_cache(cutover, (seg_name, start))
|
|
16187
|
+
_publish_account_bound_cache(
|
|
16188
|
+
cutover, (seg_name, start, revision.legacy_spool_baseline_digest),
|
|
16189
|
+
(seg_name, start + len(line)),
|
|
16190
|
+
uncertain=(None if failure is None
|
|
16191
|
+
else continued if continued is not None
|
|
16192
|
+
else (seg_name, start)))
|
|
16193
|
+
return _lib_account_protocol.with_revision(cutover_boundary, revision)
|
|
16194
|
+
|
|
16195
|
+
|
|
16196
|
+
def _cutover_record_at(cutover):
|
|
16197
|
+
"""The cutover op record at its coordinate (it was located before)."""
|
|
16198
|
+
raw = _read_line_at(cutover[0], cutover[1])
|
|
16199
|
+
rec = None if raw is None else _decode_anchor_record(raw)
|
|
16200
|
+
if rec is None or rec["payload"]["kind"] != (
|
|
16201
|
+
_lib_account_protocol.CUTOVER_OP_KIND):
|
|
16202
|
+
raise AccountBoundCutoverUnavailable(
|
|
16203
|
+
errno.EIO, "account-bound cutover op could not be re-read",
|
|
16204
|
+
str(cutover[0]))
|
|
16205
|
+
return rec
|
|
16206
|
+
|
|
16207
|
+
|
|
16208
|
+
def _latest_anchor(cutover, revision=None):
|
|
15212
16209
|
"""``(segment, offset, digest, scanned_segment, scanned_offset,
|
|
15213
16210
|
uncertain)`` of the latest stamped anchoring op at or after ``cutover``.
|
|
16211
|
+
|
|
16212
|
+
#797/#823/#840 (spec 4.6 B2, B3): ``revision`` is the revision-2
|
|
16213
|
+
boundary when one exists. The revision op is itself an anchor (its
|
|
16214
|
+
expanded baseline digest, or inventory uncertainty when its inventory
|
|
16215
|
+
failed), and after it only an op carrying ``wpr == 2`` anchors or opens
|
|
16216
|
+
an uncertainty episode: an older process's post-boundary op is excluded
|
|
16217
|
+
input.
|
|
15214
16218
|
Current anchors are appended only by this binary, always to the
|
|
15215
16219
|
canonically-last segment, so a forward scan from a verified anchor is
|
|
15216
16220
|
complete.
|
|
@@ -15267,13 +16271,29 @@ def _latest_anchor(cutover):
|
|
|
15267
16271
|
if not any(token in raw for token in _ANCHOR_KIND_BYTES):
|
|
15268
16272
|
continue
|
|
15269
16273
|
rec = _lib_journal.decode_line(raw)
|
|
16274
|
+
if _post_revision_without_revision(rec, seg, off, revision):
|
|
16275
|
+
continue
|
|
15270
16276
|
# The cutover op is the initial anchor; only a later stamped
|
|
15271
|
-
# activity op advances it.
|
|
16277
|
+
# activity op, or the revision op, advances it.
|
|
15272
16278
|
anchored = _anchor_record(rec)
|
|
15273
|
-
if anchored is
|
|
15274
|
-
|
|
16279
|
+
kind = None if anchored is None else (
|
|
16280
|
+
anchored["payload"]["kind"])
|
|
16281
|
+
if kind == _lib_account_protocol.LEGACY_SPOOL_ACTIVITY_KIND:
|
|
15275
16282
|
anchor = (seg, off, _anchor_digest(anchored))
|
|
15276
16283
|
uncertain = None
|
|
16284
|
+
elif kind == _lib_account_protocol.REVISION_OP_KIND:
|
|
16285
|
+
anchor = (seg, off, _anchor_digest(anchored))
|
|
16286
|
+
revision_payload = anchored["payload"]
|
|
16287
|
+
episode = revision_payload.get("inventoryEpisode")
|
|
16288
|
+
if revision_payload.get(
|
|
16289
|
+
"legacySpoolInventoryFailure") is None:
|
|
16290
|
+
uncertain = None
|
|
16291
|
+
elif isinstance(episode, dict):
|
|
16292
|
+
# It continues an open episode: that episode's
|
|
16293
|
+
# record stays the uncertainty (and its closure).
|
|
16294
|
+
uncertain = (episode["segment"], episode["offset"])
|
|
16295
|
+
else:
|
|
16296
|
+
uncertain = (seg, off)
|
|
15277
16297
|
elif (_inventory_uncertainty_record(rec) is not None
|
|
15278
16298
|
and rec["payload"]["kind"]
|
|
15279
16299
|
== _lib_account_protocol.LEGACY_SPOOL_ACTIVITY_KIND):
|
|
@@ -15358,6 +16378,12 @@ class LegacySpoolAnchorState:
|
|
|
15358
16378
|
episode_open: bool
|
|
15359
16379
|
|
|
15360
16380
|
|
|
16381
|
+
def _revision_of(cutover):
|
|
16382
|
+
"""The `RevisionBoundary` recorded after the located ``cutover``, or
|
|
16383
|
+
None."""
|
|
16384
|
+
return _revision_boundary_of(_locate_revision(cutover[:2]))
|
|
16385
|
+
|
|
16386
|
+
|
|
15361
16387
|
def legacy_spool_anchor_state() -> "LegacySpoolAnchorState | None":
|
|
15362
16388
|
"""The anchor state, or None when no cutover exists. Raises
|
|
15363
16389
|
`AccountBoundCutoverUnavailable` when a segment it needs is unreadable.
|
|
@@ -15369,7 +16395,7 @@ def legacy_spool_anchor_state() -> "LegacySpoolAnchorState | None":
|
|
|
15369
16395
|
if cutover is None:
|
|
15370
16396
|
return None
|
|
15371
16397
|
(seg, off, digest, scanned_seg, scanned_off,
|
|
15372
|
-
uncertain) = _latest_anchor(cutover)
|
|
16398
|
+
uncertain) = _latest_anchor(cutover, _revision_of(cutover))
|
|
15373
16399
|
if not _anchor_cache_is_current(
|
|
15374
16400
|
cutover[:2], (seg, off, digest), (scanned_seg, scanned_off),
|
|
15375
16401
|
uncertain):
|
|
@@ -15474,7 +16500,7 @@ def record_legacy_spool_inventory_incomplete(failure, *, now_utc=None):
|
|
|
15474
16500
|
lock_fd = _acquire_leaf_lock()
|
|
15475
16501
|
try:
|
|
15476
16502
|
(a_seg, a_off, a_digest, _s_seg, _s_off,
|
|
15477
|
-
uncertain) = _latest_anchor(cutover)
|
|
16503
|
+
uncertain) = _latest_anchor(cutover, _revision_of(cutover))
|
|
15478
16504
|
if (uncertain is not None
|
|
15479
16505
|
and not _inventory_episode_closed(uncertain)):
|
|
15480
16506
|
# Fix round K12 (K11R-7, spec section 7 *Durability*): a
|
|
@@ -15675,20 +16701,46 @@ class _PassBoundary:
|
|
|
15675
16701
|
and BEFORE any interpretation. `resolve` also scans the unvisited suffix
|
|
15676
16702
|
past the pass's high-water, where an op appended after the pin lives."""
|
|
15677
16703
|
|
|
15678
|
-
def __init__(self):
|
|
15679
|
-
|
|
16704
|
+
def __init__(self, cutover=None):
|
|
16705
|
+
#: The cutover boundary: met in the pass, or known beforehand
|
|
16706
|
+
#: (``cutover``, e.g. an index's admission summary).
|
|
16707
|
+
self._boundary = cutover
|
|
16708
|
+
#: #797/#823/#840: the first revision-2 op the pass observed.
|
|
16709
|
+
self._revision = None
|
|
15680
16710
|
|
|
15681
16711
|
def observe(self, record) -> None:
|
|
15682
|
-
if
|
|
15683
|
-
|
|
16712
|
+
if not isinstance(record, dict):
|
|
16713
|
+
return
|
|
16714
|
+
rid = record.get("id")
|
|
16715
|
+
if (self._boundary is None
|
|
16716
|
+
and rid == _lib_account_protocol.CUTOVER_OP_ID):
|
|
15684
16717
|
anchor = _anchor_record(record)
|
|
15685
16718
|
if anchor is not None and anchor["payload"]["kind"] == (
|
|
15686
16719
|
_lib_account_protocol.CUTOVER_OP_KIND):
|
|
15687
|
-
self.
|
|
16720
|
+
self._boundary = _lib_account_protocol.boundary_from_op(
|
|
16721
|
+
anchor)
|
|
16722
|
+
elif (self._revision is None and self._boundary is not None
|
|
16723
|
+
and rid == _lib_account_protocol.REVISION_OP_ID):
|
|
16724
|
+
# A revision op belongs to the cutover the pass already met: it
|
|
16725
|
+
# is appended after it, so a pass sees them in that order.
|
|
16726
|
+
self._revision = _revision_record(record)
|
|
15688
16727
|
|
|
15689
16728
|
def resolve(self, high_water):
|
|
15690
|
-
if self.
|
|
15691
|
-
|
|
16729
|
+
if self._boundary is not None:
|
|
16730
|
+
boundary = _dc_replace(self._boundary, revision=None)
|
|
16731
|
+
revision = None
|
|
16732
|
+
if self._revision is not None:
|
|
16733
|
+
revision = _lib_account_protocol.revision_boundary_from_op(
|
|
16734
|
+
self._revision)
|
|
16735
|
+
if ((revision.cutover_segment, revision.cutover_offset)
|
|
16736
|
+
!= (boundary.segment, boundary.end_offset)):
|
|
16737
|
+
revision = None
|
|
16738
|
+
if revision is None and high_water is not None:
|
|
16739
|
+
# Appended past the pass's high-water (or never observed):
|
|
16740
|
+
# every record of the pass precedes it, and the lookup keeps
|
|
16741
|
+
# what admission consumers see one function of the journal.
|
|
16742
|
+
return _attach_revision(boundary)
|
|
16743
|
+
return _lib_account_protocol.with_revision(boundary, revision)
|
|
15692
16744
|
if high_water is None:
|
|
15693
16745
|
return None
|
|
15694
16746
|
start_key = _lib_journal.segment_sort_key(high_water[0])
|
|
@@ -15700,7 +16752,8 @@ class _PassBoundary:
|
|
|
15700
16752
|
ranges[seg] = (high_water[1] if seg == high_water[0] else 0, size)
|
|
15701
16753
|
found = _scan_for_cutover(ranges)
|
|
15702
16754
|
return (None if found is None
|
|
15703
|
-
else
|
|
16755
|
+
else _attach_revision(
|
|
16756
|
+
_lib_account_protocol.boundary_from_op(found[2])))
|
|
15704
16757
|
|
|
15705
16758
|
|
|
15706
16759
|
def _apply_pass_admission(slots, coordinates, boundary, placeholder=None):
|
|
@@ -15796,6 +16849,10 @@ class AdmissionSummary:
|
|
|
15796
16849
|
coordinate_digest: str | None = None
|
|
15797
16850
|
complete_through: "tuple[str, int] | None" = None
|
|
15798
16851
|
boundary: "object | None" = None
|
|
16852
|
+
#: #797/#823/#840 (spec 4.6 B3): the revision-2 boundary the summary's
|
|
16853
|
+
#: exclusions were classified against (`boundary.revision`), named here
|
|
16854
|
+
#: because a `CutoverBoundary`'s equality is the cutover op's alone.
|
|
16855
|
+
revision_id: str | None = None
|
|
15799
16856
|
|
|
15800
16857
|
def extend(self, excluded) -> None:
|
|
15801
16858
|
if not excluded:
|
|
@@ -15826,15 +16883,30 @@ def _read_admission_summary(conn):
|
|
|
15826
16883
|
if row is None:
|
|
15827
16884
|
return None
|
|
15828
16885
|
boundary = None
|
|
16886
|
+
revision_id = None
|
|
15829
16887
|
if row[8]:
|
|
15830
16888
|
try:
|
|
16889
|
+
doc = json.loads(row[8])
|
|
16890
|
+
revision_doc = doc.pop("revision", None)
|
|
15831
16891
|
boundary = _lib_account_protocol.boundary_from_op({
|
|
15832
16892
|
"t": "op",
|
|
15833
16893
|
"payload": {"kind": _lib_account_protocol.CUTOVER_OP_KIND,
|
|
15834
|
-
**
|
|
16894
|
+
**doc},
|
|
15835
16895
|
})
|
|
15836
|
-
|
|
16896
|
+
if revision_doc is not None:
|
|
16897
|
+
# #797/#823/#840 (spec 4.6 B3): the summary binds both fences.
|
|
16898
|
+
boundary = _lib_account_protocol.with_revision(
|
|
16899
|
+
boundary,
|
|
16900
|
+
_lib_account_protocol.revision_boundary_from_op({
|
|
16901
|
+
"t": "op",
|
|
16902
|
+
"payload": {
|
|
16903
|
+
"kind": _lib_account_protocol.REVISION_OP_KIND,
|
|
16904
|
+
**revision_doc},
|
|
16905
|
+
}))
|
|
16906
|
+
revision_id = _lib_account_protocol.REVISION_OP_ID
|
|
16907
|
+
except (ValueError, TypeError, AttributeError):
|
|
15837
16908
|
boundary = None
|
|
16909
|
+
revision_id = None
|
|
15838
16910
|
|
|
15839
16911
|
def _hw(text):
|
|
15840
16912
|
coord = _coordinate_from_json(text)
|
|
@@ -15854,6 +16926,7 @@ def _read_admission_summary(conn):
|
|
|
15854
16926
|
coordinate_digest=row[6],
|
|
15855
16927
|
complete_through=_hw(row[7]),
|
|
15856
16928
|
boundary=boundary,
|
|
16929
|
+
revision_id=revision_id,
|
|
15857
16930
|
)
|
|
15858
16931
|
|
|
15859
16932
|
|
|
@@ -15881,17 +16954,55 @@ def _write_admission_summary(conn, summary: AdmissionSummary) -> None:
|
|
|
15881
16954
|
summary.coordinate_digest,
|
|
15882
16955
|
_hw(summary.complete_through),
|
|
15883
16956
|
None if summary.boundary is None else json.dumps(
|
|
15884
|
-
|
|
16957
|
+
_summary_boundary_document(summary.boundary),
|
|
15885
16958
|
sort_keys=True, separators=(",", ":")),
|
|
15886
16959
|
),
|
|
15887
16960
|
)
|
|
15888
16961
|
|
|
15889
16962
|
|
|
16963
|
+
def _summary_boundary_document(boundary) -> dict:
|
|
16964
|
+
"""The summary's ``boundary_json``: the cutover payload, plus the
|
|
16965
|
+
revision-2 boundary's under ``revision`` once one applies."""
|
|
16966
|
+
doc = _lib_account_protocol.boundary_payload(boundary)
|
|
16967
|
+
if boundary.revision is not None:
|
|
16968
|
+
doc["revision"] = _lib_account_protocol.revision_boundary_payload(
|
|
16969
|
+
boundary.revision)
|
|
16970
|
+
return doc
|
|
16971
|
+
|
|
16972
|
+
|
|
15890
16973
|
def _evidence_now():
|
|
15891
16974
|
"""The wall clock the legacy-writer evidence records (a test seam)."""
|
|
15892
16975
|
return _utc_now()
|
|
15893
16976
|
|
|
15894
16977
|
|
|
16978
|
+
def read_legacy_writer_evidence_document():
|
|
16979
|
+
"""``(document, seeded)``: the durable legacy-writer evidence sidecar,
|
|
16980
|
+
or — when the v2 sidecar is absent — the previous release's
|
|
16981
|
+
`statusline-account-bound-v1/legacy-writer-evidence.json` as its seed
|
|
16982
|
+
(``seeded`` True), else ``(None, False)``.
|
|
16983
|
+
|
|
16984
|
+
Review RV-5 (#797/#823/#840): the sidecar moved to the v2 directory and
|
|
16985
|
+
nothing carried it over, so the first scan after the upgrade recorded an
|
|
16986
|
+
already-old excluded population as first observed now and the doctor
|
|
16987
|
+
leg warned again for seven days — although a rebuild over unchanged
|
|
16988
|
+
evidence never restarts the age. The v1 file is diagnostic evidence,
|
|
16989
|
+
not coordination state (spec 4.6 B4 forbids importing that), and it is
|
|
16990
|
+
only ever READ: nothing here writes, renames or deletes under v1."""
|
|
16991
|
+
try:
|
|
16992
|
+
return json.loads(
|
|
16993
|
+
_cctally_core.LEGACY_WRITER_EVIDENCE_PATH.read_bytes()), False
|
|
16994
|
+
except FileNotFoundError:
|
|
16995
|
+
pass
|
|
16996
|
+
except (OSError, ValueError):
|
|
16997
|
+
return None, False
|
|
16998
|
+
try:
|
|
16999
|
+
doc = json.loads((_cctally_core.STATUSLINE_ACCOUNT_BOUND_V1_DIR
|
|
17000
|
+
/ "legacy-writer-evidence.json").read_bytes())
|
|
17001
|
+
except (OSError, ValueError):
|
|
17002
|
+
return None, False
|
|
17003
|
+
return doc, True
|
|
17004
|
+
|
|
17005
|
+
|
|
15895
17006
|
def _record_legacy_writer_evidence(summary) -> None:
|
|
15896
17007
|
"""Refresh the durable legacy-writer evidence sidecar from a COMPLETE scan
|
|
15897
17008
|
result (spec section 8): a new excluded population replaces the evidence
|
|
@@ -15909,16 +17020,17 @@ def _record_legacy_writer_evidence(summary) -> None:
|
|
|
15909
17020
|
# new and restart the seven-day age (case F9); the rebuild records it.
|
|
15910
17021
|
return
|
|
15911
17022
|
path = _cctally_core.LEGACY_WRITER_EVIDENCE_PATH
|
|
15912
|
-
|
|
15913
|
-
|
|
15914
|
-
|
|
15915
|
-
|
|
17023
|
+
doc, seeded = read_legacy_writer_evidence_document()
|
|
17024
|
+
existing = (None if doc is None
|
|
17025
|
+
else _lib_legacy_writer_evidence.from_document(doc))
|
|
17026
|
+
if (seeded and existing is not None
|
|
17027
|
+
and existing.cutover_id != summary.cutover_id):
|
|
15916
17028
|
existing = None
|
|
15917
17029
|
updated = _lib_legacy_writer_evidence.observe(
|
|
15918
17030
|
existing, cutover_id=summary.cutover_id,
|
|
15919
17031
|
excluded_count=summary.excluded_total,
|
|
15920
17032
|
coordinate_digest=summary.coordinate_digest, now=_evidence_now())
|
|
15921
|
-
if updated is None or updated is existing:
|
|
17033
|
+
if updated is None or (updated is existing and not seeded):
|
|
15922
17034
|
return
|
|
15923
17035
|
try:
|
|
15924
17036
|
_atomic_write_json(path, _lib_legacy_writer_evidence.to_document(updated))
|
|
@@ -15937,7 +17049,19 @@ def _ingest_admission_boundary(conn, decoded, cursor):
|
|
|
15937
17049
|
index that does not describe its own cursor falls back to the lookup."""
|
|
15938
17050
|
summary = _read_admission_summary(conn)
|
|
15939
17051
|
if summary is not None and summary.boundary is not None:
|
|
15940
|
-
|
|
17052
|
+
if summary.boundary.revision is not None:
|
|
17053
|
+
return summary.boundary
|
|
17054
|
+
# #797/#823/#840 (spec 4.6 B3): the cached summary names the
|
|
17055
|
+
# original boundary only. The revision op is appended like the
|
|
17056
|
+
# cutover op — at the end of the canonically-last segment, after
|
|
17057
|
+
# everything the index consumed — so it is inside this batch or
|
|
17058
|
+
# beyond it, and a batch record before it is historical for it.
|
|
17059
|
+
# Return the cached original boundary with whatever this batch
|
|
17060
|
+
# shows, never the cached summary alone.
|
|
17061
|
+
pass_boundary = _PassBoundary(cutover=summary.boundary)
|
|
17062
|
+
for record, _segment, _offset in decoded:
|
|
17063
|
+
pass_boundary.observe(record)
|
|
17064
|
+
return pass_boundary.resolve(None)
|
|
15941
17065
|
if cursor is None or (
|
|
15942
17066
|
summary is not None and summary.scanned_high_water == cursor):
|
|
15943
17067
|
pass_boundary = _PassBoundary()
|
|
@@ -15963,6 +17087,9 @@ def _advance_admission_summary(conn, boundary, excluded, cursor, target) -> None
|
|
|
15963
17087
|
summary.cutover_id = (
|
|
15964
17088
|
None if boundary is None else _lib_account_protocol.CUTOVER_OP_ID)
|
|
15965
17089
|
summary.boundary = boundary
|
|
17090
|
+
summary.revision_id = (
|
|
17091
|
+
None if boundary is None or boundary.revision is None
|
|
17092
|
+
else _lib_account_protocol.REVISION_OP_ID)
|
|
15966
17093
|
summary.extend(excluded)
|
|
15967
17094
|
summary.scanned_high_water = target
|
|
15968
17095
|
if summary.complete_through is not None:
|