cctally 1.115.0 → 1.116.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. package/CHANGELOG.md +21 -0
  2. package/bin/_cctally_cache.py +85 -4
  3. package/bin/_cctally_completion.py +80 -12
  4. package/bin/_cctally_core.py +106 -9
  5. package/bin/_cctally_dashboard.py +6 -9
  6. package/bin/_cctally_dashboard_envelope.py +26 -32
  7. package/bin/_cctally_db.py +85 -4
  8. package/bin/_cctally_doctor.py +38 -5
  9. package/bin/_cctally_five_hour.py +83 -1017
  10. package/bin/_cctally_journal.py +1307 -180
  11. package/bin/_cctally_journal_repair.py +9 -7
  12. package/bin/_cctally_oauth_samples.py +280 -0
  13. package/bin/_cctally_parser.py +5 -0
  14. package/bin/_cctally_quarantine.py +14 -4
  15. package/bin/_cctally_quota.py +70 -8
  16. package/bin/_cctally_quota_model.py +35 -4
  17. package/bin/_cctally_record.py +414 -123
  18. package/bin/_cctally_rederive.py +632 -235
  19. package/bin/_cctally_refresh.py +29 -0
  20. package/bin/_cctally_statusline.py +984 -47
  21. package/bin/_cctally_store.py +972 -58
  22. package/bin/_cctally_tui.py +62 -7
  23. package/bin/_cctally_update.py +25 -0
  24. package/bin/_lib_account_protocol.py +201 -18
  25. package/bin/_lib_clone_copy.py +46 -11
  26. package/bin/_lib_dashboard_sources.py +57 -15
  27. package/bin/_lib_detector_access_registry.py +273 -52
  28. package/bin/_lib_doctor.py +302 -1
  29. package/bin/_lib_journal.py +145 -25
  30. package/bin/_lib_journal_router.py +166 -48
  31. package/bin/_lib_oauth_samples.py +238 -0
  32. package/bin/_lib_pricing_check.py +0 -12
  33. package/bin/_lib_provider_activity.py +770 -0
  34. package/bin/_lib_quarantine.py +3 -2
  35. package/bin/_lib_quota_model.py +96 -6
  36. package/bin/_lib_rebuild_progress.py +146 -0
  37. package/bin/_lib_record.py +22 -0
  38. package/bin/_lib_rederive.py +299 -3
  39. package/bin/_lib_snapshot_cache.py +43 -1
  40. package/bin/_lib_stale_proof.py +708 -0
  41. package/bin/_lib_statusline_candidates.py +65 -5
  42. package/bin/_lib_worker_launches.py +315 -0
  43. package/bin/cctally +6 -11
  44. package/dashboard/static/assets/{ConversationsView-Bc8LCOAZ.js → ConversationsView-Dflb_myA.js} +1 -1
  45. package/dashboard/static/assets/DoctorModal-DFX5CfIs.js +1 -0
  46. package/dashboard/static/assets/{ModalRoot-DLPgyAM_.js → ModalRoot-HlxoagIa.js} +1 -1
  47. package/dashboard/static/assets/{ProjectsDrillPanel-z53a4v4M.js → ProjectsDrillPanel-C3PJy1xx.js} +1 -1
  48. package/dashboard/static/assets/{SourceDetailModal-Bw0T8W-W.js → SourceDetailModal-BczsWKhD.js} +1 -1
  49. package/dashboard/static/assets/{UpdateModal-D5AKx4Yw.js → UpdateModal-Os-Lo5cs.js} +1 -1
  50. package/dashboard/static/assets/{index-BXDOzSng.js → index-CYZ3NLqF.js} +3 -3
  51. package/dashboard/static/assets/{outlineNavigation-JObY7cUY.js → outlineNavigation-k-zIif6W.js} +1 -1
  52. package/dashboard/static/dashboard.html +1 -1
  53. package/package.json +7 -2
  54. package/bin/_lib_merge_gate.py +0 -452
  55. package/dashboard/static/assets/DoctorModal-D--qVBwe.js +0 -1
@@ -40,6 +40,7 @@ import sqlite3
40
40
  import stat
41
41
  import sys
42
42
  import time
43
+ from array import array
43
44
  from dataclasses import dataclass, field, replace as _dc_replace
44
45
 
45
46
  import _cctally_core
@@ -50,6 +51,7 @@ import _lib_journal
50
51
  import _lib_journal_router
51
52
  import _lib_legacy_spool
52
53
  import _lib_legacy_writer_evidence
54
+ import _lib_rebuild_progress
53
55
  import _lib_record
54
56
  import _lib_segment_summary
55
57
  import _lib_selector_state
@@ -512,12 +514,15 @@ def _require_account_bound_cutover(now_utc=None) -> None:
512
514
  boundary = ensure_account_bound_cutover(now_utc=now_utc)
513
515
  except AccountBoundCutoverDeferred:
514
516
  return
517
+ # #797/#823/#840 (spec 4.6 B2): the memo names the LAST boundary op —
518
+ # the revision op — so a memo hit proves both fences exist.
519
+ last = boundary.revision if boundary.revision is not None else boundary
515
520
  try:
516
- st = os.stat(journal_dir / boundary.segment)
521
+ st = os.stat(journal_dir / last.segment)
517
522
  except OSError:
518
523
  return
519
524
  _ACCOUNT_BOUND_CUTOVER_MEMO = (
520
- str(journal_dir), boundary.segment, int(boundary.end_offset),
525
+ str(journal_dir), last.segment, int(last.end_offset),
521
526
  (st.st_dev, st.st_ino))
522
527
 
523
528
 
@@ -1077,6 +1082,136 @@ def plan_segment_elision(segments, high_water):
1077
1082
  summaries=summaries, covered=covered, verdict=reason)
1078
1083
 
1079
1084
 
1085
+ class QuotaLineSourceError(Exception):
1086
+ """A coordinate no longer names the line the read pass saw (#997 D7)."""
1087
+
1088
+
1089
+ class QuotaLineSource:
1090
+ """The final rebuild's Codex quota observations, as coordinates (#997 D7).
1091
+
1092
+ The read pass used to retain every observation's raw line until the cache
1093
+ leg ran (1.64 GB on the maintainer's journal, alive across the whole pass).
1094
+ This keeps ``(segment, offset, length)`` instead, in per-segment
1095
+ ``array('Q')`` runs, which is 16 bytes per observation, and re-reads exactly
1096
+ the slice the leg asks for. ``_run_bounded_recovery`` therefore keeps its
1097
+ contract: chunk planning by index from ``lengths()`` (no read), a restart
1098
+ at chunk zero re-reads from disk, and only one chunk's bytes are resident.
1099
+
1100
+ ``offset`` is the line's start and ``length`` excludes its newline, the
1101
+ shape ``_iter_segment_lines`` yields. Every read stays inside the extent
1102
+ the pass pinned, because it reads only bytes the pass itself saw.
1103
+ """
1104
+
1105
+ __slots__ = ("_runs", "_count")
1106
+
1107
+ def __init__(self) -> None:
1108
+ #: ``[segment, offsets, lengths]`` per run, in journal order. A run
1109
+ #: holds one segment's consecutive observations; a segment can only
1110
+ #: appear in two runs when the elided refill splices another between.
1111
+ self._runs: list = []
1112
+ self._count = 0
1113
+
1114
+ def append(self, segment: str, offset: int, length: int) -> None:
1115
+ runs = self._runs
1116
+ if not runs or runs[-1][0] != segment:
1117
+ runs.append([segment, array("Q"), array("Q")])
1118
+ runs[-1][1].append(int(offset))
1119
+ runs[-1][2].append(int(length))
1120
+ self._count += 1
1121
+
1122
+ def _extend_run(self, segment, offsets, lengths) -> None:
1123
+ if not offsets:
1124
+ return
1125
+ runs = self._runs
1126
+ if not runs or runs[-1][0] != segment:
1127
+ runs.append([segment, array("Q"), array("Q")])
1128
+ runs[-1][1].extend(offsets)
1129
+ runs[-1][2].extend(lengths)
1130
+ self._count += len(offsets)
1131
+
1132
+ def _copy_range(self, source: "QuotaLineSource", start: int,
1133
+ stop: int) -> None:
1134
+ """Append ``source``'s observations ``[start, stop)`` to this one."""
1135
+ base = 0
1136
+ for segment, offsets, lengths in source._runs:
1137
+ size = len(offsets)
1138
+ lo = max(start - base, 0)
1139
+ hi = min(stop - base, size)
1140
+ if lo < hi:
1141
+ self._extend_run(segment, offsets[lo:hi], lengths[lo:hi])
1142
+ base += size
1143
+ if base >= stop:
1144
+ break
1145
+
1146
+ def __len__(self) -> int:
1147
+ return self._count
1148
+
1149
+ def lengths(self) -> "array":
1150
+ """Every observation's line length (without its newline), no read."""
1151
+ out = array("Q")
1152
+ for _segment, _offsets, lengths in self._runs:
1153
+ out.extend(lengths)
1154
+ return out
1155
+
1156
+ def lines(self, start: int, stop: int) -> list:
1157
+ """Observations ``[start, stop)`` re-read from their segments.
1158
+
1159
+ Raises `QuotaLineSourceError` when a slice is short or no longer framed
1160
+ as one line (a newline before it, unless it starts the segment; one
1161
+ after it; none inside), which is what a truncated or rewritten segment
1162
+ produces. The leg treats that exactly as an unreadable elided segment.
1163
+ """
1164
+ out: list = []
1165
+ base = 0
1166
+ for segment, offsets, lengths in self._runs:
1167
+ size = len(offsets)
1168
+ lo = max(start - base, 0)
1169
+ hi = min(stop - base, size)
1170
+ if lo < hi:
1171
+ path = _cctally_core.JOURNAL_DIR / segment
1172
+ try:
1173
+ with _open_segment_for_read(path) as fh:
1174
+ for index in range(lo, hi):
1175
+ out.append(_read_framed_line(
1176
+ fh, segment, offsets[index], lengths[index]))
1177
+ except OSError as exc:
1178
+ raise QuotaLineSourceError(
1179
+ f"{segment}: {type(exc).__name__}: {exc}") from exc
1180
+ base += size
1181
+ if base >= stop:
1182
+ break
1183
+ return out
1184
+
1185
+
1186
+ def _read_framed_line(fh, segment, offset, length) -> bytes:
1187
+ """The ``length`` bytes at ``offset``, verified to still be one line."""
1188
+ lead = 1 if offset > 0 else 0
1189
+ fh.seek(offset - lead)
1190
+ data = fh.read(length + lead + 1)
1191
+ if (len(data) != length + lead + 1
1192
+ or (lead and data[0:1] != b"\n")
1193
+ or data[-1:] != b"\n"
1194
+ or b"\n" in data[lead:lead + length]):
1195
+ raise QuotaLineSourceError(
1196
+ f"{segment}@{offset}: the line the read pass saw is no longer "
1197
+ "there")
1198
+ return data[lead:lead + length]
1199
+
1200
+
1201
+ def _quota_line_sizes(quota_raw):
1202
+ """Encoded size per observation (line plus newline), for chunk planning."""
1203
+ if isinstance(quota_raw, QuotaLineSource):
1204
+ return array("Q", (length + 1 for length in quota_raw.lengths()))
1205
+ return [len(raw) + 1 for raw in quota_raw]
1206
+
1207
+
1208
+ def _quota_lines(quota_raw, start: int, stop: int) -> list:
1209
+ """Observations ``[start, stop)`` as raw lines, from either input form."""
1210
+ if isinstance(quota_raw, QuotaLineSource):
1211
+ return quota_raw.lines(start, stop)
1212
+ return quota_raw[start:stop]
1213
+
1214
+
1080
1215
  def _refill_elided_quota_raw(quota_raw, gaps, *, failures=None):
1081
1216
  """``(stream, complete)`` — ``quota_raw`` with the elided observations back.
1082
1217
 
@@ -1110,20 +1245,26 @@ def _refill_elided_quota_raw(quota_raw, gaps, *, failures=None):
1110
1245
  a re-read that returns a different number reports the shortfall rather than
1111
1246
  leaving the leg to certify a stream it never saw.
1112
1247
  """
1248
+ coordinates = isinstance(quota_raw, QuotaLineSource)
1113
1249
  by_index: dict = {}
1114
1250
  complete = True
1115
1251
  if failures is None:
1116
1252
  failures = []
1117
1253
  for name, index, extent, expected_lines in gaps:
1118
- recovered = []
1254
+ # #997 D7: a coordinate source gets each recovered observation's
1255
+ # coordinates, not its bytes, so the refill keeps nothing either.
1256
+ recovered = QuotaLineSource() if coordinates else []
1119
1257
  seen = 0
1120
1258
  try:
1121
- for _seg, _off, raw in _iter_segment_lines(
1259
+ for seg, off, raw in _iter_segment_lines(
1122
1260
  _cctally_core.JOURNAL_DIR / name, 0, int(extent)):
1123
1261
  seen += 1
1124
1262
  record = _lib_journal.decode_line(raw)
1125
1263
  if record is not None and _is_codex_quota_obs(record):
1126
- recovered.append(raw)
1264
+ if coordinates:
1265
+ recovered.append(seg, off, len(raw))
1266
+ else:
1267
+ recovered.append(raw)
1127
1268
  except Exception as exc:
1128
1269
  # A vanished segment self-heals, because the pinned vector no longer
1129
1270
  # matches the journal and the certificate is invalid the moment it
@@ -1144,13 +1285,25 @@ def _refill_elided_quota_raw(quota_raw, gaps, *, failures=None):
1144
1285
  continue
1145
1286
  if seen != int(expected_lines):
1146
1287
  complete = False
1147
- by_index.setdefault(int(index), []).extend(recovered)
1288
+ by_index.setdefault(int(index), []).append(recovered)
1289
+ if coordinates:
1290
+ spliced = QuotaLineSource()
1291
+ cursor = 0
1292
+ for index in sorted(by_index):
1293
+ bounded = min(int(index), len(quota_raw))
1294
+ spliced._copy_range(quota_raw, cursor, bounded)
1295
+ for recovered in by_index[index]:
1296
+ spliced._copy_range(recovered, 0, len(recovered))
1297
+ cursor = max(cursor, bounded)
1298
+ spliced._copy_range(quota_raw, cursor, len(quota_raw))
1299
+ return spliced, complete
1148
1300
  out: list = []
1149
1301
  cursor = 0
1150
1302
  for index in sorted(by_index):
1151
1303
  bounded = min(int(index), len(quota_raw))
1152
1304
  out.extend(quota_raw[cursor:bounded])
1153
- out.extend(by_index[index])
1305
+ for recovered in by_index[index]:
1306
+ out.extend(recovered)
1154
1307
  cursor = max(cursor, bounded)
1155
1308
  out.extend(quota_raw[cursor:])
1156
1309
  return out, complete
@@ -1770,16 +1923,26 @@ def _acquire_ingest_lock(mode: str, timeout_s: float) -> int | None:
1770
1923
  # busy-timeout branch closes + returns before this handler can fire, and a
1771
1924
  # successful acquire returns fd, so the fd is closed exactly once.
1772
1925
  try:
1773
- while True:
1774
- try:
1775
- fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
1776
- return fd
1777
- except (BlockingIOError, OSError) as exc:
1778
- if time.monotonic() >= deadline:
1779
- _note_lock_failure(exc)
1780
- os.close(fd)
1781
- return None
1782
- time.sleep(0.02)
1926
+ try:
1927
+ fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
1928
+ return fd
1929
+ except (BlockingIOError, OSError) as exc:
1930
+ if time.monotonic() >= deadline:
1931
+ _note_lock_failure(exc)
1932
+ os.close(fd)
1933
+ return None
1934
+ # #997 D9: contended, so this is a wait a worker reports.
1935
+ with _lib_rebuild_progress.waiting("ingest-lock"):
1936
+ while True:
1937
+ try:
1938
+ fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
1939
+ return fd
1940
+ except (BlockingIOError, OSError) as exc:
1941
+ if time.monotonic() >= deadline:
1942
+ _note_lock_failure(exc)
1943
+ os.close(fd)
1944
+ return None
1945
+ time.sleep(0.02)
1783
1946
  except BaseException:
1784
1947
  os.close(fd)
1785
1948
  raise
@@ -1811,17 +1974,28 @@ def _acquire_maintenance_shared(mode: str, timeout_s: float) -> int | None:
1811
1974
  return None
1812
1975
  deadline = time.monotonic() + timeout_s
1813
1976
  try:
1814
- while True:
1815
- try:
1816
- fcntl.flock(fd, fcntl.LOCK_SH | fcntl.LOCK_NB)
1817
- _cctally_core.note_stats_maintenance_acquired()
1818
- return fd
1819
- except (BlockingIOError, OSError) as exc:
1820
- if time.monotonic() >= deadline:
1821
- _note_lock_failure(exc)
1822
- os.close(fd)
1823
- return None
1824
- time.sleep(0.02)
1977
+ try:
1978
+ fcntl.flock(fd, fcntl.LOCK_SH | fcntl.LOCK_NB)
1979
+ _cctally_core.note_stats_maintenance_acquired()
1980
+ return fd
1981
+ except (BlockingIOError, OSError) as exc:
1982
+ if time.monotonic() >= deadline:
1983
+ _note_lock_failure(exc)
1984
+ os.close(fd)
1985
+ return None
1986
+ # #997 D9: contended, so this is a wait a worker reports.
1987
+ with _lib_rebuild_progress.waiting("maintenance-lock"):
1988
+ while True:
1989
+ try:
1990
+ fcntl.flock(fd, fcntl.LOCK_SH | fcntl.LOCK_NB)
1991
+ _cctally_core.note_stats_maintenance_acquired()
1992
+ return fd
1993
+ except (BlockingIOError, OSError) as exc:
1994
+ if time.monotonic() >= deadline:
1995
+ _note_lock_failure(exc)
1996
+ os.close(fd)
1997
+ return None
1998
+ time.sleep(0.02)
1825
1999
  except BaseException:
1826
2000
  os.close(fd)
1827
2001
  raise
@@ -1846,17 +2020,28 @@ def _acquire_maintenance_exclusive(mode: str, timeout_s: float) -> int | None:
1846
2020
  return None
1847
2021
  deadline = time.monotonic() + timeout_s
1848
2022
  try:
1849
- while True:
1850
- try:
1851
- fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
1852
- _cctally_core.note_stats_maintenance_acquired()
1853
- return fd
1854
- except (BlockingIOError, OSError) as exc:
1855
- if time.monotonic() >= deadline:
1856
- _note_lock_failure(exc)
1857
- os.close(fd)
1858
- return None
1859
- time.sleep(0.02)
2023
+ try:
2024
+ fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
2025
+ _cctally_core.note_stats_maintenance_acquired()
2026
+ return fd
2027
+ except (BlockingIOError, OSError) as exc:
2028
+ if time.monotonic() >= deadline:
2029
+ _note_lock_failure(exc)
2030
+ os.close(fd)
2031
+ return None
2032
+ # #997 D9: contended, so this is a wait a worker reports.
2033
+ with _lib_rebuild_progress.waiting("maintenance-lock"):
2034
+ while True:
2035
+ try:
2036
+ fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
2037
+ _cctally_core.note_stats_maintenance_acquired()
2038
+ return fd
2039
+ except (BlockingIOError, OSError) as exc:
2040
+ if time.monotonic() >= deadline:
2041
+ _note_lock_failure(exc)
2042
+ os.close(fd)
2043
+ return None
2044
+ time.sleep(0.02)
1860
2045
  except BaseException:
1861
2046
  os.close(fd)
1862
2047
  raise
@@ -1957,8 +2142,11 @@ def _write_cursor(conn: sqlite3.Connection, segment: str, offset: int) -> None:
1957
2142
  #: #825/#832 fix round A: the first stats index epoch that carries
1958
2143
  #: `journal_derivation_frontier`. A frontier read from any earlier epoch — in
1959
2144
  #: practice the previous release republishing its own index — is never trusted
1960
- #: as proof that derivation completed (Codex fix-round-1 Q1).
1961
- DERIVATION_FRONTIER_EPOCH = 1018
2145
+ #: as proof that derivation completed (Codex fix-round-1 Q1). #797/#823/#840
2146
+ #: (spec 4.6 B6): raised to 1019 with the revision-2 fence, so a completion
2147
+ #: proof an epoch-1018 binary published, which never applied the revision-2
2148
+ #: admission, is never trusted by this binary.
2149
+ DERIVATION_FRONTIER_EPOCH = 1019
1962
2150
 
1963
2151
  #: #825/#832 revision 2 (#946, Codex design-fix-round-3 Q5): the generation of
1964
2152
  #: the derivation-completeness proof. Generation 1 (stored as NULL) advanced
@@ -2184,6 +2372,21 @@ def _open_segment_for_read(seg_path):
2184
2372
  return open(seg_path, "rb")
2185
2373
 
2186
2374
 
2375
+ def _open_segment_by_name(name):
2376
+ """`_open_segment_for_read` for a segment basename, resolved against the
2377
+ journal directory at call time. The prefix hasher's evidence re-read
2378
+ (#997 D12) comes through here, so it stays visible at the one seam."""
2379
+ return _open_segment_for_read(_cctally_core.JOURNAL_DIR / name)
2380
+
2381
+
2382
+ def _new_prefix_hasher():
2383
+ """A `PrefixHashAccumulator` whose evidence re-reads go through the
2384
+ segment-read seam (#997 D12). Every streaming pass that captures prefix
2385
+ evidence builds its hasher here."""
2386
+ return _lib_journal_router.PrefixHashAccumulator(
2387
+ open_segment=_open_segment_by_name)
2388
+
2389
+
2187
2390
  def _iter_segment_lines(seg_path, lo: int, hi: int, *, on_bytes=None):
2188
2391
  """Stream `(basename, absolute-offset, raw-line-without-newline)` for every
2189
2392
  complete line in `[lo, hi)`, holding at most one chunk plus one partial line
@@ -2255,9 +2458,12 @@ def _iter_range_with_segments(cursor, hw, segments, *, on_segment=None,
2255
2458
  pinned high-water and drives every pass from it, so two passes of the same
2256
2459
  rebuild cannot disagree about the journal's shape.
2257
2460
 
2258
- `on_segment` is called for EVERY segment in the range, including one this
2259
- function then skips because it holds no bytes in range: `journal_prefix_hash`
2260
- frames a zero-byte segment, so a hash accumulator has to be told it exists.
2461
+ `on_segment(seg, size)` is called for EVERY segment in the range, including
2462
+ one this function then skips because it holds no bytes in range:
2463
+ `journal_prefix_hash` frames a zero-byte segment, so a hash accumulator has
2464
+ to be told it exists. `size` is `max(hi - lo, 0)`, the exact byte count
2465
+ `on_bytes` will receive for the segment unless it shrinks under the pass;
2466
+ the streamed prefix hash frames the segment header with it (#997 D12).
2261
2467
  `on_bytes` is forwarded to `_iter_segment_lines`.
2262
2468
 
2263
2469
  `on_extent(seg, lo, hi, stat_result)` is called after this pass has pinned
@@ -2311,7 +2517,7 @@ def _iter_range_with_segments(cursor, hw, segments, *, on_segment=None,
2311
2517
  if on_extent is not None:
2312
2518
  on_extent(seg, lo, hi, stat_result)
2313
2519
  if on_segment is not None:
2314
- on_segment(seg)
2520
+ on_segment(seg, max(hi - lo, 0))
2315
2521
  if lo >= hi:
2316
2522
  continue
2317
2523
  yield from _iter_segment_lines(seg_path, lo, hi, on_bytes=on_bytes)
@@ -4296,6 +4502,43 @@ def fold_policy_of(document, key=FOLD_POLICY_KEY) -> int:
4296
4502
  return FOLD_POLICY_HISTORICAL
4297
4503
 
4298
4504
 
4505
+ # ── Detector policy (#797/#823/#840 spec 4.9, D12) ──────────────────────────
4506
+ #
4507
+ #: The raw Claude usage observation's payload key that records the detector
4508
+ #: revision it was captured under. It is separate from `fold_policy` because
4509
+ #: the previous release's `fold_policy_of` maps every value other than 1 to
4510
+ #: its historical fold: new observations keep `fold_policy: 1` and add this
4511
+ #: additive key, which that release reads past with `.get`. Both detector
4512
+ #: changes of spec 4.9 (the five-hour credit generation in place of the pair
4513
+ #: check, and projection fills that do not advance detector state) are
4514
+ #: versioned together under it.
4515
+ DETECTOR_POLICY_KEY = "detector_policy"
4516
+ #: The same revision on a frozen statusline publication plan, frozen beside
4517
+ #: `foldPolicy` before the publication identity is computed. Recovery copies
4518
+ #: the frozen value or preserves its absence, never the running default.
4519
+ PLAN_DETECTOR_POLICY_KEY = "detectorPolicy"
4520
+ #: The revision newly captured observations and newly frozen plans carry.
4521
+ DETECTOR_POLICY = 2
4522
+
4523
+
4524
+ def detector_policy_of(payload) -> str:
4525
+ """The detector a raw observation payload selects: ``"historical"`` when
4526
+ it records no `detector_policy` (every observation an earlier release
4527
+ journaled: the pair check and projection-fill detection, so a plain
4528
+ rederive of old history adds or removes no credit), ``"v2"`` for exactly
4529
+ the integer 2, and ``"unsupported"`` for any other value — a bool, a
4530
+ float, a string, null or another revision — which selects neither
4531
+ behavior and withholds that observation's detector step (an outcome may
4532
+ always be withheld). Detector policy is semantic data, never transport
4533
+ metadata."""
4534
+ if not isinstance(payload, dict) or DETECTOR_POLICY_KEY not in payload:
4535
+ return "historical"
4536
+ value = payload[DETECTOR_POLICY_KEY]
4537
+ if type(value) is int and value == DETECTOR_POLICY:
4538
+ return "v2"
4539
+ return "unsupported"
4540
+
4541
+
4299
4542
  @dataclass(frozen=True)
4300
4543
  class AxisFloor:
4301
4544
  """One axis's floor for EXACTLY one account, resolved on the caller's
@@ -4916,7 +5159,10 @@ _ACCOUNTS_MACHINERY_KINDS = frozenset(
4916
5159
  # two strict administrative kinds. None carries a target row to stamp.
4917
5160
  "account_bound_cutover", "legacy_spool_activity",
4918
5161
  "journal_protocol_resolution_v2",
4919
- "claude_weekly_observation_decision_v2"))
5162
+ "claude_weekly_observation_decision_v2",
5163
+ # #797/#823/#840 spec 4.6 B2, B5: the revision-2 boundary op and the
5164
+ # successor of the repair acknowledgement.
5165
+ "account_bound_revision", "journal_protocol_resolution_v3"))
4920
5166
 
4921
5167
  # Legacy-classifier exhaustiveness guard (#341, review finding P2-1). EVERY evt
4922
5168
  # kind in `_EVT_SPECS` and every harvest kind in `_HARVEST_SPECS` must carry a
@@ -5076,10 +5322,22 @@ def _normalize_legacy_account_stamp(record, claude_legacy_account: str) -> None:
5076
5322
  # Obs/op fold registry (spec §5.3 "fold op"). Keyed by `payload.kind`; the
5077
5323
  # built-in `_pipeline_op_fold` pipeline hook dispatches through it. 6b may
5078
5324
  # register more op folds; obs no longer fold directly (see the NOTE above).
5325
+ def _apply_op_weekly_observation_decision(conn, record) -> None:
5326
+ """#797/#823/#840 spec 4.3: project a reviewed weekly decision op into
5327
+ `claude_weekly_observation_decisions`, the read-time weekly eligibility
5328
+ of provider samples. It writes nothing else: the decision's accounting
5329
+ effect is still the scratch replay's (`db rederive`)."""
5330
+ import _cctally_oauth_samples
5331
+ _cctally_oauth_samples.apply_weekly_observation_decision_op(conn, record)
5332
+
5333
+
5079
5334
  FOLD_APPLIERS = {
5080
5335
  "weekly_credit_floor": _apply_op_weekly_credit_floor,
5081
5336
  "account_observe": _apply_op_account_observe,
5082
5337
  "account_label": _apply_op_account_label,
5338
+ "claude_weekly_observation_decision": _apply_op_weekly_observation_decision,
5339
+ "claude_weekly_observation_decision_v2": (
5340
+ _apply_op_weekly_observation_decision),
5083
5341
  }
5084
5342
 
5085
5343
 
@@ -5877,6 +6135,16 @@ class ScratchRetention:
5877
6135
  for event_id in event_ids:
5878
6136
  self.applied.setdefault(event_id, _PREDECESSOR_HELD)
5879
6137
 
6138
+ def applied_records(self):
6139
+ """The decisions the projection holds at the current replay point, as
6140
+ records (#840): what each id materialized, or for an id the
6141
+ predecessor index holds its effective record; a tombstone is none."""
6142
+ for event_id, held in list(self.applied.items()):
6143
+ record = (self.effective.get(event_id)
6144
+ if held is _PREDECESSOR_HELD else held)
6145
+ if record is not None:
6146
+ yield record
6147
+
5880
6148
  def _affected(self, evt) -> bool:
5881
6149
  if self.reference is None:
5882
6150
  return False
@@ -5916,7 +6184,8 @@ class ScratchRetention:
5916
6184
  # The predecessor index holds this id as of an earlier
5917
6185
  # revision; the retained decision is its effective one.
5918
6186
  self.applied[event_id] = target
5919
- _rematerialize_scratch_row(conn, event_id, target)
6187
+ _rematerialize_scratch_row(
6188
+ conn, event_id, target, suppressors=self.applied_records())
5920
6189
  return
5921
6190
  self.applied[event_id] = target
5922
6191
  if target is not None and (
@@ -5924,9 +6193,11 @@ class ScratchRetention:
5924
6193
  _apply_evt(conn, target)
5925
6194
 
5926
6195
 
5927
- def _rematerialize_scratch_row(conn, event_id, target) -> None:
6196
+ def _rematerialize_scratch_row(conn, event_id, target, *,
6197
+ suppressors=None) -> None:
5928
6198
  """Bring a scratch projection row to ``target`` (None: remove it). An
5929
- effects-only family has no row to bring, and its effects already ran."""
6199
+ effects-only family has no row to bring, and its effects already ran.
6200
+ ``suppressors`` are the decisions the replay has applied so far (#840)."""
5930
6201
  kind = None if target is None else (target.get("payload") or {}).get("kind")
5931
6202
  tables = ({_EVT_SPECS[kind].table} if kind in _EVT_SPECS
5932
6203
  else {spec.table for spec in _EVT_SPECS.values()})
@@ -5940,7 +6211,8 @@ def _rematerialize_scratch_row(conn, event_id, target) -> None:
5940
6211
  _delete_scratch_row(conn, table, row[0])
5941
6212
  else:
5942
6213
  _converge_row_from_effective(
5943
- conn, event_id, table=table, rowid=row[0], record=target)
6214
+ conn, event_id, table=table, rowid=row[0], record=target,
6215
+ suppressors=suppressors)
5944
6216
  return
5945
6217
 
5946
6218
 
@@ -6251,7 +6523,7 @@ def _emit_harvest_row(ctx, spec, row):
6251
6523
  != _lib_journal.event_content_hash(evt)):
6252
6524
  _converge_row_from_effective(
6253
6525
  conn, chosen["id"], table=spec.table, rowid=row["id"],
6254
- record=chosen)
6526
+ record=chosen, suppressors=retention.applied_records())
6255
6527
  conn.execute(
6256
6528
  f"UPDATE {spec.table} SET journal_id = ? WHERE id = ?",
6257
6529
  (evt["id"], row["id"]),
@@ -7643,9 +7915,77 @@ _CONVERGE_OUTCOME_WORDING: dict = {
7643
7915
  }
7644
7916
 
7645
7917
 
7646
- def _keep_five_hour_retirement(conn, cols: dict, rowid: int) -> None:
7918
+ #: The suppression families by the axis their exact snapshot list retires
7919
+ #: (#840, spec section 4.8): a `week_reset`'s and a `weekly_credit_effects`'
7920
+ #: list retire the WEEKLY axis, a `five_hour_credit`'s the FIVE-HOUR axis.
7921
+ _SUPPRESSOR_KINDS_BY_AXIS = {
7922
+ "weekly": frozenset({"week_reset", "weekly_credit_effects"}),
7923
+ "five_hour": frozenset({"five_hour_credit"}),
7924
+ }
7925
+
7926
+ #: The effective, active decisions whose payload names a snapshot id; the
7927
+ #: membership itself is decided on the decoded payload.
7928
+ _SUPPRESSOR_CANDIDATES_SQL = (
7929
+ "SELECT event_id, event_json FROM journal_effective_events "
7930
+ "WHERE status = 'active' AND event_json LIKE ?")
7931
+
7932
+
7933
+ def _suppression_retires(conn, snapshot_id, axis, *, suppressors=None) -> bool:
7934
+ """Whether an effective suppression decision retires ``axis`` of the
7935
+ snapshot ``snapshot_id`` (#840, spec section 4.8).
7936
+
7937
+ The relation is exact membership: `(suppressor event id and revision,
7938
+ snapshot journal id, retired axis)`. A suppressor counts when it is an
7939
+ admitted, selected, active decision of a family that retires ``axis``,
7940
+ its `suppression` list (on `weekly_usage_snapshots`) names the snapshot's
7941
+ retained id (`sa:<observation-id>` or a bootstrap id, never a rowid), and
7942
+ its effect applies at the current replay point. The decision's EFFECTIVE
7943
+ revision is the one read, so replacing a suppressor updates its
7944
+ memberships and tombstoning it removes them, while another active
7945
+ suppressor can independently keep the retirement. An old event without a
7946
+ suppression list asserts nothing, and no percentage band reconstructs
7947
+ one.
7948
+
7949
+ ``suppressors`` is the replay's own set of applied decisions (a
7950
+ derivation-completion scratch passes what its retention has applied so
7951
+ far, so a future credit's suppression never applies early); ``None``
7952
+ reads the live effective metadata, which ingest writes as each decision
7953
+ folds and a rebuild writes in the same transaction as its folds, before
7954
+ any convergence can run. A rebuild applies the same lists directly in
7955
+ those folds."""
7956
+ kinds = _SUPPRESSOR_KINDS_BY_AXIS[axis]
7957
+ if not isinstance(snapshot_id, str):
7958
+ return False
7959
+ if suppressors is None:
7960
+ records = []
7961
+ for _event_id, event_json in conn.execute(
7962
+ _SUPPRESSOR_CANDIDATES_SQL,
7963
+ ("%" + json.dumps(snapshot_id) + "%",)).fetchall():
7964
+ if event_json is None:
7965
+ continue
7966
+ record = _lib_journal.decode_line(event_json.encode("utf-8"))
7967
+ if record is not None:
7968
+ records.append(record)
7969
+ else:
7970
+ records = suppressors
7971
+ for record in records:
7972
+ if not isinstance(record, dict) or record.get("t") != "evt":
7973
+ continue
7974
+ payload = record.get("payload") or {}
7975
+ if (payload.get("kind") not in kinds
7976
+ or payload.get("suppression_table", "weekly_usage_snapshots")
7977
+ != "weekly_usage_snapshots"):
7978
+ continue
7979
+ if snapshot_id in (payload.get("suppression") or ()):
7980
+ return True
7981
+ return False
7982
+
7983
+
7984
+ def _keep_five_hour_retirement(conn, cols: dict, rowid: int, event_id,
7985
+ *, suppressors=None) -> None:
7647
7986
  """Keep an admitted five-hour retirement through a row convergence
7648
- (#825/#832 spec 4.7 revision 6, fix round K15, review finding K14R-5).
7987
+ (#825/#832 spec 4.7 revision 6, fix round K15, review finding K14R-5;
7988
+ exact membership since #840).
7649
7989
 
7650
7990
  A five-hour credit retires an unheld stale replica in place: its
7651
7991
  five-hour percent, reset and window key are NULL while its
@@ -7655,9 +7995,10 @@ def _keep_five_hour_retirement(conn, cols: dict, rowid: int) -> None:
7655
7995
  values back, so live state would differ from a rebuild, which folds the
7656
7996
  event and then the credit. The row is that retired replica when it is
7657
7997
  unheld, all three five-hour columns are NULL while the payload carries a
7658
- five-hour reading, and the account holds an admitted five-hour credit of
7659
- the payload's window effective at or before its capture. ``cols`` then
7660
- converges every other column and keeps the three NULL."""
7998
+ five-hour reading, and an effective `five_hour_credit` suppression names
7999
+ its snapshot id (`_suppression_retires`). ``cols`` then converges every
8000
+ other column, keeping its accepted weekly evidence, and keeps the three
8001
+ NULL."""
7661
8002
  if cols.get("five_hour_percent") is None or cols.get(
7662
8003
  "five_hour_window_key") is None:
7663
8004
  return
@@ -7668,37 +8009,30 @@ def _keep_five_hour_retirement(conn, cols: dict, rowid: int) -> None:
7668
8009
  if (row is None or row[0] is not None or row[1] is not None
7669
8010
  or row[2] is not None or row[3]):
7670
8011
  return
7671
- credited = conn.execute(
7672
- "SELECT 1 FROM five_hour_reset_events "
7673
- "WHERE account_key = ? AND five_hour_window_key = ? "
7674
- " AND unixepoch(effective_reset_at_utc) <= unixepoch(?) LIMIT 1",
7675
- (cols.get("account_key") or _lib_accounts.UNATTRIBUTED,
7676
- int(cols["five_hour_window_key"]), cols.get("captured_at_utc")),
7677
- ).fetchone()
7678
- if credited is None:
8012
+ if not _suppression_retires(conn, event_id, "five_hour",
8013
+ suppressors=suppressors):
7679
8014
  return
7680
8015
  cols["five_hour_percent"] = None
7681
8016
  cols["five_hour_resets_at"] = None
7682
8017
  cols["five_hour_window_key"] = None
7683
8018
 
7684
8019
 
7685
- def _keep_weekly_retirement(conn, cols: dict, rowid: int) -> None:
8020
+ def _keep_weekly_retirement(conn, cols: dict, rowid: int, event_id,
8021
+ *, suppressors=None) -> None:
7686
8022
  """Keep an admitted weekly retirement through a row convergence (#825/#832
7687
- spec 4.7 revision 7, fix round K16b), the weekly twin of
7688
- `_keep_five_hour_retirement`.
7689
-
7690
- A weekly credit's stale-replica cleanup keeps a band row that carries a
7691
- five-hour reading and makes it weekly-held, while its `snapshot_accept`
7692
- payload still says unheld. Converging the row from that payload would hand
7693
- the retired weekly value back as accepted weekly evidence. The row is that
7694
- converted replica when it is weekly-held while the payload is not, carries
7695
- a five-hour reading, and its capture and weekly value lie in the band of
7696
- an admitted weekly credit of its own account and week: an in-place
7697
- `week_reset_events` row ending that week (inclusive band) or a
7698
- `weekly_credit_floors` row of its start date (strict band), the same
7699
- bands `_cctally_record._credit_retirement_bands` reads and
7700
- `_weekly_value_is_retired_replica` applies. ``cols`` then converges every
7701
- other column and keeps the row held."""
8023
+ spec 4.7 revision 7, fix round K16b; exact membership since #840), the
8024
+ weekly twin of `_keep_five_hour_retirement`.
8025
+
8026
+ A weekly suppression keeps a named row that carries a five-hour reading
8027
+ and makes it weekly-held, while its `snapshot_accept` payload still says
8028
+ unheld. Converging the row from that payload would hand the retired
8029
+ weekly value back as accepted weekly evidence. The row is that converted
8030
+ replica when it is weekly-held while the payload is not, carries a
8031
+ five-hour reading, and an effective `week_reset` or
8032
+ `weekly_credit_effects` suppression names its snapshot id
8033
+ (`_suppression_retires`). ``cols`` then converges every other column,
8034
+ keeping its valid five-hour data, and keeps the row held. An explicitly
8035
+ held payload is converged as it is."""
7702
8036
  if cols.get("weekly_observation_held") not in (None, 0):
7703
8037
  return
7704
8038
  row = conn.execute(
@@ -7707,41 +8041,13 @@ def _keep_weekly_retirement(conn, cols: dict, rowid: int) -> None:
7707
8041
  "FROM weekly_usage_snapshots WHERE id = ?", (rowid,)).fetchone()
7708
8042
  if row is None or not row[0] or row[1] is None or row[2] is None:
7709
8043
  return
7710
- import _cctally_record
7711
- account_key = cols.get("account_key") or _lib_accounts.UNATTRIBUTED
7712
- bands = []
7713
- try:
7714
- week_end = _cctally_core._canonicalize_optional_iso(
7715
- cols.get("week_end_at"), "converge.week_end_at")
7716
- except (TypeError, ValueError):
7717
- week_end = None
7718
- if week_end is not None:
7719
- for floor_at, retired in conn.execute(
7720
- "SELECT effective_reset_at_utc, observed_pre_credit_pct "
7721
- "FROM week_reset_events "
7722
- "WHERE account_key = ? AND new_week_end_at = ? "
7723
- " AND observed_pre_credit_pct IS NOT NULL",
7724
- (account_key, week_end)).fetchall():
7725
- bands.append((str(floor_at), float(retired), True))
7726
- for floor_at, retired in conn.execute(
7727
- "SELECT effective_at_utc, observed_pre_credit_pct "
7728
- "FROM weekly_credit_floors "
7729
- "WHERE week_start_date = ? AND account_key = ? "
7730
- " AND observed_pre_credit_pct IS NOT NULL",
7731
- (cols.get("week_start_date"), account_key)).fetchall():
7732
- bands.append((str(floor_at), float(retired), False))
7733
- try:
7734
- weekly_percent = float(cols.get("weekly_percent"))
7735
- except (TypeError, ValueError):
7736
- return
7737
- if _cctally_record._weekly_value_is_retired_replica(
7738
- bands, captured_at_utc=cols.get("captured_at_utc"),
7739
- weekly_percent=weekly_percent):
8044
+ if _suppression_retires(conn, event_id, "weekly",
8045
+ suppressors=suppressors):
7740
8046
  cols["weekly_observation_held"] = 1
7741
8047
 
7742
8048
 
7743
8049
  def _converge_row_from_effective(conn, event_id, *, table=None, rowid=None,
7744
- record=None) -> str:
8050
+ record=None, suppressors=None) -> str:
7745
8051
  """Bring the live physical row into agreement with the already-journaled
7746
8052
  effective event, and stamp `journal_id` in the SAME operation (#374 §6).
7747
8053
 
@@ -7768,7 +8074,9 @@ def _converge_row_from_effective(conn, event_id, *, table=None, rowid=None,
7768
8074
 
7769
8075
  ``record`` (#825/#832 fix round E) converges to that evt-shaped record
7770
8076
  instead of the live effective metadata's: a derivation-completion scratch
7771
- replay passes the retained effective decision it holds.
8077
+ replay passes the retained effective decision it holds, and with it
8078
+ ``suppressors``, the decisions it has applied so far, against which a
8079
+ retired snapshot's exact suppression membership is decided (#840).
7772
8080
  """
7773
8081
  record = (_effective_event_for_convergence(conn, event_id)
7774
8082
  if record is None else record)
@@ -7809,8 +8117,10 @@ def _converge_row_from_effective(conn, event_id, *, table=None, rowid=None,
7809
8117
  )
7810
8118
  return CONVERGE_DROPPED
7811
8119
  if target == "weekly_usage_snapshots":
7812
- _keep_five_hour_retirement(conn, cols, int(rowid))
7813
- _keep_weekly_retirement(conn, cols, int(rowid))
8120
+ _keep_five_hour_retirement(conn, cols, int(rowid), event_id,
8121
+ suppressors=suppressors)
8122
+ _keep_weekly_retirement(conn, cols, int(rowid), event_id,
8123
+ suppressors=suppressors)
7814
8124
  assignments = ", ".join(f"{name} = ?" for name in cols)
7815
8125
  conn.execute(
7816
8126
  f"UPDATE {target} SET {assignments} WHERE id = ?",
@@ -9526,6 +9836,9 @@ def run_stats_ingest(
9526
9836
  the caller owns the set — and surfaces the same signal with the same manual
9527
9837
  remedy rather than inventing a second recovery path.
9528
9838
  """
9839
+ # #797/#823/#840 spec 4.6 B2: ingest is an authoritative decision, so
9840
+ # the revision-2 boundary is in the journal before it interprets it.
9841
+ establish_revision_before_decision()
9529
9842
  kwargs = {
9530
9843
  "mode": mode,
9531
9844
  "timeout_s": timeout_s,
@@ -9818,6 +10131,8 @@ _REBUILD_REQUIRED_TABLES = frozenset(
9818
10131
  {
9819
10132
  "accounts",
9820
10133
  "budget_milestones",
10134
+ "claude_oauth_samples",
10135
+ "claude_weekly_observation_decisions",
9821
10136
  "five_hour_block_models",
9822
10137
  "five_hour_block_projects",
9823
10138
  "five_hour_blocks",
@@ -9870,6 +10185,8 @@ _REBUILD_REQUIRED_INDEXES = frozenset(
9870
10185
  "idx_budget_milestones_codex_digest",
9871
10186
  "idx_budget_milestones_journal_id",
9872
10187
  "idx_budget_milestones_journal_id_null",
10188
+ "idx_claude_oauth_samples_five_hour",
10189
+ "idx_claude_oauth_samples_weekly",
9873
10190
  "idx_cost_week_start_at_time",
9874
10191
  "idx_cost_week_time",
9875
10192
  "idx_five_hour_block_models_block",
@@ -9933,8 +10250,11 @@ _REBUILD_REQUIRED_INDEXES = frozenset(
9933
10250
  # omitted column, constraint, partial predicate, or index definition. An epoch
9934
10251
  # schema change must update this contract alongside STATS_INDEX_EPOCH.
9935
10252
  # #825/#832 revision 2 (#946): `journal_derivation_frontier.proof_generation`.
10253
+ # #797/#823/#840 epoch 1019: `claude_oauth_samples` and its two indexes,
10254
+ # `claude_weekly_observation_decisions`, and the four D11 identity columns of
10255
+ # `five_hour_credit_confirmation_state`.
9936
10256
  _REBUILD_SCHEMA_FINGERPRINT = (
9937
- "92692b90b8446b0d40aff266bb142b76b0319a749d9e75b2caf469d5461ec952"
10257
+ "6fe811013635e0e23c5c7fc59ed88ec6563664e0a3c5df11b0964c01f253c8fc"
9938
10258
  )
9939
10259
 
9940
10260
 
@@ -10126,7 +10446,7 @@ def stats_index_matches_journal_prefix(
10126
10446
  prior_high_water = None
10127
10447
  cutover_captured = _CUTOVER_UNSEEN
10128
10448
  marker_coordinates: dict = {}
10129
- hasher = _lib_journal_router.PrefixHashAccumulator()
10449
+ hasher = _new_prefix_hasher()
10130
10450
  # #825/#832: the same admission the rebuild applied.
10131
10451
  admission_boundary = _whole_prefix_admission_boundary()
10132
10452
  if high_water is not None:
@@ -10150,8 +10470,8 @@ def stats_index_matches_journal_prefix(
10150
10470
 
10151
10471
  for segment, offset, raw in _iter_range_with_segments(
10152
10472
  None, high_water, segments,
10153
- on_segment=lambda name: hasher.begin_segment(
10154
- name, prior_high_water),
10473
+ on_segment=lambda name, size: hasher.begin_segment(
10474
+ name, prior_high_water, size=size),
10155
10475
  on_bytes=hasher.extend,
10156
10476
  elide=_elide_segment,
10157
10477
  ):
@@ -12002,7 +12322,7 @@ def _resolve_quota_cache_coverage(cache_path, high_water, decoded_end=None):
12002
12322
  def _rebuild_quota_cache_leg_raw(
12003
12323
  quota_raw, decoded, cutover_claude, counters=None, *,
12004
12324
  high_water=None, coverage=None, decoded_end=None, snapshot_out=None,
12005
- quiet=False, elision_gaps=None,
12325
+ quiet=False, elision_gaps=None, hold_stats=None,
12006
12326
  ) -> float:
12007
12327
  """Re-materialize cache.db `quota_window_snapshots` AND the #416 Codex
12008
12328
  attribution map from the journal (spec §5.4 + #416 spec §3.4), fed RAW
@@ -12035,7 +12355,12 @@ def _rebuild_quota_cache_leg_raw(
12035
12355
  begins, exactly as the §3.5 precedence rule requires.
12036
12356
 
12037
12357
  Returns the measured flock hold in seconds, which acceptance criterion 7
12038
- caps.
12358
+ caps. ``hold_stats``, when given, receives ``maxSeconds``: the longest
12359
+ single hold, which #997 D7 adds to the rebuild record.
12360
+
12361
+ ``quota_raw`` is a list of raw lines or, from the final rebuild, a
12362
+ `QuotaLineSource` of coordinates that the recovery re-reads one chunk at a
12363
+ time (#997 D7).
12039
12364
  """
12040
12365
  file_accounts = [
12041
12366
  r for r in decoded if r is not None and _is_codex_file_account_op(r)
@@ -12136,7 +12461,7 @@ def _rebuild_quota_cache_leg_raw(
12136
12461
  quota_raw, file_accounts, cutover_claude, counters,
12137
12462
  cache_path=cache_path, vector=vector, covered=covered,
12138
12463
  high_water=high_water, coverage=coverage, quiet=quiet,
12139
- attributions=attributions,
12464
+ attributions=attributions, hold_stats=hold_stats,
12140
12465
  )
12141
12466
 
12142
12467
 
@@ -12195,7 +12520,7 @@ def _recovery_state(cache):
12195
12520
  def _run_bounded_recovery(
12196
12521
  quota_raw, file_accounts, cutover_claude, counters, *,
12197
12522
  cache_path, vector, covered, high_water, coverage, quiet=False,
12198
- attributions=(),
12523
+ attributions=(), hold_stats=None,
12199
12524
  ) -> float:
12200
12525
  """Recovery as resumable chunks, each capped by bytes AND record count.
12201
12526
 
@@ -12227,7 +12552,16 @@ def _run_bounded_recovery(
12227
12552
  of one pass, so a single recovery reports at most what a single unchunked
12228
12553
  call reports today.
12229
12554
 
12230
- Returns the total measured flock hold across every chunk.
12555
+ Returns the total measured flock hold across every chunk; ``hold_stats``,
12556
+ when given, receives the longest single one as ``maxSeconds``.
12557
+
12558
+ **A coordinate source that can no longer be re-read** (#997 D7: a short,
12559
+ missing or reframed slice) fails the pass the way an unreadable elided
12560
+ segment does: that chunk's observations are not applied, the covered
12561
+ boundary is dropped so nothing is certified over them, and the remaining
12562
+ chunks still apply, ending in `noCoverageEstablished` with `complete`
12563
+ False. The failures are listed in the coverage block's
12564
+ `lineSourceFailures`.
12231
12565
  """
12232
12566
  import _cctally_cache
12233
12567
  from _lib_cache_writer_lock import (
@@ -12238,7 +12572,7 @@ def _run_bounded_recovery(
12238
12572
  pass_id, started_at = _recovery_pass_identity()
12239
12573
  identity_root = _lib_cache_coverage.identity_root(vector)
12240
12574
  spans = _lib_cache_coverage.chunk_spans(
12241
- [len(raw) + 1 for raw in quota_raw],
12575
+ _quota_line_sizes(quota_raw),
12242
12576
  byte_cap=_RECOVERY_CHUNK_BYTES, record_cap=_RECOVERY_CHUNK_RECORDS,
12243
12577
  )
12244
12578
  # NOT cleared on a restart, deliberately. §4.6 requires that one recovery
@@ -12251,6 +12585,8 @@ def _run_bounded_recovery(
12251
12585
  # are snapshotted here and restored on each restart.
12252
12586
  counter_baseline = dict(counters) if counters is not None else None
12253
12587
  hold_seconds = 0.0
12588
+ max_hold_seconds = 0.0
12589
+ source_failures: list = []
12254
12590
  restarts = 0
12255
12591
  chunk_index = 0
12256
12592
  applied = 0
@@ -12306,14 +12642,33 @@ def _run_bounded_recovery(
12306
12642
  decoded_chunk = None
12307
12643
  if span is not None:
12308
12644
  start, stop, _bytes = span
12309
- decoded_chunk = list(_decoded_quota_stream(
12310
- quota_raw[start:stop], cutover_claude, counters))
12645
+ try:
12646
+ decoded_chunk = list(_decoded_quota_stream(
12647
+ _quota_lines(quota_raw, start, stop), cutover_claude,
12648
+ counters))
12649
+ except QuotaLineSourceError as exc:
12650
+ source_failures.append(
12651
+ {"chunk": chunk_index, "error": str(exc)})
12652
+ if not quiet:
12653
+ print(f"[rebuild] quota cache leg re-read failed: {exc}",
12654
+ file=sys.stderr)
12655
+ decoded_chunk = None
12656
+ covered = None
12311
12657
  try:
12658
+ # #997 D9: a first non-blocking try, so free flocks cost no
12659
+ # progress write; only a contended acquisition is a reported wait.
12312
12660
  held = acquire_cache_writer_flocks(
12313
12661
  _cctally_core.CACHE_LOCK_PATH,
12314
12662
  _cctally_core.CACHE_LOCK_CODEX_PATH,
12315
- timeout=15.0,
12663
+ timeout=None,
12316
12664
  )
12665
+ if held is None:
12666
+ with _lib_rebuild_progress.waiting("quota-flocks"):
12667
+ held = acquire_cache_writer_flocks(
12668
+ _cctally_core.CACHE_LOCK_PATH,
12669
+ _cctally_core.CACHE_LOCK_CODEX_PATH,
12670
+ timeout=15.0,
12671
+ )
12317
12672
  except OSError as exc:
12318
12673
  if not quiet:
12319
12674
  print(f"[rebuild] quota cache leg lock failed: {exc}",
@@ -12490,13 +12845,22 @@ def _run_bounded_recovery(
12490
12845
  if cache is not None:
12491
12846
  cache.close()
12492
12847
  finally:
12493
- hold_seconds += time.monotonic() - held_from
12848
+ held_for = time.monotonic() - held_from
12849
+ hold_seconds += held_for
12850
+ max_hold_seconds = max(max_hold_seconds, held_for)
12494
12851
  release_cache_writer_flocks(held)
12495
12852
  decoded_chunk = None
12853
+ _lib_rebuild_progress.tick(
12854
+ chunks=committed_chunks, plannedChunks=len(plan),
12855
+ observations=applied)
12496
12856
  if _RECOVERY_BETWEEN_CHUNKS is not None:
12497
12857
  _RECOVERY_BETWEEN_CHUNKS(committed_chunks)
12498
12858
 
12499
12859
  _record(outcome)
12860
+ if source_failures and coverage is not None:
12861
+ coverage["lineSourceFailures"] = source_failures
12862
+ if hold_stats is not None:
12863
+ hold_stats["maxSeconds"] = max_hold_seconds
12500
12864
  return hold_seconds
12501
12865
 
12502
12866
 
@@ -12731,6 +13095,12 @@ def rebuild_stats_index(
12731
13095
  raise ValueError(
12732
13096
  "complete_derivation applies to the live destination only")
12733
13097
  import _cctally_completion
13098
+ if high_water is None:
13099
+ # #797/#823/#840 spec 4.6 B2: recovery and maintenance rebuilds
13100
+ # are authoritative decisions; the revision-2 boundary is in the
13101
+ # journal before completion reads its prefix, so completion's own
13102
+ # append never races the boundary's.
13103
+ establish_revision_before_decision(scan=True)
12734
13104
  return _cctally_completion.complete_and_rebuild(
12735
13105
  context=context,
12736
13106
  high_water=high_water,
@@ -12952,6 +13322,7 @@ def _rebuild_stats_index_locked(
12952
13322
  "quota_replay")
12953
13323
  }
12954
13324
  quota_lock_hold = 0.0
13325
+ quota_hold_stats: dict = {}
12955
13326
  #: Whether this generation's quota projection was materialized from a cache
12956
13327
  #: whose recovery left an uncovered remainder. Declared out here because the
12957
13328
  #: record and the result are assembled outside the try that sets it.
@@ -12980,9 +13351,11 @@ def _rebuild_stats_index_locked(
12980
13351
  # consumer actually needs: the decision records decoded (5.08% of a
12981
13352
  # production journal) and the Codex quota observations as RAW BYTES
12982
13353
  # (1.64 GB against roughly six gigabytes of dicts). Everything else is
12983
- # dropped as soon as it has contributed (#496 S4 §4).
13354
+ # dropped as soon as it has contributed (#496 S4 §4). #997 D7: the
13355
+ # quota observations are now retained as COORDINATES (16 bytes each)
13356
+ # and the cache leg re-reads them one chunk at a time.
12984
13357
  decoded: list = []
12985
- quota_raw: list = []
13358
+ quota_raw = QuotaLineSource()
12986
13359
  protocol_evidence = []
12987
13360
  prior_high_water = None
12988
13361
  cutover_captured = _CUTOVER_UNSEEN
@@ -12993,15 +13366,25 @@ def _rebuild_stats_index_locked(
12993
13366
  # 64,248 correction records carry far fewer markers than lines.
12994
13367
  marker_coordinates: dict = {}
12995
13368
  last_seen = _lib_journal_router.LastSeenAccumulator()
13369
+ # #797/#823/#840 spec 4.3 D8: this rebuild is apply-only and never runs
13370
+ # the pipeline that projects provider samples live, so the pass offers
13371
+ # every ADMITTED record to the same projection, in bounded batches on
13372
+ # the scratch index.
13373
+ import _cctally_oauth_samples
13374
+ samples = _cctally_oauth_samples.RebuildProjector(conn)
12996
13375
  summaries = _SegmentSummaryCollector()
12997
13376
  # F12 (#496 S5b §5). Planned before the first segment is reached, from
12998
13377
  # the sidecar this pass's predecessor wrote and the coverage certificate
12999
13378
  # Stage 3 mints. A plan that elides nothing is exactly today's pass.
13000
13379
  elision = plan_segment_elision(segments, hw)
13001
- hasher = _lib_journal_router.PrefixHashAccumulator()
13380
+ hasher = _new_prefix_hasher()
13002
13381
  evidence_seconds = 0.0
13003
13382
  prefix = traversal["stats_prefix"]
13004
13383
  read_started = time.monotonic()
13384
+ # #997 D9: progress for a detached worker's state file (a no-op
13385
+ # unless one installed a sink); ticks are throttled to one write per
13386
+ # ten seconds.
13387
+ _lib_rebuild_progress.phase("rebuild:read", lines=0, bytes=0)
13005
13388
 
13006
13389
  def _elide_segment(name, lo, hi, stat_result) -> bool:
13007
13390
  """Contribute an elided segment's share instead of reading it."""
@@ -13044,8 +13427,8 @@ def _rebuild_stats_index_locked(
13044
13427
  if hw is not None:
13045
13428
  for segment, offset, raw in _iter_range_with_segments(
13046
13429
  None, hw, segments,
13047
- on_segment=lambda name: hasher.begin_segment(
13048
- name, prior_high_water),
13430
+ on_segment=lambda name, size: hasher.begin_segment(
13431
+ name, prior_high_water, size=size),
13049
13432
  on_bytes=hasher.extend,
13050
13433
  on_extent=lambda name, lo, hi, st: summaries.begin(
13051
13434
  name, lo, hi, st, last_seen),
@@ -13053,8 +13436,16 @@ def _rebuild_stats_index_locked(
13053
13436
  ):
13054
13437
  prefix["lines"] += 1
13055
13438
  prefix["bytes"] += len(raw) + 1
13439
+ if not prefix["lines"] & 4095:
13440
+ _lib_rebuild_progress.tick(
13441
+ lines=prefix["lines"], bytes=prefix["bytes"])
13056
13442
  summaries.line(raw, offset + len(raw) + 1)
13057
13443
  rec = _lib_journal.decode_line(raw)
13444
+ if (rec is not None and rec.get("t")
13445
+ in _lib_journal_router.RETAINED_RECORD_TYPES):
13446
+ # #997 D15: a record this pass may retain is compacted
13447
+ # before anything holds a reference to it.
13448
+ rec = _lib_journal.compact_retained(rec)
13058
13449
  if rec is None:
13059
13450
  malformed += 1
13060
13451
  summaries.malformed_line()
@@ -13088,6 +13479,7 @@ def _rebuild_stats_index_locked(
13088
13479
  continue
13089
13480
  rebuild_quarantine.on_record(
13090
13481
  rec, segment=segment, offset=offset)
13482
+ samples.offer(rec)
13091
13483
  # `_capture_protocol_prefix_evidence` returns immediately for
13092
13484
  # anything that is not an op, so this guard changes nothing it
13093
13485
  # does — it moves the phase attribution's two clock reads from
@@ -13146,7 +13538,7 @@ def _rebuild_stats_index_locked(
13146
13538
  decoded.append(rec)
13147
13539
  else:
13148
13540
  if update_quota_cache and _is_codex_quota_obs(rec):
13149
- quota_raw.append(raw)
13541
+ quota_raw.append(segment, offset, len(raw))
13150
13542
  # A PLACEHOLDER, not a dropped element. `resolve_effective_events`
13151
13543
  # numbers candidates with `enumerate(records)`, and three of the
13152
13544
  # seven structural violation kinds put that number inside
@@ -13165,6 +13557,8 @@ def _rebuild_stats_index_locked(
13165
13557
  offset + len(raw) + 1,
13166
13558
  )
13167
13559
  summaries.close(last_seen)
13560
+ samples.flush()
13561
+ samples = None
13168
13562
  # §6.3 "recorded, not silent": every elision refusal is a SILENT
13169
13563
  # fallback, and this block is the only surface that says which one.
13170
13564
  traversal["elision"] = elision.counters()
@@ -13193,6 +13587,7 @@ def _rebuild_stats_index_locked(
13193
13587
  # streamed prefix contained it, otherwise from the unvisited suffix alone
13194
13588
  # (falls back to `unattributed` when neither has it), so a fresh
13195
13589
  # single-account rebuild is byte-neutral.
13590
+ _lib_rebuild_progress.phase("rebuild:selection")
13196
13591
  cutover_started = time.monotonic()
13197
13592
  cutover_claude = _resolve_cutover_for_rebuild(
13198
13593
  cutover_captured, hw, all_segments, traversal["cutover_suffix"],
@@ -13238,13 +13633,14 @@ def _rebuild_stats_index_locked(
13238
13633
  # Codex quota obs -> cache.db quota_window_snapshots. The retained bytes
13239
13634
  # are decoded inside the leg's existing transaction and freed here, so
13240
13635
  # the SQLite fold never runs on top of them.
13636
+ _lib_rebuild_progress.phase("rebuild:quota-leg")
13241
13637
  leg_started = time.monotonic()
13242
13638
  if update_quota_cache:
13243
13639
  quota_lock_hold = _rebuild_quota_cache_leg_raw(
13244
13640
  quota_raw, decoded, cutover_claude, traversal["quota_replay"],
13245
13641
  high_water=hw, coverage=quota_coverage,
13246
13642
  decoded_end=prior_high_water, snapshot_out=quota_snapshot,
13247
- elision_gaps=elision.quota_gaps)
13643
+ elision_gaps=elision.quota_gaps, hold_stats=quota_hold_stats)
13248
13644
  quota_raw = []
13249
13645
  phase_seconds["quota_cache_leg"] = round(
13250
13646
  time.monotonic() - leg_started, 6)
@@ -13268,6 +13664,7 @@ def _rebuild_stats_index_locked(
13268
13664
 
13269
13665
  _stats_rebuild_test_pause("rebuild_fold_started")
13270
13666
  fold_started = time.monotonic()
13667
+ _lib_rebuild_progress.phase("rebuild:fold", rows=0)
13271
13668
 
13272
13669
  # Phase 1 (txn A) — structural folds: op floors, snapshot_accept, cost
13273
13670
  # snapshots, resets+suppression, block_close, arming, credit effects.
@@ -13280,6 +13677,8 @@ def _rebuild_stats_index_locked(
13280
13677
  else:
13281
13678
  _apply_evt(conn, rec)
13282
13679
  lines_folded += 1
13680
+ if not lines_folded & 1023:
13681
+ _lib_rebuild_progress.tick(rows=lines_folded)
13283
13682
  conn.commit()
13284
13683
  except BaseException:
13285
13684
  try:
@@ -13318,12 +13717,34 @@ def _rebuild_stats_index_locked(
13318
13717
  # later (see the note on `structural_fold` above). It cannot move any
13319
13718
  # earlier without holding the bundle's 254 MiB of observations (measured
13320
13719
  # on the same store, 232,466 rows) across the structural fold as well.
13720
+ #
13721
+ # #997 D14: the read-pass state is released FIRST, so the bundle's
13722
+ # observations (and, without a snapshot, the projection's own load in
13723
+ # transaction B) reuse its memory instead of stacking on it. The
13724
+ # open-block projection reads that state only through
13725
+ # `journaled_block_first_observations`, so its compact result is
13726
+ # computed here, with the three `effective` fields the result reads
13727
+ # later. `selector_accumulators` goes too: its `SelectorFold`
13728
+ # candidates hold decoded event records. `selector_rows` (materialized
13729
+ # above, for `_validate_selector_state`) and the records `tail` retains
13730
+ # for transaction B stay. The pin on the cache snapshot grows only by
13731
+ # this compact computation; the bundle is still read before phase 2a.
13732
+ # `rec` is the folds' loop variable, still naming the last structural
13733
+ # event.
13734
+ journaled_first = journaled_block_first_observations(
13735
+ decoded, effective.by_id)
13736
+ conflicts = effective.conflicts
13737
+ protocol_violations = effective.protocol_violations
13738
+ acknowledged = effective.acknowledged_protocol_violations
13739
+ decoded = effective = stream = structural = rec = None
13740
+ selector_accumulators = None
13321
13741
  quota_bundle = _read_quota_projection_bundle(quota_snapshot)
13322
13742
 
13323
13743
  # Phase 2a — OPEN 5h block projection (own txn; block-only). Closed blocks
13324
13744
  # came from block_close evts; this materializes the never-closed window(s)
13325
13745
  # so the five_hour_milestone block_id derived_fk resolves. Best-effort
13326
13746
  # (the open block is a projection, §5.3).
13747
+ _lib_rebuild_progress.phase("rebuild:open-blocks")
13327
13748
  projection_started = time.monotonic()
13328
13749
  try:
13329
13750
  cctally = sys.modules.get("cctally")
@@ -13334,15 +13755,15 @@ def _rebuild_stats_index_locked(
13334
13755
  # `snapshot_accept` in journal order, which a credit's
13335
13756
  # stale-replica removal cannot delete and an earlier capture
13336
13757
  # appended later cannot displace.
13337
- bf(conn, only_missing=True,
13338
- journaled_first=journaled_block_first_observations(
13339
- decoded, effective.by_id))
13758
+ bf(conn, only_missing=True, journaled_first=journaled_first)
13340
13759
  except Exception as exc: # pragma: no cover — projection is best-effort
13341
13760
  print(f"[rebuild] open 5h block re-materialization failed: {exc}",
13342
13761
  file=sys.stderr)
13343
13762
  phase_seconds["open_block_projection"] = round(
13344
13763
  time.monotonic() - projection_started, 6)
13764
+ journaled_first = None
13345
13765
 
13766
+ _lib_rebuild_progress.phase("rebuild:stats-fold", rows=lines_folded)
13346
13767
  # Phase 2b + 3 (txn B) — quota projection re-materialization (after the
13347
13768
  # order-45 arming folds) + milestone/budget folds + cursor advance.
13348
13769
  conn.execute("BEGIN IMMEDIATE")
@@ -13372,6 +13793,8 @@ def _rebuild_stats_index_locked(
13372
13793
  for _order, _seq, _kind, rec in tail:
13373
13794
  _apply_evt(conn, rec)
13374
13795
  lines_folded += 1
13796
+ if not lines_folded & 1023:
13797
+ _lib_rebuild_progress.tick(rows=lines_folded)
13375
13798
  # Fold-time `last_seen_utc` derivation (#341): re-derive each
13376
13799
  # account's last-seen from the whole journal. The map was
13377
13800
  # accumulated during the single read pass — `decoded` no longer
@@ -13396,6 +13819,7 @@ def _rebuild_stats_index_locked(
13396
13819
  raise
13397
13820
  phase_seconds["stats_fold"] = round(time.monotonic() - fold_started, 6)
13398
13821
 
13822
+ _lib_rebuild_progress.phase("rebuild:validate")
13399
13823
  validate_started = time.monotonic()
13400
13824
  rows_by_table = {}
13401
13825
  for tbl in _REBUILD_COUNT_TABLES:
@@ -13442,15 +13866,15 @@ def _rebuild_stats_index_locked(
13442
13866
  # database-sized rollback journal while the scratch still exists, so the
13443
13867
  # multi-gigabyte replay peak must not still be resident on top of it.
13444
13868
  segments_read = len(segments)
13445
- conflicts = effective.conflicts
13446
- protocol_violations = effective.protocol_violations
13447
- acknowledged = effective.acknowledged_protocol_violations
13869
+ # `conflicts`, `protocol_violations` and `acknowledged` were captured
13870
+ # before the quota projection bundle loaded, where the read-pass state was
13871
+ # released (#997 D14).
13448
13872
  # The pre-publication window is what F9's memory acceptance is measured
13449
13873
  # over: everything after this point is publication, whose rollback-journal
13450
13874
  # cost S3 already accounts for through projected headroom.
13451
13875
  peak_heap_bytes = (
13452
13876
  tracemalloc.get_traced_memory()[1] if tracing else 0)
13453
- decoded = effective = stream = structural = tail = None
13877
+ tail = None
13454
13878
  segments = protocol_evidence = last_seen = None
13455
13879
 
13456
13880
  # First fresh-connection validation (#496 S1 F1). A failure here raises
@@ -13465,6 +13889,7 @@ def _rebuild_stats_index_locked(
13465
13889
  scratch, phase="fresh pre-publication validation"
13466
13890
  )
13467
13891
 
13892
+ _lib_rebuild_progress.phase("rebuild:publish")
13468
13893
  publication_started = time.monotonic()
13469
13894
  incident = _publish_rebuilt_stats_index(
13470
13895
  scratch=scratch,
@@ -13500,7 +13925,14 @@ def _rebuild_stats_index_locked(
13500
13925
  name: dict(counts) for name, counts in traversal.items()
13501
13926
  },
13502
13927
  "peakHeapBytes": peak_heap_bytes,
13928
+ # #997 D11: the process-lifetime resident peak, which for the
13929
+ # detached worker covers completion and this rebuild.
13930
+ "peakResidentBytes": _lib_rebuild_progress.peak_resident_bytes(),
13503
13931
  "quotaLockHoldSeconds": round(quota_lock_hold, 6),
13932
+ # #997 D7: the longest single hold; the field above stays the
13933
+ # cumulative hold across chunks and retries.
13934
+ "quotaLockMaxHoldSeconds": round(
13935
+ quota_hold_stats.get("maxSeconds", 0.0), 6),
13504
13936
  "selectorDesynchronized": selector_desync,
13505
13937
  "quotaCacheCoverage": quota_coverage,
13506
13938
  # §4.7: distinct from `quotaCacheCoverage`, which describes the
@@ -14277,10 +14709,19 @@ def _account_bound_seam(step: str) -> None:
14277
14709
  seam(step)
14278
14710
 
14279
14711
 
14280
- def _legacy_spool_paths() -> list:
14281
- """The previous release's statusline coordination paths (spec section 7)."""
14712
+ def _legacy_spool_paths(*, include_v1: bool = True) -> list:
14713
+ """The previous releases' statusline coordination paths (spec section 7).
14714
+
14715
+ #797/#823/#840 (spec 4.6 B4): the retired v1 account-bound root's
14716
+ coordination state — its ``accounts/`` tree and ``transport.last`` —
14717
+ joins the pre-#825 paths. The v1 root's own caches, baseline and
14718
+ diagnostics carry no accounting input and stay out, so an older process
14719
+ refreshing a diagnostic file raises no trigger. ``include_v1=False`` is
14720
+ the ORIGINAL inventory the cutover op and its anchors describe; the
14721
+ revision boundary records the expanded one, and every rescan after it
14722
+ compares the expanded inventory."""
14282
14723
  core = _cctally_core
14283
- return [
14724
+ paths = [
14284
14725
  core.STATUSLINE_OBSERVE_MARKER_PATH,
14285
14726
  core.STATUSLINE_PERSIST_LOCK_PATH,
14286
14727
  core.STATUSLINE_CANDIDATE_DIR,
@@ -14289,6 +14730,12 @@ def _legacy_spool_paths() -> list:
14289
14730
  core.STATUSLINE_AUTHORITATIVE_7D_PATH,
14290
14731
  core.STATUSLINE_AUTHORITATIVE_5H_PATH,
14291
14732
  ]
14733
+ if include_v1:
14734
+ paths.extend([
14735
+ core.STATUSLINE_ACCOUNT_BOUND_V1_DIR / "accounts",
14736
+ core.STATUSLINE_ACCOUNT_BOUND_V1_DIR / "transport.last",
14737
+ ])
14738
+ return paths
14292
14739
 
14293
14740
 
14294
14741
  def _legacy_relative_name(path) -> str:
@@ -14341,9 +14788,9 @@ def _legacy_file_entry(path, rel: str):
14341
14788
  _lib_legacy_spool.content_digest(data))
14342
14789
 
14343
14790
 
14344
- def capture_legacy_spool_state():
14791
+ def capture_legacy_spool_state(*, include_v1: bool = True):
14345
14792
  """Observe the legacy statusline spool. Content is read only to digest it;
14346
- it is never accounting input.
14793
+ it is never accounting input. ``include_v1`` as in `_legacy_spool_paths`.
14347
14794
 
14348
14795
  Complete or raising (review cycle 7 825-832-CD7-J01): a spool path that
14349
14796
  is absent, or an entry that vanished since its directory was listed, is
@@ -14361,7 +14808,7 @@ def capture_legacy_spool_state():
14361
14808
  raise _legacy_inventory_incomplete(
14362
14809
  getattr(exc, "filename", None) or walking[0], exc, "list") from exc
14363
14810
 
14364
- for path in _legacy_spool_paths():
14811
+ for path in _legacy_spool_paths(include_v1=include_v1):
14365
14812
  path = pathlib.Path(path)
14366
14813
  walking[0] = path
14367
14814
  try:
@@ -14393,8 +14840,9 @@ def _decode_anchor_record(raw: bytes):
14393
14840
 
14394
14841
 
14395
14842
  def _anchor_record(rec):
14396
- """``rec`` when it is a stamped cutover or legacy-spool-activity op, else
14397
- None."""
14843
+ """``rec`` when it is a stamped cutover, revision or legacy-spool-activity
14844
+ op, else None. Position-free: after the revision boundary a reader also
14845
+ requires ``wpr == 2`` (`_post_revision_without_revision`)."""
14398
14846
  if (not isinstance(rec, dict) or rec.get("t") != "op"
14399
14847
  or not _lib_account_protocol.is_stamped(rec)):
14400
14848
  return None
@@ -14402,6 +14850,8 @@ def _anchor_record(rec):
14402
14850
  if not isinstance(payload, dict):
14403
14851
  return None
14404
14852
  kind = payload.get("kind")
14853
+ if kind == _lib_account_protocol.REVISION_OP_KIND:
14854
+ return _revision_record(rec)
14405
14855
  if kind == _lib_account_protocol.CUTOVER_OP_KIND:
14406
14856
  if rec.get("id") != _lib_account_protocol.CUTOVER_OP_ID:
14407
14857
  return None
@@ -14437,7 +14887,9 @@ def _inventory_uncertainty_record(rec):
14437
14887
  return rec
14438
14888
  return None
14439
14889
  anchor = _anchor_record(rec)
14440
- if (anchor is not None and kind == _lib_account_protocol.CUTOVER_OP_KIND
14890
+ if (anchor is not None
14891
+ and kind in (_lib_account_protocol.CUTOVER_OP_KIND,
14892
+ _lib_account_protocol.REVISION_OP_KIND)
14441
14893
  and payload.get("legacySpoolInventoryFailure") is not None):
14442
14894
  return rec
14443
14895
  return None
@@ -14453,11 +14905,26 @@ def _cutover_uncertainty(found):
14453
14905
 
14454
14906
  def _anchor_digest(rec) -> str:
14455
14907
  payload = rec["payload"]
14456
- if payload["kind"] == _lib_account_protocol.CUTOVER_OP_KIND:
14908
+ if payload["kind"] in (_lib_account_protocol.CUTOVER_OP_KIND,
14909
+ _lib_account_protocol.REVISION_OP_KIND):
14457
14910
  return payload["legacySpoolBaselineDigest"]
14458
14911
  return payload["digest"]
14459
14912
 
14460
14913
 
14914
+ def _post_revision_without_revision(rec, segment, offset, revision) -> bool:
14915
+ """Whether ``rec`` at ``(segment, offset)`` lies after the revision-2
14916
+ boundary without carrying ``wpr == 2`` (spec 4.6 B3): such a record is
14917
+ excluded input, so an older process's anchoring or uncertainty op there
14918
+ never advances or opens anything ("old post-boundary anchoring ops never
14919
+ advance the revision-2 baseline")."""
14920
+ if revision is None:
14921
+ return False
14922
+ if _lib_account_protocol.is_historical(
14923
+ _lib_account_protocol.Coordinate(segment, offset), revision):
14924
+ return False
14925
+ return not _lib_account_protocol.has_current_revision(rec)
14926
+
14927
+
14461
14928
  def _read_line_at(segment: str, offset: int):
14462
14929
  """The raw complete line starting at ``(segment, offset)``, or None."""
14463
14930
  if not isinstance(segment, str) or type(offset) is not int or offset < 0:
@@ -14483,9 +14950,11 @@ def _read_line_at(segment: str, offset: int):
14483
14950
 
14484
14951
 
14485
14952
  _CUTOVER_ID_BYTES = _lib_account_protocol.CUTOVER_OP_ID.encode("ascii")
14953
+ _REVISION_ID_BYTES = _lib_account_protocol.REVISION_OP_ID.encode("ascii")
14486
14954
  _ANCHOR_KIND_BYTES = (
14487
14955
  _lib_account_protocol.CUTOVER_OP_KIND.encode("ascii"),
14488
14956
  _lib_account_protocol.LEGACY_SPOOL_ACTIVITY_KIND.encode("ascii"),
14957
+ _lib_account_protocol.REVISION_OP_KIND.encode("ascii"),
14489
14958
  )
14490
14959
 
14491
14960
 
@@ -14941,7 +15410,8 @@ def _whole_prefix_admission_boundary():
14941
15410
  completion abort before they publish instead of disabling admission."""
14942
15411
  found = _verified_cutover_from_cache(_read_account_bound_cache())
14943
15412
  if found is not None:
14944
- return _lib_account_protocol.boundary_from_op(found[2])
15413
+ return _attach_revision(
15414
+ _lib_account_protocol.boundary_from_op(found[2]))
14945
15415
  summaries = read_segment_summaries()
14946
15416
  ranges = {}
14947
15417
  for seg in _lookup_segments():
@@ -14960,7 +15430,8 @@ def _whole_prefix_admission_boundary():
14960
15430
  ranges[seg] = (0, st.st_size)
14961
15431
  found = _scan_for_cutover(ranges)
14962
15432
  return (None if found is None
14963
- else _lib_account_protocol.boundary_from_op(found[2]))
15433
+ else _attach_revision(
15434
+ _lib_account_protocol.boundary_from_op(found[2])))
14964
15435
 
14965
15436
 
14966
15437
  def find_account_bound_cutover(*, publish=False):
@@ -14973,7 +15444,9 @@ def find_account_bound_cutover(*, publish=False):
14973
15444
  found = _locate_account_bound_cutover(publish=publish)
14974
15445
  if found is None:
14975
15446
  return None
14976
- return _lib_account_protocol.boundary_from_op(found[2])
15447
+ # #797/#823/#840 (spec 4.6 B3): the revision-2 boundary rides along.
15448
+ return _attach_revision(
15449
+ _lib_account_protocol.boundary_from_op(found[2]), publish=publish)
14977
15450
 
14978
15451
 
14979
15452
  def _stats_journal_cutover_pending() -> bool:
@@ -15044,7 +15517,118 @@ def ensure_account_bound_cutover(*, now_utc=None):
15044
15517
  """Return the account-bound cutover boundary, establishing it if absent
15045
15518
  (spec 5.5). Never establishes a second boundary; a producer that loses the
15046
15519
  locked race installs nothing. Raises `AccountBoundCutoverDeferred` while
15047
- the one-time journal cutover is still pending."""
15520
+ the one-time journal cutover is still pending.
15521
+
15522
+ #797/#823/#840 (spec 4.6 B2): every caller is an authoritative producer
15523
+ — the append chokepoint (ingest emission, refresh, manual recording, the
15524
+ dashboard), maintenance planning, recovery and the statusline's detached
15525
+ child, never render — so this also establishes the revision-2 boundary
15526
+ right after the original cutover, and the returned boundary carries it.
15527
+ On a fresh install the revision op follows the cutover op immediately."""
15528
+ boundary = _ensure_original_cutover(now_utc=now_utc)
15529
+ return _establish_account_bound_revision(boundary, now_utc=now_utc)
15530
+
15531
+
15532
+ def establish_revision_before_decision(*, scan: bool = False) -> None:
15533
+ """Establish the revision-2 boundary before an authoritative decision
15534
+ that does not itself append (#797/#823/#840 spec 4.6 B2): an ingest or
15535
+ rebuild interpreting a store whose original cutover exists but whose
15536
+ revision op does not yet. A store without the original cutover is left
15537
+ exactly as #825 leaves it: its first stamped append establishes both.
15538
+
15539
+ By default (the ingest tick) the original cutover is looked up WITHOUT
15540
+ a journal scan. Proving it ABSENT takes a scan of the whole journal
15541
+ prefix, and doing that on every tick re-read every segment the index has
15542
+ already consumed, which #496 S5b removed: a tick opens only the segments
15543
+ holding its unread records. So the tick asks its verified coordinate
15544
+ cache, then the boundary its own index already holds
15545
+ (`_index_cutover_boundary`; review RV-1: after an upgrade the v2
15546
+ directory starts empty, and deferring every tick until the first stamped
15547
+ append admitted an old-release write in that gap as history). Neither
15548
+ answering, no cutover is known without a scan and the establishment
15549
+ defers like any other unavailable lookup. ``scan=True`` (the completing
15550
+ rebuild, which reads the whole prefix anyway) finds the cutover by a
15551
+ scan when the cache cannot answer.
15552
+
15553
+ A revision op found by a scan is published to the revision cache (review
15554
+ RV-2), so a missing revision cache costs one scan, never one per tick.
15555
+
15556
+ Best-effort: a deferred, unreadable, uncached or contended establishment
15557
+ leaves the decision to run on the prefix it has. Admission is a function
15558
+ of the journal alone, so that prefix is still interpreted consistently;
15559
+ the next producer establishes the boundary (every stamped append runs
15560
+ `ensure_account_bound_cutover`, which also republishes the cache)."""
15561
+ if not account_bound_cutover_enforced():
15562
+ return
15563
+ try:
15564
+ if scan:
15565
+ cutover = find_account_bound_cutover()
15566
+ if cutover is None or cutover.revision is not None:
15567
+ return
15568
+ ensure_account_bound_cutover()
15569
+ return
15570
+ found = _verified_cutover_from_cache(_read_account_bound_cache())
15571
+ original = (_lib_account_protocol.boundary_from_op(found[2])
15572
+ if found is not None else _index_cutover_boundary())
15573
+ if original is None:
15574
+ return
15575
+ cutover = _attach_revision(original, publish=True)
15576
+ if cutover.revision is not None:
15577
+ return
15578
+ # Established from the known cutover directly: the original
15579
+ # cutover's own lookup would scan for it on a cache miss.
15580
+ _establish_account_bound_revision(original)
15581
+ except (AccountBoundCutoverDeferred, AccountBoundCutoverUnavailable,
15582
+ OSError):
15583
+ return
15584
+
15585
+
15586
+ def _index_cutover_boundary():
15587
+ """The original cutover boundary the live stats index recorded in its
15588
+ admission summary, confirmed against the journal by one line read at its
15589
+ coordinate, or None (no index, no summary boundary, an unproven WAL
15590
+ family, or a coordinate that no longer holds that cutover op).
15591
+
15592
+ The index derived the boundary from the journal itself — it is the one
15593
+ every admission decision of that index applied — so it names the first
15594
+ cutover op without a scan. Read only: the summary is read through a
15595
+ `mode=ro` connection only when the index's WAL family is raw-classified
15596
+ safe (as `read_trusted_derivation_state` reads it)."""
15597
+ import _lib_stats_wal
15598
+
15599
+ path = pathlib.Path(_cctally_core.DB_PATH)
15600
+ if not path.exists():
15601
+ return None
15602
+ wal_index = _lib_stats_wal.inspect_wal_index_family(path)
15603
+ if wal_index.get("verdict") not in {"coherent", "wal_absent", "wal_empty"}:
15604
+ return None
15605
+ try:
15606
+ conn = sqlite3.connect(f"file:{path}?mode=ro", uri=True, timeout=5.0)
15607
+ except sqlite3.Error:
15608
+ return None
15609
+ try:
15610
+ summary = _read_admission_summary(conn)
15611
+ except (sqlite3.Error, TypeError, ValueError, KeyError):
15612
+ return None
15613
+ finally:
15614
+ conn.close()
15615
+ if summary is None or summary.boundary is None:
15616
+ return None
15617
+ original = _lib_account_protocol.with_revision(summary.boundary, None)
15618
+ raw = _read_line_at(original.segment, original.end_offset)
15619
+ rec = None if raw is None else _decode_anchor_record(raw)
15620
+ if rec is None or rec["payload"]["kind"] != (
15621
+ _lib_account_protocol.CUTOVER_OP_KIND):
15622
+ return None
15623
+ try:
15624
+ recorded = _lib_account_protocol.boundary_from_op(rec)
15625
+ except ValueError:
15626
+ return None
15627
+ return recorded if recorded == original else None
15628
+
15629
+
15630
+ def _ensure_original_cutover(*, now_utc=None):
15631
+ """The #825 establishment of the original cutover (spec 5.5)."""
15048
15632
  doc = _read_account_bound_cache()
15049
15633
  found = _verified_cutover_from_cache(doc)
15050
15634
  if found is not None:
@@ -15114,7 +15698,9 @@ def ensure_account_bound_cutover(*, now_utc=None):
15114
15698
  # the cutover op — no digest, no baseline file — so no crash can
15115
15699
  # leave a boundary without it.
15116
15700
  try:
15117
- state = capture_legacy_spool_state()
15701
+ # The ORIGINAL inventory: the revision boundary that follows
15702
+ # records the expanded one (#797/#823/#840 spec 4.6 B4).
15703
+ state = capture_legacy_spool_state(include_v1=False)
15118
15704
  failure = None
15119
15705
  except LegacySpoolInventoryIncomplete as exc:
15120
15706
  state = None
@@ -15208,9 +15794,427 @@ def _frozen_extents(journal_dir, seg_name, seg_created, start) -> dict:
15208
15794
  return extents
15209
15795
 
15210
15796
 
15211
- def _latest_anchor(cutover):
15797
+ # --------------------------------------------------------------------------
15798
+ # #797/#823/#840 revision 2 of the account-bound fence (spec 4.6 B1-B4)
15799
+ # --------------------------------------------------------------------------
15800
+ #
15801
+ # The revision boundary is a second durable op, `account_bound_revision` with
15802
+ # the stable id `account-bound-revision-2`, recording with the cutover's codec
15803
+ # the physical position at which it was appended and the EXPANDED legacy spool
15804
+ # baseline digest (the pre-#825 paths plus the retired v1 root's coordination
15805
+ # state). It always follows the original cutover in canonical order: like
15806
+ # every anchor it is appended under `journal.lock` to the canonically-last
15807
+ # segment, so a forward scan from the cutover op is complete. It is located
15808
+ # through its own coordinate cache (`revision-coordinate.json`), which only
15809
+ # accelerates the lookup and never confers authority, and it is established
15810
+ # with #825's 5.5 durability sequence before any authoritative decision —
15811
+ # through `ensure_account_bound_cutover`, never from render.
15812
+ #
15813
+ # Establishment preserves the original boundary, reconciles the existing
15814
+ # legacy baseline (the ORIGINAL inventory against the latest pre-revision
15815
+ # anchor, whose baseline file an older binary may have installed in the v1
15816
+ # root) and journals any difference as an ordinary `legacy_spool_activity`
15817
+ # trigger in the same locked write, carries every unresolved spool marker
15818
+ # forward (markers are journal facts and nothing here clears one), and only
15819
+ # then installs the expanded baseline under the revision op.
15820
+
15821
+ _ACCOUNT_BOUND_REVISION_SRC = "account-bound-revision"
15822
+ _REVISION_CACHE_SCHEMA = 1
15823
+
15824
+
15825
+ def _revision_record(rec):
15826
+ """``rec`` when it is a stamped revision-2 ``account_bound_revision`` op
15827
+ under its stable id whose payload decodes, else None. An op without
15828
+ ``wpr == 2`` is never a revision boundary (spec 4.6 B1)."""
15829
+ if (not isinstance(rec, dict) or rec.get("t") != "op"
15830
+ or not _lib_account_protocol.is_stamped(rec)
15831
+ or not _lib_account_protocol.has_current_revision(rec)
15832
+ or rec.get("id") != _lib_account_protocol.REVISION_OP_ID):
15833
+ return None
15834
+ payload = rec.get("payload")
15835
+ if (not isinstance(payload, dict)
15836
+ or payload.get("kind") != _lib_account_protocol.REVISION_OP_KIND):
15837
+ return None
15838
+ try:
15839
+ _lib_account_protocol.revision_boundary_from_op(rec)
15840
+ except ValueError:
15841
+ return None
15842
+ return rec
15843
+
15844
+
15845
+ def _revision_found_for(found, cutover):
15846
+ """``found`` (``(segment, offset, record)``) when its revision op records
15847
+ exactly that coordinate as its own and names ``cutover`` (the cutover
15848
+ op's ``(segment, offset)``), else None."""
15849
+ if found is None:
15850
+ return None
15851
+ rec = _revision_record(found[2])
15852
+ if rec is None:
15853
+ return None
15854
+ revision = _lib_account_protocol.revision_boundary_from_op(rec)
15855
+ if ((revision.segment, revision.end_offset) != (found[0], found[1])
15856
+ or (revision.cutover_segment, revision.cutover_offset)
15857
+ != tuple(cutover)):
15858
+ return None
15859
+ return found
15860
+
15861
+
15862
+ def _scan_for_revision_traversed(cutover, start=None):
15863
+ """The first revision op after ``cutover`` in canonical order, plus the
15864
+ traversal endpoints (``{segment: complete-line end read}``).
15865
+
15866
+ ``start`` maps segments to an offset a previous traversal proved holds no
15867
+ revision op before it; every other segment at or after the cutover's is
15868
+ read from its start (the cutover's own from the cutover op). A segment
15869
+ sorting before the cutover's cannot hold the op and is not read. A read
15870
+ failure raises `AccountBoundCutoverUnavailable` (absence unproven)."""
15871
+ seg0, off0 = cutover
15872
+ floor = _lib_journal.segment_sort_key(seg0)
15873
+ start = start or {}
15874
+ traversed = {}
15875
+ sizes = _segment_sizes()
15876
+ for seg in sorted(sizes, key=_lib_journal.segment_sort_key):
15877
+ if _lib_journal.segment_sort_key(seg) < floor:
15878
+ continue
15879
+ lo = start.get(seg, off0 if seg == seg0 else 0)
15880
+ if seg == seg0:
15881
+ lo = max(lo, off0)
15882
+ hi = sizes[seg]
15883
+ traversed[seg] = lo
15884
+ if hi <= lo:
15885
+ continue
15886
+ try:
15887
+ for _name, off, raw in _iter_segment_lines(
15888
+ _cctally_core.JOURNAL_DIR / seg, lo, hi):
15889
+ if _REVISION_ID_BYTES in raw:
15890
+ found = _revision_found_for(
15891
+ (seg, off, _lib_journal.decode_line(raw)), cutover)
15892
+ if found is not None:
15893
+ traversed[seg] = off
15894
+ return found, traversed
15895
+ traversed[seg] = off + len(raw) + 1
15896
+ except FileNotFoundError:
15897
+ continue
15898
+ except OSError as exc:
15899
+ raise _cutover_lookup_incomplete(seg, exc) from exc
15900
+ return None, traversed
15901
+
15902
+
15903
+ def _read_revision_cache(cutover):
15904
+ """The revision cache document when it names ``cutover``, else None."""
15905
+ try:
15906
+ doc = json.loads(
15907
+ _cctally_core.ACCOUNT_BOUND_REVISION_CACHE_PATH.read_bytes())
15908
+ except (OSError, ValueError):
15909
+ return None
15910
+ if (not isinstance(doc, dict)
15911
+ or doc.get("schemaVersion") != _REVISION_CACHE_SCHEMA):
15912
+ return None
15913
+ named = _cached_coordinate(doc, "cutover")
15914
+ if named is None or (named["segment"], named["offset"]) != tuple(cutover):
15915
+ return None
15916
+ return doc
15917
+
15918
+
15919
+ def _verified_revision_from_cache(doc, cutover):
15920
+ """``(segment, offset, record)`` when the cache names a coordinate that
15921
+ holds this cutover's revision op, else None."""
15922
+ coord = _cached_coordinate(doc, "revision")
15923
+ if coord is None:
15924
+ return None
15925
+ raw = _read_line_at(coord["segment"], coord["offset"])
15926
+ if raw is None:
15927
+ return None
15928
+ return _revision_found_for(
15929
+ (coord["segment"], coord["offset"], _lib_journal.decode_line(raw)),
15930
+ cutover)
15931
+
15932
+
15933
+ def _cached_revision_absence(doc):
15934
+ """The cache's ``{segment: offset}`` absence prefix, or None."""
15935
+ if doc is None or doc.get("revision") is not None:
15936
+ return None
15937
+ absent = doc.get("absentThrough")
15938
+ if not isinstance(absent, dict):
15939
+ return None
15940
+ sizes = None
15941
+ for seg, off in absent.items():
15942
+ if not isinstance(seg, str) or type(off) is not int or off < 0:
15943
+ return None
15944
+ if sizes is None:
15945
+ try:
15946
+ sizes = _segment_sizes()
15947
+ except OSError:
15948
+ return None
15949
+ if seg not in sizes or off > sizes[seg]:
15950
+ return None
15951
+ return absent
15952
+
15953
+
15954
+ def _publish_revision_cache(cutover, revision=None, absent_through=None):
15955
+ """Best-effort accelerator; a failure only costs the next lookup a scan.
15956
+ Sound for an absence prefix for the same reason the cutover's is: the
15957
+ revision op is only ever appended after such a traversal, past it."""
15958
+ doc = {
15959
+ "schemaVersion": _REVISION_CACHE_SCHEMA,
15960
+ "cutover": {"segment": cutover[0], "offset": cutover[1]},
15961
+ "revision": (None if revision is None else
15962
+ {"segment": revision[0], "offset": revision[1]}),
15963
+ }
15964
+ if revision is None:
15965
+ doc["absentThrough"] = dict(sorted((absent_through or {}).items()))
15966
+ try:
15967
+ _atomic_write_json(_cctally_core.ACCOUNT_BOUND_REVISION_CACHE_PATH, doc)
15968
+ except OSError:
15969
+ pass
15970
+
15971
+
15972
+ def _locate_revision_traversed(cutover, *, publish=False):
15973
+ """``(found, traversed)`` for the revision op of ``cutover`` (its
15974
+ ``(segment, offset)``): through the verified cache, else one scan from
15975
+ the cutover op (only the suffix past a cached absence prefix).
15976
+ ``traversed`` is empty when the cache answered."""
15977
+ doc = _read_revision_cache(cutover)
15978
+ found = _verified_revision_from_cache(doc, cutover)
15979
+ if found is not None:
15980
+ return found, {}
15981
+ found, traversed = _scan_for_revision_traversed(
15982
+ cutover, _cached_revision_absence(doc))
15983
+ if publish:
15984
+ if found is not None:
15985
+ try:
15986
+ _make_cutover_durable(found[0])
15987
+ except OSError:
15988
+ return found, traversed
15989
+ _publish_revision_cache(cutover, found[:2])
15990
+ else:
15991
+ _publish_revision_cache(cutover, absent_through=traversed)
15992
+ return found, traversed
15993
+
15994
+
15995
+ def _locate_revision(cutover, *, publish=False):
15996
+ return _locate_revision_traversed(cutover, publish=publish)[0]
15997
+
15998
+
15999
+ def _revision_boundary_of(found):
16000
+ return (None if found is None
16001
+ else _lib_account_protocol.revision_boundary_from_op(found[2]))
16002
+
16003
+
16004
+ def _attach_revision(boundary, *, publish=False):
16005
+ """``boundary`` (a `CutoverBoundary` or None) carrying the revision-2
16006
+ boundary the journal records after it (spec 4.6 B3)."""
16007
+ if boundary is None:
16008
+ return None
16009
+ cutover = (boundary.segment, boundary.end_offset)
16010
+ return _lib_account_protocol.with_revision(
16011
+ boundary, _revision_boundary_of(_locate_revision(
16012
+ cutover, publish=publish)))
16013
+
16014
+
16015
+ def find_account_bound_revision(*, publish=False):
16016
+ """The recorded `RevisionBoundary`, or None (no cutover, or no revision
16017
+ yet). A function of the journal alone; see `find_account_bound_cutover`."""
16018
+ boundary = find_account_bound_cutover(publish=publish)
16019
+ return None if boundary is None else boundary.revision
16020
+
16021
+
16022
+ def _read_baseline_document(path, digest):
16023
+ """The legacy spool state at ``path`` when its digest is ``digest``."""
16024
+ try:
16025
+ state = _lib_legacy_spool.from_document(
16026
+ json.loads(pathlib.Path(path).read_bytes()))
16027
+ except (OSError, ValueError):
16028
+ return None
16029
+ return state if state.digest() == digest else None
16030
+
16031
+
16032
+ def _pre_revision_baseline(digest):
16033
+ """The installed baseline whose digest is the latest pre-revision anchor's
16034
+ (spec 4.6 B2 "reconciles the existing legacy baseline"): the current
16035
+ root's file (a cutover this binary established) or the retired v1 root's
16036
+ (one the previous release established and advanced). Read only."""
16037
+ if not isinstance(digest, str):
16038
+ return None
16039
+ for path in (_cctally_core.LEGACY_SPOOL_BASELINE_PATH,
16040
+ _cctally_core.STATUSLINE_ACCOUNT_BOUND_V1_DIR
16041
+ / "legacy-spool-baseline.json"):
16042
+ state = _read_baseline_document(path, digest)
16043
+ if state is not None:
16044
+ return state
16045
+ return None
16046
+
16047
+
16048
+ def _reconciliation_record(cutover_found):
16049
+ """The `legacy_spool_activity` op (current-stamped, prepared) that
16050
+ reconciles the ORIGINAL legacy inventory with the latest pre-revision
16051
+ anchor, or None when nothing differs, no baseline was ever anchored (a
16052
+ cutover that recorded uncertainty and nothing after it), or the original
16053
+ inventory cannot complete (the revision op then records the failure).
16054
+ Called under `journal.lock`, before the revision op is appended."""
16055
+ (_seg, _off, digest, _sseg, _soff, _uncertain) = _latest_anchor(
16056
+ cutover_found)
16057
+ if digest is None:
16058
+ return None
16059
+ try:
16060
+ original = capture_legacy_spool_state(include_v1=False)
16061
+ except LegacySpoolInventoryIncomplete:
16062
+ return None
16063
+ baseline = _pre_revision_baseline(digest)
16064
+ if baseline is None:
16065
+ reason = "baseline-unavailable"
16066
+ changed = tuple(sorted(entry[0] for entry in original.entries))
16067
+ elif baseline.digest() == original.digest():
16068
+ return None
16069
+ else:
16070
+ reason = "changed"
16071
+ changed = tuple(sorted(set(_lib_legacy_spool.diff(baseline, original))))
16072
+ return _lib_journal.prepare_record(_lib_journal.make_op(
16073
+ at=_now_iso(), src=_LEGACY_SPOOL_ACTIVITY_SRC,
16074
+ payload={
16075
+ "kind": _lib_account_protocol.LEGACY_SPOOL_ACTIVITY_KIND,
16076
+ "reason": reason,
16077
+ "changed": list(changed),
16078
+ "evidence": _lib_legacy_spool.change_evidence(changed, original),
16079
+ "digest": original.digest(),
16080
+ }))
16081
+
16082
+
16083
+ def _establish_account_bound_revision(cutover_boundary, *, now_utc=None):
16084
+ """Return ``cutover_boundary`` carrying its revision-2 boundary,
16085
+ establishing the revision op when absent (spec 4.6 B2; #825 spec 5.5's
16086
+ durability sequence, unchanged). Never establishes a second revision; a
16087
+ producer that loses the locked race installs nothing."""
16088
+ cutover = (cutover_boundary.segment, cutover_boundary.end_offset)
16089
+ # (1) The verified cache; (2) else a scan of a pinned prefix outside the
16090
+ # lock, whose traversal the locked recheck resumes from.
16091
+ found, traversed = _locate_revision_traversed(cutover, publish=False)
16092
+ if found is not None:
16093
+ if traversed:
16094
+ # Found by a scan, not the verified cache: visible is not
16095
+ # durable (fix round K13), and the cache is worth publishing.
16096
+ # A cache hit is the steady state and writes nothing (#901).
16097
+ _make_cutover_durable(found[0])
16098
+ _publish_revision_cache(cutover, found[:2])
16099
+ return _lib_account_protocol.with_revision(
16100
+ cutover_boundary, _revision_boundary_of(found))
16101
+ _account_bound_seam("revision-after-scan")
16102
+
16103
+ explicit_now = now_utc is not None
16104
+ if now_utc is None:
16105
+ now_utc = _utc_now()
16106
+ journal_dir, dir_created = _journal_dir_for_append()
16107
+ lock_fd = _acquire_leaf_lock()
16108
+ try:
16109
+ # (3) Recheck the suffix the unlocked scan did not traverse.
16110
+ winner, _rechecked = _scan_for_revision_traversed(cutover, traversed)
16111
+ if winner is not None:
16112
+ _make_cutover_durable(winner[0])
16113
+ _publish_revision_cache(cutover, winner[:2])
16114
+ return _lib_account_protocol.with_revision(
16115
+ cutover_boundary, _revision_boundary_of(winner))
16116
+ cutover_found = (cutover[0], cutover[1], _cutover_record_at(cutover))
16117
+ reconciliation = _reconciliation_record(cutover_found)
16118
+ try:
16119
+ state = capture_legacy_spool_state(include_v1=True)
16120
+ failure = None
16121
+ except LegacySpoolInventoryIncomplete as exc:
16122
+ state = None
16123
+ failure = exc.evidence()
16124
+ continued = None
16125
+ if failure is not None and reconciliation is None:
16126
+ # One trigger per failure episode (#825 spec revision 5): an
16127
+ # episode still open before the boundary already has its
16128
+ # uncertainty record, which this failure continues.
16129
+ prior = _latest_anchor(cutover_found)[5]
16130
+ if prior is not None and not _inventory_episode_closed(prior):
16131
+ continued = prior
16132
+ _account_bound_seam("revision-after-baseline-capture")
16133
+
16134
+ seg_name, _seg_path, seg_created, fd = _open_append_target(
16135
+ now_utc, explicit_now)
16136
+ try:
16137
+ if seg_created:
16138
+ try:
16139
+ os.fchmod(fd, 0o600)
16140
+ except OSError:
16141
+ pass
16142
+ _repair_torn_tail(fd)
16143
+ first = os.fstat(fd).st_size
16144
+ head = (b"" if reconciliation is None
16145
+ else _lib_journal.encode_line(reconciliation))
16146
+ start = first + len(head)
16147
+ extents = _frozen_extents(journal_dir, seg_name,
16148
+ seg_created and not head, start)
16149
+ revision = _lib_account_protocol.RevisionBoundary(
16150
+ segment=seg_name,
16151
+ end_offset=start,
16152
+ frozen_extents=extents,
16153
+ legacy_spool_baseline_digest=(
16154
+ None if state is None else state.digest()),
16155
+ cutover_segment=cutover[0],
16156
+ cutover_offset=cutover[1],
16157
+ legacy_spool_inventory_failure=failure,
16158
+ continued_inventory_episode=continued,
16159
+ )
16160
+ record = _lib_journal.make_op(
16161
+ at=_now_iso(), src=_ACCOUNT_BOUND_REVISION_SRC,
16162
+ payload={"kind": _lib_account_protocol.REVISION_OP_KIND,
16163
+ **_lib_account_protocol.revision_boundary_payload(
16164
+ revision)})
16165
+ # A stable semantic id, recognisable without its timestamp.
16166
+ record["id"] = _lib_account_protocol.REVISION_OP_ID
16167
+ line = _lib_journal.encode_line(_lib_journal.prepare_record(record))
16168
+ # The reconciliation trigger and the boundary are one write: no
16169
+ # crash leaves the trigger without the boundary it precedes.
16170
+ _write_all(fd, head + line)
16171
+ _account_bound_seam("revision-after-append")
16172
+ os.fsync(fd)
16173
+ finally:
16174
+ os.close(fd)
16175
+ if seg_created:
16176
+ _fsync_dir(journal_dir)
16177
+ if dir_created:
16178
+ _fsync_dir(journal_dir.parent)
16179
+ _account_bound_seam("revision-after-fsync")
16180
+ if state is not None:
16181
+ _atomic_write_json(_cctally_core.LEGACY_SPOOL_BASELINE_PATH,
16182
+ _lib_legacy_spool.to_document(state))
16183
+ _account_bound_seam("revision-after-baseline-install")
16184
+ finally:
16185
+ _release_leaf_lock(lock_fd)
16186
+ _publish_revision_cache(cutover, (seg_name, start))
16187
+ _publish_account_bound_cache(
16188
+ cutover, (seg_name, start, revision.legacy_spool_baseline_digest),
16189
+ (seg_name, start + len(line)),
16190
+ uncertain=(None if failure is None
16191
+ else continued if continued is not None
16192
+ else (seg_name, start)))
16193
+ return _lib_account_protocol.with_revision(cutover_boundary, revision)
16194
+
16195
+
16196
+ def _cutover_record_at(cutover):
16197
+ """The cutover op record at its coordinate (it was located before)."""
16198
+ raw = _read_line_at(cutover[0], cutover[1])
16199
+ rec = None if raw is None else _decode_anchor_record(raw)
16200
+ if rec is None or rec["payload"]["kind"] != (
16201
+ _lib_account_protocol.CUTOVER_OP_KIND):
16202
+ raise AccountBoundCutoverUnavailable(
16203
+ errno.EIO, "account-bound cutover op could not be re-read",
16204
+ str(cutover[0]))
16205
+ return rec
16206
+
16207
+
16208
+ def _latest_anchor(cutover, revision=None):
15212
16209
  """``(segment, offset, digest, scanned_segment, scanned_offset,
15213
16210
  uncertain)`` of the latest stamped anchoring op at or after ``cutover``.
16211
+
16212
+ #797/#823/#840 (spec 4.6 B2, B3): ``revision`` is the revision-2
16213
+ boundary when one exists. The revision op is itself an anchor (its
16214
+ expanded baseline digest, or inventory uncertainty when its inventory
16215
+ failed), and after it only an op carrying ``wpr == 2`` anchors or opens
16216
+ an uncertainty episode: an older process's post-boundary op is excluded
16217
+ input.
15214
16218
  Current anchors are appended only by this binary, always to the
15215
16219
  canonically-last segment, so a forward scan from a verified anchor is
15216
16220
  complete.
@@ -15267,13 +16271,29 @@ def _latest_anchor(cutover):
15267
16271
  if not any(token in raw for token in _ANCHOR_KIND_BYTES):
15268
16272
  continue
15269
16273
  rec = _lib_journal.decode_line(raw)
16274
+ if _post_revision_without_revision(rec, seg, off, revision):
16275
+ continue
15270
16276
  # The cutover op is the initial anchor; only a later stamped
15271
- # activity op advances it.
16277
+ # activity op, or the revision op, advances it.
15272
16278
  anchored = _anchor_record(rec)
15273
- if anchored is not None and anchored["payload"]["kind"] == (
15274
- _lib_account_protocol.LEGACY_SPOOL_ACTIVITY_KIND):
16279
+ kind = None if anchored is None else (
16280
+ anchored["payload"]["kind"])
16281
+ if kind == _lib_account_protocol.LEGACY_SPOOL_ACTIVITY_KIND:
15275
16282
  anchor = (seg, off, _anchor_digest(anchored))
15276
16283
  uncertain = None
16284
+ elif kind == _lib_account_protocol.REVISION_OP_KIND:
16285
+ anchor = (seg, off, _anchor_digest(anchored))
16286
+ revision_payload = anchored["payload"]
16287
+ episode = revision_payload.get("inventoryEpisode")
16288
+ if revision_payload.get(
16289
+ "legacySpoolInventoryFailure") is None:
16290
+ uncertain = None
16291
+ elif isinstance(episode, dict):
16292
+ # It continues an open episode: that episode's
16293
+ # record stays the uncertainty (and its closure).
16294
+ uncertain = (episode["segment"], episode["offset"])
16295
+ else:
16296
+ uncertain = (seg, off)
15277
16297
  elif (_inventory_uncertainty_record(rec) is not None
15278
16298
  and rec["payload"]["kind"]
15279
16299
  == _lib_account_protocol.LEGACY_SPOOL_ACTIVITY_KIND):
@@ -15358,6 +16378,12 @@ class LegacySpoolAnchorState:
15358
16378
  episode_open: bool
15359
16379
 
15360
16380
 
16381
+ def _revision_of(cutover):
16382
+ """The `RevisionBoundary` recorded after the located ``cutover``, or
16383
+ None."""
16384
+ return _revision_boundary_of(_locate_revision(cutover[:2]))
16385
+
16386
+
15361
16387
  def legacy_spool_anchor_state() -> "LegacySpoolAnchorState | None":
15362
16388
  """The anchor state, or None when no cutover exists. Raises
15363
16389
  `AccountBoundCutoverUnavailable` when a segment it needs is unreadable.
@@ -15369,7 +16395,7 @@ def legacy_spool_anchor_state() -> "LegacySpoolAnchorState | None":
15369
16395
  if cutover is None:
15370
16396
  return None
15371
16397
  (seg, off, digest, scanned_seg, scanned_off,
15372
- uncertain) = _latest_anchor(cutover)
16398
+ uncertain) = _latest_anchor(cutover, _revision_of(cutover))
15373
16399
  if not _anchor_cache_is_current(
15374
16400
  cutover[:2], (seg, off, digest), (scanned_seg, scanned_off),
15375
16401
  uncertain):
@@ -15474,7 +16500,7 @@ def record_legacy_spool_inventory_incomplete(failure, *, now_utc=None):
15474
16500
  lock_fd = _acquire_leaf_lock()
15475
16501
  try:
15476
16502
  (a_seg, a_off, a_digest, _s_seg, _s_off,
15477
- uncertain) = _latest_anchor(cutover)
16503
+ uncertain) = _latest_anchor(cutover, _revision_of(cutover))
15478
16504
  if (uncertain is not None
15479
16505
  and not _inventory_episode_closed(uncertain)):
15480
16506
  # Fix round K12 (K11R-7, spec section 7 *Durability*): a
@@ -15675,20 +16701,46 @@ class _PassBoundary:
15675
16701
  and BEFORE any interpretation. `resolve` also scans the unvisited suffix
15676
16702
  past the pass's high-water, where an op appended after the pin lives."""
15677
16703
 
15678
- def __init__(self):
15679
- self._op = None
16704
+ def __init__(self, cutover=None):
16705
+ #: The cutover boundary: met in the pass, or known beforehand
16706
+ #: (``cutover``, e.g. an index's admission summary).
16707
+ self._boundary = cutover
16708
+ #: #797/#823/#840: the first revision-2 op the pass observed.
16709
+ self._revision = None
15680
16710
 
15681
16711
  def observe(self, record) -> None:
15682
- if (self._op is None and isinstance(record, dict)
15683
- and record.get("id") == _lib_account_protocol.CUTOVER_OP_ID):
16712
+ if not isinstance(record, dict):
16713
+ return
16714
+ rid = record.get("id")
16715
+ if (self._boundary is None
16716
+ and rid == _lib_account_protocol.CUTOVER_OP_ID):
15684
16717
  anchor = _anchor_record(record)
15685
16718
  if anchor is not None and anchor["payload"]["kind"] == (
15686
16719
  _lib_account_protocol.CUTOVER_OP_KIND):
15687
- self._op = anchor
16720
+ self._boundary = _lib_account_protocol.boundary_from_op(
16721
+ anchor)
16722
+ elif (self._revision is None and self._boundary is not None
16723
+ and rid == _lib_account_protocol.REVISION_OP_ID):
16724
+ # A revision op belongs to the cutover the pass already met: it
16725
+ # is appended after it, so a pass sees them in that order.
16726
+ self._revision = _revision_record(record)
15688
16727
 
15689
16728
  def resolve(self, high_water):
15690
- if self._op is not None:
15691
- return _lib_account_protocol.boundary_from_op(self._op)
16729
+ if self._boundary is not None:
16730
+ boundary = _dc_replace(self._boundary, revision=None)
16731
+ revision = None
16732
+ if self._revision is not None:
16733
+ revision = _lib_account_protocol.revision_boundary_from_op(
16734
+ self._revision)
16735
+ if ((revision.cutover_segment, revision.cutover_offset)
16736
+ != (boundary.segment, boundary.end_offset)):
16737
+ revision = None
16738
+ if revision is None and high_water is not None:
16739
+ # Appended past the pass's high-water (or never observed):
16740
+ # every record of the pass precedes it, and the lookup keeps
16741
+ # what admission consumers see one function of the journal.
16742
+ return _attach_revision(boundary)
16743
+ return _lib_account_protocol.with_revision(boundary, revision)
15692
16744
  if high_water is None:
15693
16745
  return None
15694
16746
  start_key = _lib_journal.segment_sort_key(high_water[0])
@@ -15700,7 +16752,8 @@ class _PassBoundary:
15700
16752
  ranges[seg] = (high_water[1] if seg == high_water[0] else 0, size)
15701
16753
  found = _scan_for_cutover(ranges)
15702
16754
  return (None if found is None
15703
- else _lib_account_protocol.boundary_from_op(found[2]))
16755
+ else _attach_revision(
16756
+ _lib_account_protocol.boundary_from_op(found[2])))
15704
16757
 
15705
16758
 
15706
16759
  def _apply_pass_admission(slots, coordinates, boundary, placeholder=None):
@@ -15796,6 +16849,10 @@ class AdmissionSummary:
15796
16849
  coordinate_digest: str | None = None
15797
16850
  complete_through: "tuple[str, int] | None" = None
15798
16851
  boundary: "object | None" = None
16852
+ #: #797/#823/#840 (spec 4.6 B3): the revision-2 boundary the summary's
16853
+ #: exclusions were classified against (`boundary.revision`), named here
16854
+ #: because a `CutoverBoundary`'s equality is the cutover op's alone.
16855
+ revision_id: str | None = None
15799
16856
 
15800
16857
  def extend(self, excluded) -> None:
15801
16858
  if not excluded:
@@ -15826,15 +16883,30 @@ def _read_admission_summary(conn):
15826
16883
  if row is None:
15827
16884
  return None
15828
16885
  boundary = None
16886
+ revision_id = None
15829
16887
  if row[8]:
15830
16888
  try:
16889
+ doc = json.loads(row[8])
16890
+ revision_doc = doc.pop("revision", None)
15831
16891
  boundary = _lib_account_protocol.boundary_from_op({
15832
16892
  "t": "op",
15833
16893
  "payload": {"kind": _lib_account_protocol.CUTOVER_OP_KIND,
15834
- **json.loads(row[8])},
16894
+ **doc},
15835
16895
  })
15836
- except (ValueError, TypeError):
16896
+ if revision_doc is not None:
16897
+ # #797/#823/#840 (spec 4.6 B3): the summary binds both fences.
16898
+ boundary = _lib_account_protocol.with_revision(
16899
+ boundary,
16900
+ _lib_account_protocol.revision_boundary_from_op({
16901
+ "t": "op",
16902
+ "payload": {
16903
+ "kind": _lib_account_protocol.REVISION_OP_KIND,
16904
+ **revision_doc},
16905
+ }))
16906
+ revision_id = _lib_account_protocol.REVISION_OP_ID
16907
+ except (ValueError, TypeError, AttributeError):
15837
16908
  boundary = None
16909
+ revision_id = None
15838
16910
 
15839
16911
  def _hw(text):
15840
16912
  coord = _coordinate_from_json(text)
@@ -15854,6 +16926,7 @@ def _read_admission_summary(conn):
15854
16926
  coordinate_digest=row[6],
15855
16927
  complete_through=_hw(row[7]),
15856
16928
  boundary=boundary,
16929
+ revision_id=revision_id,
15857
16930
  )
15858
16931
 
15859
16932
 
@@ -15881,17 +16954,55 @@ def _write_admission_summary(conn, summary: AdmissionSummary) -> None:
15881
16954
  summary.coordinate_digest,
15882
16955
  _hw(summary.complete_through),
15883
16956
  None if summary.boundary is None else json.dumps(
15884
- _lib_account_protocol.boundary_payload(summary.boundary),
16957
+ _summary_boundary_document(summary.boundary),
15885
16958
  sort_keys=True, separators=(",", ":")),
15886
16959
  ),
15887
16960
  )
15888
16961
 
15889
16962
 
16963
+ def _summary_boundary_document(boundary) -> dict:
16964
+ """The summary's ``boundary_json``: the cutover payload, plus the
16965
+ revision-2 boundary's under ``revision`` once one applies."""
16966
+ doc = _lib_account_protocol.boundary_payload(boundary)
16967
+ if boundary.revision is not None:
16968
+ doc["revision"] = _lib_account_protocol.revision_boundary_payload(
16969
+ boundary.revision)
16970
+ return doc
16971
+
16972
+
15890
16973
  def _evidence_now():
15891
16974
  """The wall clock the legacy-writer evidence records (a test seam)."""
15892
16975
  return _utc_now()
15893
16976
 
15894
16977
 
16978
+ def read_legacy_writer_evidence_document():
16979
+ """``(document, seeded)``: the durable legacy-writer evidence sidecar,
16980
+ or — when the v2 sidecar is absent — the previous release's
16981
+ `statusline-account-bound-v1/legacy-writer-evidence.json` as its seed
16982
+ (``seeded`` True), else ``(None, False)``.
16983
+
16984
+ Review RV-5 (#797/#823/#840): the sidecar moved to the v2 directory and
16985
+ nothing carried it over, so the first scan after the upgrade recorded an
16986
+ already-old excluded population as first observed now and the doctor
16987
+ leg warned again for seven days — although a rebuild over unchanged
16988
+ evidence never restarts the age. The v1 file is diagnostic evidence,
16989
+ not coordination state (spec 4.6 B4 forbids importing that), and it is
16990
+ only ever READ: nothing here writes, renames or deletes under v1."""
16991
+ try:
16992
+ return json.loads(
16993
+ _cctally_core.LEGACY_WRITER_EVIDENCE_PATH.read_bytes()), False
16994
+ except FileNotFoundError:
16995
+ pass
16996
+ except (OSError, ValueError):
16997
+ return None, False
16998
+ try:
16999
+ doc = json.loads((_cctally_core.STATUSLINE_ACCOUNT_BOUND_V1_DIR
17000
+ / "legacy-writer-evidence.json").read_bytes())
17001
+ except (OSError, ValueError):
17002
+ return None, False
17003
+ return doc, True
17004
+
17005
+
15895
17006
  def _record_legacy_writer_evidence(summary) -> None:
15896
17007
  """Refresh the durable legacy-writer evidence sidecar from a COMPLETE scan
15897
17008
  result (spec section 8): a new excluded population replaces the evidence
@@ -15909,16 +17020,17 @@ def _record_legacy_writer_evidence(summary) -> None:
15909
17020
  # new and restart the seven-day age (case F9); the rebuild records it.
15910
17021
  return
15911
17022
  path = _cctally_core.LEGACY_WRITER_EVIDENCE_PATH
15912
- try:
15913
- existing = _lib_legacy_writer_evidence.from_document(
15914
- json.loads(path.read_bytes()))
15915
- except (OSError, ValueError):
17023
+ doc, seeded = read_legacy_writer_evidence_document()
17024
+ existing = (None if doc is None
17025
+ else _lib_legacy_writer_evidence.from_document(doc))
17026
+ if (seeded and existing is not None
17027
+ and existing.cutover_id != summary.cutover_id):
15916
17028
  existing = None
15917
17029
  updated = _lib_legacy_writer_evidence.observe(
15918
17030
  existing, cutover_id=summary.cutover_id,
15919
17031
  excluded_count=summary.excluded_total,
15920
17032
  coordinate_digest=summary.coordinate_digest, now=_evidence_now())
15921
- if updated is None or updated is existing:
17033
+ if updated is None or (updated is existing and not seeded):
15922
17034
  return
15923
17035
  try:
15924
17036
  _atomic_write_json(path, _lib_legacy_writer_evidence.to_document(updated))
@@ -15937,7 +17049,19 @@ def _ingest_admission_boundary(conn, decoded, cursor):
15937
17049
  index that does not describe its own cursor falls back to the lookup."""
15938
17050
  summary = _read_admission_summary(conn)
15939
17051
  if summary is not None and summary.boundary is not None:
15940
- return summary.boundary
17052
+ if summary.boundary.revision is not None:
17053
+ return summary.boundary
17054
+ # #797/#823/#840 (spec 4.6 B3): the cached summary names the
17055
+ # original boundary only. The revision op is appended like the
17056
+ # cutover op — at the end of the canonically-last segment, after
17057
+ # everything the index consumed — so it is inside this batch or
17058
+ # beyond it, and a batch record before it is historical for it.
17059
+ # Return the cached original boundary with whatever this batch
17060
+ # shows, never the cached summary alone.
17061
+ pass_boundary = _PassBoundary(cutover=summary.boundary)
17062
+ for record, _segment, _offset in decoded:
17063
+ pass_boundary.observe(record)
17064
+ return pass_boundary.resolve(None)
15941
17065
  if cursor is None or (
15942
17066
  summary is not None and summary.scanned_high_water == cursor):
15943
17067
  pass_boundary = _PassBoundary()
@@ -15963,6 +17087,9 @@ def _advance_admission_summary(conn, boundary, excluded, cursor, target) -> None
15963
17087
  summary.cutover_id = (
15964
17088
  None if boundary is None else _lib_account_protocol.CUTOVER_OP_ID)
15965
17089
  summary.boundary = boundary
17090
+ summary.revision_id = (
17091
+ None if boundary is None or boundary.revision is None
17092
+ else _lib_account_protocol.REVISION_OP_ID)
15966
17093
  summary.extend(excluded)
15967
17094
  summary.scanned_high_water = target
15968
17095
  if summary.complete_through is not None: