cctally 1.92.3 → 1.93.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -25,6 +25,8 @@ segment files.
25
25
  """
26
26
  from __future__ import annotations
27
27
 
28
+ import collections
29
+ import contextlib
28
30
  import datetime as dt
29
31
  import fcntl
30
32
  import hashlib
@@ -39,9 +41,12 @@ from dataclasses import dataclass, field, replace as _dc_replace
39
41
 
40
42
  import _cctally_core
41
43
  import _lib_accounts
44
+ import _lib_cache_coverage
42
45
  import _lib_journal
43
46
  import _lib_journal_router
44
47
  import _lib_record
48
+ import _lib_segment_summary
49
+ import _lib_selector_state
45
50
 
46
51
 
47
52
  # Torn-tail scan window (spec §4.3). A single line must fit inside it — a
@@ -65,12 +70,32 @@ class JournalError(Exception):
65
70
  """A structural journal-append failure (line too long, unrepairable tail)."""
66
71
 
67
72
 
73
+ #: A newly completed correction batch whose effect the live index lacks. The
74
+ #: commit marker IS the narrowest complete prefix, and convergence is decided by
75
+ #: the exact `(rev, status, content_hash, batch_id)` the signal carries.
76
+ CORRECTION_KIND_NEWLY_COMPLETED = "newly_completed"
77
+
78
+ #: A batch whose durable status was `completed` and which a later record
79
+ #: tainted. Its recovery coordinate is the END OFFSET OF THAT RECORD, never the
80
+ #: batch's earliest commit: a rebuild bounded at the commit excludes the
81
+ #: tainting record, faithfully reproduces the completed correction, and meets
82
+ #: the same taint on the next tick — signalling forever instead of converging
83
+ #: (#496 S5b §3.7).
84
+ CORRECTION_KIND_COMPLETED_TO_TAINTED = "completed_to_tainted"
85
+
86
+
68
87
  class CorrectionRebuildRequired(JournalError):
69
88
  """A completed correction cannot be applied incrementally to a live index.
70
89
 
71
90
  The recovery boundary needs more than a message: it must rebuild through
72
- the exact completed-batch commit that triggered the mismatch, then
73
- revalidate the exact effective metadata under exclusive ownership.
91
+ the exact prefix that triggered the mismatch, then revalidate under
92
+ exclusive ownership.
93
+
94
+ `kind` selects WHICH prefix and WHICH revalidation. The two kinds do not
95
+ share a convergence predicate: after a taint withdraws a completed batch the
96
+ post-rebuild winner may be an older candidate that durable selector state
97
+ deliberately does not store, so an exact expected metadata tuple is not
98
+ computable for it.
74
99
  """
75
100
 
76
101
  def __init__(
@@ -82,6 +107,7 @@ class CorrectionRebuildRequired(JournalError):
82
107
  high_water=None,
83
108
  expected_metadata=None,
84
109
  recovery_eligible=False,
110
+ kind=CORRECTION_KIND_NEWLY_COMPLETED,
85
111
  ):
86
112
  super().__init__(message)
87
113
  self.batch_id = batch_id
@@ -89,12 +115,26 @@ class CorrectionRebuildRequired(JournalError):
89
115
  self.high_water = high_water
90
116
  self.expected_metadata = expected_metadata
91
117
  self.recovery_eligible = recovery_eligible
118
+ self.kind = kind
92
119
 
93
120
 
94
121
  class CorrectionRecoveryError(JournalError):
95
122
  """Bounded correction recovery could not safely replace the live index."""
96
123
 
97
124
 
125
+ class JournalAppendTargetStale(JournalError):
126
+ """The resolved append target is no longer the canonically-last segment.
127
+
128
+ Raised when a writer resolved its month segment before taking the leaf lock
129
+ and the journal moved on underneath it (#511, #496 S5b §2.4). Retryable:
130
+ the caller re-resolves and appends again. It is a DISTINCT class precisely
131
+ so `_cctally_cache._append_codex_quota_obs` can re-raise it while still
132
+ swallowing genuine errors — swallowing this one would advance a file offset
133
+ past bytes whose observation was never journaled, and the rollout JSONL
134
+ those bytes came from evaporates.
135
+ """
136
+
137
+
98
138
  # --------------------------------------------------------------------------
99
139
  # leaf lock
100
140
  # --------------------------------------------------------------------------
@@ -276,6 +316,50 @@ def _load_quota_dedup_keys() -> None:
276
316
  _QUOTA_DEDUP_LOADED = True
277
317
 
278
318
 
319
+ def _utc_now() -> dt.datetime:
320
+ """The append path's clock, named so both reads are the same call.
321
+
322
+ An appender resolves its month segment once before the leaf lock and once
323
+ again after taking it (#511). Reading the clock through one helper keeps the
324
+ two reads identical in everything but time.
325
+ """
326
+ return dt.datetime.now(dt.timezone.utc)
327
+
328
+
329
+ def _validate_append_target(journal_dir, seg_name: str) -> None:
330
+ """Refuse an append whose target is not the canonically-last segment.
331
+
332
+ MUST be called with the leaf lock held: the whole point is that the
333
+ canonical order cannot move between this check and the write. Both
334
+ appenders route through it rather than duplicating the comparison, because
335
+ fixing only one leaves the claimed immutability false for exactly the
336
+ correction and audit group appends the durable selector depends on.
337
+
338
+ An EXISTING target must equal the canonically-last segment. An ABSENT
339
+ target must sort last when provisionally added to the segment list, which
340
+ is the ordinary month rollover: the new month's file does not exist yet.
341
+
342
+ It refuses rather than redirects. Refusal preserves physical order and lets
343
+ a writer retry against a freshly resolved target, where silently
344
+ redirecting a planned correction group would move it out from under a
345
+ caller that had already reasoned about its placement (#496 S5b §2.4).
346
+ """
347
+ segments = list_segments()
348
+ if not segments:
349
+ return
350
+ if seg_name == segments[-1]:
351
+ return
352
+ if seg_name not in segments:
353
+ provisional = sorted(
354
+ [*segments, seg_name], key=_lib_journal.segment_sort_key)
355
+ if provisional[-1] == seg_name:
356
+ return
357
+ raise JournalAppendTargetStale(
358
+ f"append target {seg_name} is not the canonically-last segment "
359
+ f"({segments[-1]}) in {journal_dir}; re-resolve and retry"
360
+ )
361
+
362
+
279
363
  def _append_quota_dedup_key(natural_key: str) -> None:
280
364
  """Journal-first second leg: append+fsync one key to the compact index."""
281
365
  index_path = _cctally_core.JOURNAL_DIR / _QUOTA_DEDUP_INDEX_NAME
@@ -306,9 +390,15 @@ def append_record(
306
390
  and returns ``None`` on a skip. Otherwise returns ``(segment_basename,
307
391
  end_offset)`` where ``end_offset`` is the file size just past the appended
308
392
  line — the byte position the ingest cursor advances to when it consumes
309
- this line."""
393
+ this line.
394
+
395
+ An explicitly supplied ``now_utc`` is honoured verbatim and NOT validated
396
+ against canonical order: no production caller supplies one, and roughly a
397
+ dozen tests pass a fixed timestamp precisely to pin segment placement. A
398
+ deliberate placement choice is not a stall (#496 S5b §2.4)."""
399
+ explicit_now = now_utc is not None
310
400
  if now_utc is None:
311
- now_utc = dt.datetime.now(dt.timezone.utc)
401
+ now_utc = _utc_now()
312
402
  data = _lib_journal.encode_line(record)
313
403
  if len(data) > _MAX_LINE_BYTES:
314
404
  raise JournalError(
@@ -343,6 +433,15 @@ def append_record(
343
433
  if natural_key in _QUOTA_DEDUP_KEYS:
344
434
  return None
345
435
 
436
+ # ORDER MATTERS (#496 S5b §2.4). The dedupe no-write return above runs
437
+ # FIRST, because a skipped append writes nothing and so has no target to
438
+ # validate. Re-resolution and validation then run BEFORE the file is
439
+ # opened or torn-tail-repaired, so the repair sequence is untouched.
440
+ if not explicit_now:
441
+ seg_name = _lib_journal.segment_name(_utc_now())
442
+ seg_path = journal_dir / seg_name
443
+ _validate_append_target(journal_dir, seg_name)
444
+
346
445
  seg_created = not seg_path.exists()
347
446
  fd = os.open(str(seg_path), os.O_RDWR | os.O_APPEND | os.O_CREAT, 0o600)
348
447
  try:
@@ -385,11 +484,17 @@ def append_records(
385
484
  correction batch remains physically ordered. ``expected_high_water`` is
386
485
  checked while holding the same leaf lock that performs the append, closing
387
486
  the plan/revalidate/append race.
487
+
488
+ That check is NOT a substitute for target validation: it proves nothing
489
+ about canonical order when ``expected_high_water`` is unset, which is the
490
+ default. A defaulted ``now_utc`` is therefore re-resolved and validated
491
+ under the same lock, exactly as in ``append_record`` (#511, #496 S5b §2.4).
388
492
  """
389
493
  if not isinstance(records, list) or not records:
390
494
  raise ValueError("journal record group must be a non-empty list")
495
+ explicit_now = now_utc is not None
391
496
  if now_utc is None:
392
- now_utc = dt.datetime.now(dt.timezone.utc)
497
+ now_utc = _utc_now()
393
498
  encoded = []
394
499
  for record in records:
395
500
  data = _lib_journal.encode_line(record)
@@ -430,6 +535,13 @@ def append_records(
430
535
  f"(expected {expected_high_water!r}, found {actual_high_water!r})"
431
536
  )
432
537
 
538
+ # After the caller's own precondition, and before the file is opened or
539
+ # torn-tail-repaired (#496 S5b §2.4).
540
+ if not explicit_now:
541
+ seg_name = _lib_journal.segment_name(_utc_now())
542
+ seg_path = journal_dir / seg_name
543
+ _validate_append_target(journal_dir, seg_name)
544
+
433
545
  seg_created = not seg_path.exists()
434
546
  fd = os.open(str(seg_path), os.O_RDWR | os.O_APPEND | os.O_CREAT, 0o600)
435
547
  try:
@@ -506,6 +618,434 @@ def _journal_rebuild_snapshot() -> tuple[tuple[str, int] | None, bool]:
506
618
  _release_leaf_lock(lock_fd)
507
619
 
508
620
 
621
+ #: How far back `_complete_line_offset` will look for the last newline when a
622
+ #: segment does not end on one. A torn tail is one interrupted `write`, so it is
623
+ #: bounded by one record; a megabyte is orders of magnitude beyond any record
624
+ #: this journal produces, and reading the whole segment to answer a question
625
+ #: about its last byte is exactly the cost the coverage certificate exists to
626
+ #: remove.
627
+ _COMPLETE_LINE_TAIL_WINDOW = 1 << 20
628
+
629
+
630
+ def _complete_line_offset(path, size: int) -> int:
631
+ """The offset after the last COMPLETE line in ``path``, given its ``size``.
632
+
633
+ The coverage certificate promises decoded records, not bytes, so its covered
634
+ boundary must be a verified newline boundary — covering a raw torn-tail
635
+ extent would let `_repair_torn_tail` truncate that suffix and append a
636
+ complete record ending at the same size, leaving `(segment, size)` identical
637
+ while the covered contribution changed (spec §4.2).
638
+
639
+ Answering costs one `read` of the segment's last byte in the healthy case,
640
+ because every appender writes a trailing newline. Only a torn tail pays for
641
+ the bounded backward scan, and a tail longer than
642
+ `_COMPLETE_LINE_TAIL_WINDOW` answers 0 rather than reading the whole
643
+ segment: 0 is a valid boundary, it is the conservative direction, and it is
644
+ computed identically by every caller, so two callers cannot disagree.
645
+ """
646
+ if size <= 0:
647
+ return 0
648
+ try:
649
+ with open(path, "rb") as handle:
650
+ handle.seek(size - 1)
651
+ if handle.read(1) == b"\n":
652
+ return size
653
+ window = min(size, _COMPLETE_LINE_TAIL_WINDOW)
654
+ handle.seek(size - window)
655
+ tail = handle.read(window)
656
+ except OSError:
657
+ return 0
658
+ index = tail.rfind(b"\n")
659
+ if index < 0:
660
+ return 0
661
+ return size - window + index + 1
662
+
663
+
664
+ def segment_summary_sidecar_path():
665
+ """The journal-side segment-summary sidecar (#496 S5b section 5.3).
666
+
667
+ Beside the segments it describes rather than in `stats.db`, because a
668
+ rebuild frequently runs precisely when `stats.db` is absent or unreadable —
669
+ the one case a summary stored there could never serve.
670
+ """
671
+ return _cctally_core.JOURNAL_DIR / _lib_segment_summary.SIDECAR_NAME
672
+
673
+
674
+ def read_segment_summaries():
675
+ """``{segment_name: SegmentSummary}`` from the sidecar, or ``{}``.
676
+
677
+ A missing, torn, version-mismatched or checksum-mismatched sidecar answers
678
+ an EMPTY map rather than raising, which is exactly a pass that elides
679
+ nothing (spec section 6.3's silent full-replay fallback).
680
+ """
681
+ return _lib_segment_summary.read_sidecar(segment_summary_sidecar_path()) or {}
682
+
683
+
684
+ class SegmentElisionPlan:
685
+ """The one elision planner, shared by both whole-prefix readers.
686
+
687
+ `rebuild_stats_index` and `stats_index_matches_journal_prefix` construct it
688
+ the same way from the same inputs, so an eliding rebuild and a prefix
689
+ validation cannot reach different answers about the same journal (spec
690
+ §5.6).
691
+
692
+ **Constructed BEFORE the pass, from a short-lived read of the coverage
693
+ certificate.** The certificate has to be consulted before the first segment
694
+ is reached, and holding a `cache.db` read transaction open across a whole
695
+ journal traversal would block WAL checkpointing for the length of the
696
+ rebuild — the bloat issue #297 is about. The leg re-resolves coverage under
697
+ its own snapshot afterwards and may reach a different verdict, because an
698
+ ordinary Codex batch landing mid-pass advances the certificate over a
699
+ journal this pass did not pin. `_refill_elided_quota_raw` is what makes that
700
+ race cost a read rather than a wrong answer.
701
+ """
702
+
703
+ __slots__ = ("summaries", "covered", "verdict", "resolution_seen",
704
+ "elided", "elided_lines", "elided_bytes", "scanned",
705
+ "reasons", "quota_gaps")
706
+
707
+ def __init__(self, *, summaries, covered, verdict) -> None:
708
+ self.summaries = summaries
709
+ #: The certificate's `coveredHighWater`, or None when it is unusable.
710
+ self.covered = covered
711
+ self.verdict = verdict
712
+ #: Set once a `journal_protocol_resolution` op is decoded. From then on
713
+ #: nothing further is elided and the prefix digest is recomputed from
714
+ #: disk, because `PrefixHashAccumulator` cannot compose over a gap.
715
+ self.resolution_seen = False
716
+ self.elided = 0
717
+ self.elided_lines = 0
718
+ self.elided_bytes = 0
719
+ self.scanned = 0
720
+ self.reasons: dict = {}
721
+ #: `(segment, index into quota_raw, summarized_size, summarized line
722
+ #: count)` per elided segment, so the leg can re-read exactly what
723
+ #: elision skipped if its own coverage verdict turns out not to be `ok`.
724
+ #: The line count is carried because `_iter_segment_lines` stops
725
+ #: SILENTLY at EOF, so a short read is invisible to the re-read's
726
+ #: `except` clause and can only be caught by counting what came back.
727
+ self.quota_gaps: list = []
728
+
729
+ def decide(self, name, hi, stat_result):
730
+ """The summary to elide ``name`` with, or None to read it.
731
+
732
+ ``hi`` is this pass's pinned raw extent for the segment, which is its
733
+ `st_size` for every segment the planner is allowed to consider — the
734
+ canonically-last one is refused by the caller before this runs.
735
+ """
736
+ summary = self.summaries.get(name)
737
+ if summary is None:
738
+ self.scanned += 1
739
+ self.reasons.setdefault(name, "summaryAbsent")
740
+ return None
741
+ ok, reason = _lib_segment_summary.summary_is_elidable(
742
+ summary,
743
+ pinned_raw_extent=hi,
744
+ is_last=False,
745
+ certificate_covers=self._covers(name, summary),
746
+ resolution_seen=self.resolution_seen,
747
+ stat_identity=(stat_result.st_dev, stat_result.st_ino),
748
+ )
749
+ if not ok:
750
+ self.scanned += 1
751
+ self.reasons.setdefault(name, reason)
752
+ return None
753
+ self.elided += 1
754
+ self.elided_lines += summary.lines
755
+ self.elided_bytes += summary.bytes
756
+ self.reasons.setdefault(name, _lib_segment_summary.REASON_OK)
757
+ return summary
758
+
759
+ def _covers(self, name, summary) -> bool:
760
+ if self.covered is None:
761
+ return False
762
+ return _coordinate_covers(
763
+ self.covered, (name, int(summary.summarized_size)))
764
+
765
+ def counters(self) -> dict:
766
+ """The additive rebuild-record block (§6.3, "recorded, not silent")."""
767
+ return {
768
+ "elidedSegments": self.elided,
769
+ "elidedLines": self.elided_lines,
770
+ "elidedBytes": self.elided_bytes,
771
+ "scannedSegments": self.scanned,
772
+ "coverage": self.verdict,
773
+ "resolutionSeen": self.resolution_seen,
774
+ "refusals": dict(self.reasons),
775
+ }
776
+
777
+
778
+ def plan_segment_elision(segments, high_water):
779
+ """One `SegmentElisionPlan` for the prefix ``segments`` up to ``high_water``.
780
+
781
+ Every failure — no journal, no sidecar, no readable cache, an invalid
782
+ certificate — answers a plan that elides nothing, which is exactly today's
783
+ behaviour. Spec §6.3 makes every one of those a SILENT full replay.
784
+
785
+ The prefix's own pinned vector is deliberately NOT computed here. Only the
786
+ WHOLE journal's vector validates a certificate (see below), and the per-
787
+ segment extents this plan checks arrive from `_iter_range_with_segments`'s
788
+ own `stat`. Computing a second vector restricted to ``segments`` would cost
789
+ one `os.stat` plus one `open` per segment — through `_complete_line_offset`'s
790
+ tail probe — on every plan construction, including inside
791
+ `stats_index_matches_journal_prefix`, and nothing would read it.
792
+ """
793
+ summaries = read_segment_summaries()
794
+ if not summaries or high_water is None:
795
+ return SegmentElisionPlan(
796
+ summaries={}, covered=None,
797
+ verdict=_lib_cache_coverage.REASON_ABSENT)
798
+ snapshot = _read_coverage_snapshot(_cctally_core.CACHE_DB_PATH)
799
+ if snapshot is None:
800
+ return SegmentElisionPlan(
801
+ summaries=summaries, covered=None,
802
+ verdict=_lib_cache_coverage.REASON_ABSENT)
803
+ try:
804
+ # The vector the certificate is validated against is the WHOLE journal's,
805
+ # exactly as `_resolve_quota_cache_coverage` computes it: a root over the
806
+ # pinned prefix alone would never match one a writer stored over every
807
+ # segment.
808
+ full_vector = coverage_pinned_vector()
809
+ ok, reason = _lib_cache_coverage.certificate_is_valid(
810
+ snapshot.certificate,
811
+ pinned_vector=full_vector,
812
+ physical_seq=snapshot.physical_seq,
813
+ )
814
+ covered = (
815
+ snapshot.certificate.get("coveredHighWater") if ok else None)
816
+ finally:
817
+ _close_coverage_snapshot(snapshot)
818
+ return SegmentElisionPlan(
819
+ summaries=summaries, covered=covered, verdict=reason)
820
+
821
+
822
+ def _refill_elided_quota_raw(quota_raw, gaps):
823
+ """``(stream, complete)`` — ``quota_raw`` with the elided observations back.
824
+
825
+ Reached only when the pass elided and the cache leg's own coverage verdict
826
+ then came back something other than `ok` — an ordinary Codex batch landing
827
+ mid-pass is enough. The leg is about to REPLAY, and replaying a stream that
828
+ is missing the elided segments would leave those observations unmaterialized
829
+ while the mint asserted coverage over them.
830
+
831
+ **Rebuilt in one ascending sweep rather than spliced in place.** An elided
832
+ segment appends nothing to ``quota_raw``, so every segment of a contiguous
833
+ elided run records the SAME insertion index, and inserting each one at that
834
+ shared index pushes its predecessor later — which replays the run backwards.
835
+ Six of the maintainer's seven bootstrap segments are quota-only and
836
+ adjacent, so on that journal the whole run reverses. Order is not cosmetic:
837
+ `_QUOTA_SNAPSHOT_INSERT` is `INSERT OR IGNORE` and resolves first-wins on
838
+ the natural key, and `CodexResetAnchorResolver` decides per record in stream
839
+ order.
840
+
841
+ ``complete`` is False when any elided segment could not be re-read IN FULL.
842
+ The caller must then drop its covered boundary: the observations this stream
843
+ is missing are exactly the ones a minted certificate would claim.
844
+
845
+ **A short read is detected by counting, not by catching.**
846
+ `_iter_segment_lines` reads until its own `read()` returns nothing, so a
847
+ segment that yields fewer lines than it held ends the loop with no exception
848
+ at all. Each gap therefore carries the line count its summary recorded, and
849
+ a re-read that returns a different number reports the shortfall rather than
850
+ leaving the leg to certify a stream it never saw.
851
+ """
852
+ by_index: dict = {}
853
+ complete = True
854
+ for name, index, extent, expected_lines in gaps:
855
+ recovered = []
856
+ seen = 0
857
+ try:
858
+ for _seg, _off, raw in _iter_segment_lines(
859
+ _cctally_core.JOURNAL_DIR / name, 0, int(extent)):
860
+ seen += 1
861
+ record = _lib_journal.decode_line(raw)
862
+ if record is not None and _is_codex_quota_obs(record):
863
+ recovered.append(raw)
864
+ except Exception:
865
+ # A vanished segment self-heals, because the pinned vector no longer
866
+ # matches the journal and the certificate is invalid the moment it
867
+ # is read. A transient read error on an UNCHANGED file does not: the
868
+ # vector still matches, so a certificate minted over this shortened
869
+ # stream would be valid and would claim observations nobody applied.
870
+ # Reporting the shortfall is what stops the mint.
871
+ #
872
+ # EVERY exception, not just `OSError`: spec §6.3 makes every
873
+ # degraded state here a silent full-replay fallback, and neither the
874
+ # call site nor its caller catches anything, so one that escaped
875
+ # would abort the whole rebuild over a re-derivable optimization.
876
+ complete = False
877
+ continue
878
+ if seen != int(expected_lines):
879
+ complete = False
880
+ by_index.setdefault(int(index), []).extend(recovered)
881
+ out: list = []
882
+ cursor = 0
883
+ for index in sorted(by_index):
884
+ bounded = min(int(index), len(quota_raw))
885
+ out.extend(quota_raw[cursor:bounded])
886
+ out.extend(by_index[index])
887
+ cursor = max(cursor, bounded)
888
+ out.extend(quota_raw[cursor:])
889
+ return out, complete
890
+
891
+
892
+ class _SegmentSummaryCollector:
893
+ """Per-segment traversal facts, accumulated as the pass streams.
894
+
895
+ The counters are per segment rather than per pass because a later pass has
896
+ to contribute an ELIDED segment's share of them without reading it, and a
897
+ whole-pass total cannot be decomposed after the fact.
898
+
899
+ The last-seen fold is deliberately accumulated per segment and merged into
900
+ the caller's running accumulator at each boundary, rather than folded into
901
+ both. `LastSeenAccumulator.observe` runs once per record on a 1.95-million
902
+ line journal, and folding twice would double that; merging a small map once
903
+ per segment does not.
904
+ """
905
+
906
+ __slots__ = ("summaries", "last_seen", "_name", "_lo", "_stat",
907
+ "_lines", "_bytes", "_decodes", "_malformed", "_retained",
908
+ "_line_end")
909
+
910
+ def __init__(self) -> None:
911
+ self.summaries: dict = {}
912
+ #: The open segment's own accumulator, or None between segments.
913
+ self.last_seen = None
914
+ self._name = None
915
+ self._reset()
916
+
917
+ def _reset(self) -> None:
918
+ self._lo = 0
919
+ self._stat = None
920
+ self._lines = 0
921
+ self._bytes = 0
922
+ self._decodes = 0
923
+ self._malformed = 0
924
+ self._retained = 0
925
+ self._line_end = 0
926
+
927
+ def begin(self, name, lo, hi, stat_result, running) -> None:
928
+ """Open ``name``, closing whatever segment was open before it.
929
+
930
+ ``hi`` is this pass's pinned READ extent, which for the high-water
931
+ segment is the pinned high-water rather than the file's size. The
932
+ summary records `st_size` instead (spec §5.3), so ``hi`` is accepted for
933
+ the shared `on_extent` signature and deliberately not stored.
934
+ """
935
+ self.close(running)
936
+ self._name = name
937
+ self._reset()
938
+ self._lo = int(lo)
939
+ self._stat = stat_result
940
+ self._line_end = int(lo)
941
+ self.last_seen = _lib_journal_router.LastSeenAccumulator()
942
+
943
+ def line(self, raw, end_offset) -> None:
944
+ self._lines += 1
945
+ self._bytes += len(raw) + 1
946
+ self._line_end = int(end_offset)
947
+
948
+ def decoded(self, retained: bool) -> None:
949
+ """One decoded record: one traversal decode AND one `decoded` element.
950
+
951
+ ONE counter, because the two are the same number by construction — the
952
+ rebuild appends exactly one element to `decoded` per successfully
953
+ decoded record, the record itself when retained and a `None` placeholder
954
+ otherwise. The summary stores it under two names because they answer
955
+ different questions: `decodes` is the traversal counter the rebuild
956
+ record reports, and `decoded_entry_count` is the elision contract, which
957
+ carries a `None` sentinel a counter cannot.
958
+ """
959
+ self._decodes += 1
960
+ if retained:
961
+ self._retained += 1
962
+
963
+ def malformed_line(self) -> None:
964
+ self._malformed += 1
965
+
966
+ def close(self, running) -> None:
967
+ """Finalize the open segment and merge its last-seen into ``running``."""
968
+ if self.last_seen is not None and running is not None:
969
+ running.merge(
970
+ self.last_seen.stamped,
971
+ self.last_seen.legacy_claude_at,
972
+ self.last_seen.legacy_codex_at,
973
+ )
974
+ if self._name is not None and self._lo == 0 and self._stat is not None:
975
+ partial = self.last_seen
976
+ self.summaries[self._name] = _lib_segment_summary.SegmentSummary(
977
+ segment_name=self._name,
978
+ st_dev=int(self._stat.st_dev),
979
+ st_ino=int(self._stat.st_ino),
980
+ # The raw `st_size` observed when this summary was written (spec
981
+ # §5.3), NOT this pass's read extent. They differ only for the
982
+ # high-water segment, whose extent is the pinned high-water; a
983
+ # summary claiming that extent as the file's size would let a
984
+ # later pass elide a segment this one did not read to the end.
985
+ summarized_size=int(self._stat.st_size),
986
+ complete_line_covered_offset=self._line_end,
987
+ lines=self._lines,
988
+ bytes=self._bytes,
989
+ decodes=self._decodes,
990
+ malformed=self._malformed,
991
+ quota_only=self._retained == 0,
992
+ decoded_entry_count=self._decodes,
993
+ last_seen_stamped=(
994
+ {} if partial is None else dict(partial.stamped)),
995
+ last_seen_legacy_claude_at=(
996
+ None if partial is None else partial.legacy_claude_at),
997
+ last_seen_legacy_codex_at=(
998
+ None if partial is None else partial.legacy_codex_at),
999
+ )
1000
+ self._name = None
1001
+ self.last_seen = None
1002
+
1003
+ def adopt(self, summary) -> None:
1004
+ """Carry an ELIDED segment's stored summary forward unchanged.
1005
+
1006
+ The pass did not re-derive it, so re-deriving it here would be inventing
1007
+ it. Carrying it forward is what keeps the sidecar complete across a run
1008
+ of consecutive eliding passes.
1009
+ """
1010
+ self.summaries[summary.segment_name] = summary
1011
+
1012
+
1013
+ def coverage_pinned_vector():
1014
+ """The ordered ``(name, raw st_size, complete-line offset)`` triple per segment.
1015
+
1016
+ This is the vector the #496 S5b coverage certificate's identity root binds
1017
+ (spec §4.2), and it is deliberately ONE function rather than two: the writer
1018
+ that advances a certificate and the rebuild that validates it must compute
1019
+ the same triples from the same rules, or the root would differ for reasons
1020
+ that have nothing to do with coverage.
1021
+
1022
+ Both operands are carried because they are independent. `_repair_torn_tail`
1023
+ truncates a partial trailing line before appending, so a segment's size can
1024
+ decrease or return to a previous value — "an append strictly increases
1025
+ `st_size`" is false. A segment whose last newline sits at 100 and whose size
1026
+ is 120 is a different physical state from one where both are 100, and
1027
+ collapsing them would let a permanently torn segment certify as covered.
1028
+
1029
+ It takes no segment list. Every caller wants the WHOLE journal's vector,
1030
+ because that is the only one a stored certificate's identity root can be
1031
+ validated against: a root computed over a prefix would never match one a
1032
+ writer stored over every segment.
1033
+ """
1034
+ vector = []
1035
+ for name in list_segments():
1036
+ path = _cctally_core.JOURNAL_DIR / name
1037
+ try:
1038
+ size = os.stat(path).st_size
1039
+ except OSError:
1040
+ # A segment that vanished between the listing and the stat leaves a
1041
+ # vector nothing can match, which falls back to a full replay. That
1042
+ # is the right answer: the journal shape moved under this pass.
1043
+ size = -1
1044
+ vector.append(
1045
+ (name, size, 0 if size < 0 else _complete_line_offset(path, size)))
1046
+ return tuple(vector)
1047
+
1048
+
509
1049
  def journal_high_water() -> tuple[str, int] | None:
510
1050
  """Snapshot ``(latest segment basename, size)`` under a µs leaf-lock hold.
511
1051
 
@@ -1024,7 +1564,7 @@ def iter_range(cursor, hw):
1024
1564
 
1025
1565
 
1026
1566
  def _iter_range_with_segments(cursor, hw, segments, *, on_segment=None,
1027
- on_bytes=None):
1567
+ on_bytes=None, on_extent=None, elide=None):
1028
1568
  """`iter_range` over a segment list the CALLER snapshotted (#496 S4 §4).
1029
1569
 
1030
1570
  `list_segments()` enumerates the journal directory at call time and orders
@@ -1038,6 +1578,18 @@ def _iter_range_with_segments(cursor, hw, segments, *, on_segment=None,
1038
1578
  function then skips because it holds no bytes in range: `journal_prefix_hash`
1039
1579
  frames a zero-byte segment, so a hash accumulator has to be told it exists.
1040
1580
  `on_bytes` is forwarded to `_iter_segment_lines`.
1581
+
1582
+ `on_extent(seg, lo, hi, stat_result)` is called after this pass has pinned
1583
+ the range it will read from that segment, so a caller building a durable
1584
+ summary records the extent the pass actually covered rather than one it
1585
+ re-stats afterwards (#496 S5b section 5.3).
1586
+
1587
+ `elide(seg, lo, hi, stat_result)` decides whether to SKIP the segment
1588
+ entirely — no `_open_segment_for_read`, no bytes, no lines. It runs after
1589
+ `on_extent` and instead of `on_segment`, because a skipped segment
1590
+ contributes nothing to a hash accumulator that could not compose over it
1591
+ anyway (spec section 5.1). Only #496 S5b Stage 4's elision planner supplies
1592
+ it; every other caller reads every segment exactly as before.
1041
1593
  """
1042
1594
  hw_seg, hw_size = hw
1043
1595
  if hw_seg not in segments:
@@ -1055,7 +1607,20 @@ def _iter_range_with_segments(cursor, hw, segments, *, on_segment=None,
1055
1607
  seg = segments[idx]
1056
1608
  seg_path = _cctally_core.JOURNAL_DIR / seg
1057
1609
  lo = start_off if idx == start_idx else 0
1058
- hi = hw_size if idx == hw_idx else os.path.getsize(seg_path)
1610
+ stat_result = None
1611
+ if idx == hw_idx:
1612
+ hi = hw_size
1613
+ else:
1614
+ stat_result = os.stat(seg_path)
1615
+ hi = stat_result.st_size
1616
+ if on_extent is not None or elide is not None:
1617
+ if stat_result is None:
1618
+ stat_result = os.stat(seg_path)
1619
+ if elide is not None and lo == 0 and idx != hw_idx and elide(
1620
+ seg, lo, hi, stat_result):
1621
+ continue
1622
+ if on_extent is not None:
1623
+ on_extent(seg, lo, hi, stat_result)
1059
1624
  if on_segment is not None:
1060
1625
  on_segment(seg)
1061
1626
  if lo >= hi:
@@ -1343,7 +1908,7 @@ def _apply_file_account_records(cache, records) -> "tuple[int, int]":
1343
1908
  return restored, conflicts
1344
1909
 
1345
1910
 
1346
- def _report_file_account_conflicts(conflicts: int) -> None:
1911
+ def _report_file_account_conflicts(conflicts: int, *, quiet: bool = False) -> None:
1347
1912
  """One stderr line for a run of replayed decisions that contradicted a
1348
1913
  different account already recorded at the same
1349
1914
  ``(file_identity, incarnation, from_offset)`` and were therefore DECLINED
@@ -1353,8 +1918,13 @@ def _report_file_account_conflicts(conflicts: int) -> None:
1353
1918
 
1354
1919
  Every call site must invoke this AFTER its commit (closeout review C5): a
1355
1920
  rolled-back transaction applied nothing, so reporting from inside it would
1356
- tell the operator about a decline that did not happen."""
1357
- if conflicts > 0:
1921
+ tell the operator about a decline that did not happen.
1922
+
1923
+ ``quiet`` is the reconciliation's caller. That path is reachable from an
1924
+ ordinary command, where acceptance criterion 10 requires no new stderr
1925
+ line; the same decline is still reported by the ingest and rebuild paths
1926
+ that own the remedy this line names."""
1927
+ if conflicts > 0 and not quiet:
1358
1928
  print(
1359
1929
  f"[ingest] codex attribution replay declined {conflicts} "
1360
1930
  "contradicting decision(s); the first journalled decision for each "
@@ -1550,11 +2120,27 @@ def _resolve_obs_anchor(resolver, rec: dict) -> "str | None":
1550
2120
  return None
1551
2121
 
1552
2122
 
1553
- def _apply_quota_records(cache, records) -> None:
2123
+ def _apply_quota_records(
2124
+ cache, records, *, reported_conflicts=None, quiet=False,
2125
+ ) -> None:
1554
2126
  """Materialize Codex quota obs into an OPEN cache.db transaction, applying
1555
2127
  the §3.5 precedence rule. Callers must apply the batch's file-account
1556
2128
  decisions FIRST, so a decision arriving in the same batch already governs
1557
- the observations it covers."""
2129
+ the observations it covers.
2130
+
2131
+ ``reported_conflicts`` lets ONE logical pass thread its conflict-report set
2132
+ across several calls (spec §4.6). The rebuild's recovery pass is chunked into
2133
+ many transactions, and without threading it would emit one line per chunk for
2134
+ a condition a single unchunked call reports once. Every other caller passes
2135
+ None and gets today's per-call set.
2136
+
2137
+ ``quiet`` suppresses the conflict line, for the same reason
2138
+ `_report_file_account_conflicts` takes it: routing
2139
+ `recover_quota_cache_from_journal` through the shared leg made this line
2140
+ reachable from the open-time reconciliation, so `cache-sync` and the
2141
+ dashboard would emit a line they never emitted before, and acceptance
2142
+ criterion 10 requires no new stderr on an ordinary command. Its sibling was
2143
+ guarded and this one was not."""
1558
2144
  oracle = _CodexAttributionOracle(cache)
1559
2145
  # One line per FILE PER BATCH, not per record. A mid-file account switch
1560
2146
  # legitimately produces a run of observations whose first-stamp-wins account
@@ -1566,7 +2152,8 @@ def _apply_quota_records(cache, records) -> None:
1566
2152
  # standing condition a handful of times is the cheaper error. The condition
1567
2153
  # is worth reporting at all because a genuine correction is expressed as an
1568
2154
  # explicit new range decision, never by mutating history.
1569
- reported_conflicts: set = set()
2155
+ if reported_conflicts is None:
2156
+ reported_conflicts = set()
1570
2157
  # #416 spec §4.2: this leg is a genuine INGEST into cache.db (it materializes
1571
2158
  # observations whose source rollout may have evaporated), so it must resolve
1572
2159
  # the canonical anchor too. Without it, a journal-replayed row lands with a
@@ -1593,20 +2180,74 @@ def _apply_quota_records(cache, records) -> None:
1593
2180
  conflict_key = (payload.get("source_root_key"), payload.get("source_path"))
1594
2181
  if (observed is not None and observed != decided
1595
2182
  and conflict_key not in reported_conflicts):
2183
+ # The key is recorded even when quiet, so a later non-quiet call
2184
+ # threading the same set still reports each file at most once.
1596
2185
  reported_conflicts.add(conflict_key)
1597
- print(
1598
- "[ingest] codex attribution conflict: "
1599
- f"{payload.get('source_path')}@{payload.get('line_offset')} "
1600
- f"observation stamped {observed} but the durable decision says "
1601
- f"{decided if decided is not None else 'unattributed'}; "
1602
- "keeping the decision",
1603
- file=sys.stderr,
1604
- )
2186
+ if not quiet:
2187
+ print(
2188
+ "[ingest] codex attribution conflict: "
2189
+ f"{payload.get('source_path')}"
2190
+ f"@{payload.get('line_offset')} "
2191
+ "observation stamped "
2192
+ f"{observed} but the durable decision says "
2193
+ f"{decided if decided is not None else 'unattributed'}; "
2194
+ "keeping the decision",
2195
+ file=sys.stderr,
2196
+ )
1605
2197
  values = list(row_values)
1606
2198
  values[16] = decided
1607
2199
  cache.execute(upsert_sql, tuple(values))
1608
2200
 
1609
2201
 
2202
+ class CoverageInvariantViolation(RuntimeError):
2203
+ """A caller reached a covered-family delete without invalidating coverage.
2204
+
2205
+ Raised rather than asserted, because `python -O` strips `assert` and this is
2206
+ the only guard standing between a new `authoritative=True` caller and a
2207
+ certificate left standing over a table that was just emptied.
2208
+ """
2209
+
2210
+
2211
+ def _assert_coverage_already_invalidated(cache_conn) -> None:
2212
+ """Refuse an authoritative replay while a coverage certificate stands.
2213
+
2214
+ `rehydrate_codex_file_accounts(authoritative=True)` empties
2215
+ `codex_file_accounts`, a member of `COVERAGE_CACHE_FAMILIES`, before
2216
+ replaying, yet its inventory entry is `preserve`. That holds only because
2217
+ its one caller — `sync_codex_cache` under `--rebuild` — already ran
2218
+ `_clear_codex_derived_rows` and committed, which deleted both the
2219
+ certificate and any recovery progress. A second caller, or a reordering
2220
+ inside `sync_codex_cache`, would break it silently and the writer-surface
2221
+ scanner could not see it, because the key is already in the inventory with a
2222
+ green label. This turns that ordering into a checked precondition.
2223
+
2224
+ **Where to look when it fires.** No legitimate ordering reaches it today,
2225
+ and six were checked. If a future caller does, `sync_codex_cache`'s
2226
+ `except Exception` catches `CoverageInvariantViolation`, rolls back, sets
2227
+ `deferred_reason = "attribution_rehydration"`, prints one line and skips the
2228
+ whole Codex walk — so the symptom is a permanently deferred Codex sync
2229
+ rather than a loud failure. That containment is the right production trade
2230
+ and is not changed here; the note exists so the next debugger looks for the
2231
+ new caller rather than for a broken walk.
2232
+ """
2233
+ try:
2234
+ row = cache_conn.execute(
2235
+ "SELECT key FROM cache_meta WHERE key IN (?, ?) LIMIT 1",
2236
+ (_lib_cache_coverage.CERTIFICATE_KEY,
2237
+ _lib_cache_coverage.PROGRESS_KEY),
2238
+ ).fetchone()
2239
+ except sqlite3.Error:
2240
+ # An unreadable `cache_meta` cannot witness the invariant either way,
2241
+ # and raising here would turn a degraded cache into a failed sync.
2242
+ return
2243
+ if row is not None:
2244
+ raise CoverageInvariantViolation(
2245
+ "rehydrate_codex_file_accounts(authoritative=True) clears a covered "
2246
+ f"family while {row[0]!r} is still stored; the caller must "
2247
+ "invalidate coverage in the transaction that clears"
2248
+ )
2249
+
2250
+
1610
2251
  def rehydrate_codex_file_accounts(
1611
2252
  cache_conn, *, authoritative: bool = False, since=None,
1612
2253
  ) -> "tuple[int, tuple[str, int] | None, int]":
@@ -1674,6 +2315,8 @@ def rehydrate_codex_file_accounts(
1674
2315
  upsert already converges the counter, a clear has no upside and that
1675
2316
  downside.
1676
2317
  """
2318
+ if authoritative:
2319
+ _assert_coverage_already_invalidated(cache_conn)
1677
2320
  hw = journal_high_water()
1678
2321
  if hw is None:
1679
2322
  if authoritative:
@@ -1706,7 +2349,78 @@ def rehydrate_codex_file_accounts(
1706
2349
  return applied, hw, conflicts
1707
2350
 
1708
2351
 
1709
- def _cache_applier(decoded) -> int | None:
2352
+ def _bounded_covered_offset(segment, raw_offset, covered_offset, decoded_end):
2353
+ """The covered boundary for ``segment``, bounded by what a pass DECODED.
2354
+
2355
+ Three independent upper bounds, and the smallest wins:
2356
+
2357
+ - ``raw_offset`` — the pass's own high water; nothing past it was read.
2358
+ - ``covered_offset`` — the segment's complete-line offset, so a torn
2359
+ trailing line is never inside the claim.
2360
+ - ``decoded_end`` — the end of the last line the pass CONSUMED, never past
2361
+ it. The two callers supply that operand differently and both are safe.
2362
+ `_coverage_advance_plan` (the ingest cycle) passes the end of the last
2363
+ record it DECODED, so a malformed line the batch skipped is excluded.
2364
+ `rebuild_stats_index` passes `prior_high_water`, which the streaming pass
2365
+ advances for a malformed line too — consumed but not decoded — and that is
2366
+ still an upper bound on what was read, so the claim stays true. Reading
2367
+ the operand as "the last DECODED record" is literally true only of the
2368
+ first caller.
2369
+
2370
+ The third is not redundant with the second, and leaving it out was a real
2371
+ defect. Both other operands are read from the file, and the file is re-stat'd
2372
+ AFTER the pass finished reading: a segment torn at the pinned high water and
2373
+ repaired by a later append has a current complete-line offset at or past that
2374
+ high water, so `min(raw, covered)` returns the raw high water — a boundary
2375
+ whose trailing bytes the traversal never decoded, because the partial line
2376
+ was skipped. A newline-terminated MALFORMED trailing line produces the same
2377
+ divergence without any repair at all. ``decoded_end`` is the only operand
2378
+ the pass observed rather than inferred.
2379
+
2380
+ A ``decoded_end`` in an EARLIER segment answers 0: the pass covered
2381
+ everything before this segment and none of it, which is exactly expressible
2382
+ and stays true.
2383
+ """
2384
+ bound = min(int(raw_offset), int(covered_offset))
2385
+ if decoded_end is None:
2386
+ return bound
2387
+ if str(decoded_end[0]) != str(segment):
2388
+ return 0
2389
+ return min(bound, int(decoded_end[1]))
2390
+
2391
+
2392
+ def _coverage_advance_plan(cursor, covered_to, decoded_end=None):
2393
+ """``(pinned_vector, covered, applied_through)`` for an advance, or None.
2394
+
2395
+ Captured BEFORE the cache flocks are acquired, and that ordering is the
2396
+ safety property. A vector captured before a concurrent append describes a
2397
+ SMALLER journal than the one that exists when the certificate is stored, so
2398
+ the stored root stops matching and the next rebuild replays. A vector
2399
+ captured after such an append would match the current journal while a record
2400
+ nobody applied sat inside it — coverage asserted over an unapplied record,
2401
+ which is the one failure the certificate must not produce.
2402
+
2403
+ ``covered`` is bounded by `_bounded_covered_offset`; ``applied_through`` is
2404
+ the raw coordinate the cycle advances its cursor to. The two are returned
2405
+ separately because the certificate stores both: the next writer's contiguity
2406
+ check compares its starting cursor against `appliedThrough`, and comparing
2407
+ it against the clamped boundary instead made a single torn or malformed
2408
+ trailing line freeze the certificate permanently.
2409
+ """
2410
+ if cursor is None or covered_to is None:
2411
+ return None
2412
+ vector = coverage_pinned_vector()
2413
+ segment, offset = str(covered_to[0]), int(covered_to[1])
2414
+ for name, _raw_extent, covered_offset in vector:
2415
+ if name == segment:
2416
+ bounded = _bounded_covered_offset(
2417
+ segment, offset, covered_offset, decoded_end)
2418
+ return vector, (segment, bounded), (segment, offset)
2419
+ return None
2420
+
2421
+
2422
+ def _cache_applier(decoded, *, cursor=None, covered_to=None,
2423
+ decoded_end=None) -> int | None:
1710
2424
  """Composite cache leg (spec §5.2 step 3 + #416 spec §3.4): materialize this
1711
2425
  batch's Codex quota obs into `quota_window_snapshots` AND its
1712
2426
  `codex_file_account` ops into the attribution map, under the NON-BLOCKING
@@ -1723,6 +2437,17 @@ def _cache_applier(decoded) -> int | None:
1723
2437
  - Flock acquired + everything upserted → return None (full consumption).
1724
2438
  A quota-row change advances ``codex_physical_mutation_seq`` in the same
1725
2439
  transaction; an idempotent replay leaves the sequence unchanged.
2440
+
2441
+ ``cursor``, ``covered_to`` and ``decoded_end`` carry the cycle's journal
2442
+ range so this leg can ADVANCE the #496 S5b coverage certificate (spec §4.3).
2443
+ It is the writer that can do so soundly, because it owns a CONTIGUOUS batch:
2444
+ it consumed every record in `[cursor, covered_to]` and applied every
2445
+ cache-relevant one, so a predecessor applied through exactly `cursor`
2446
+ extends to `covered_to`. ``decoded_end`` is the end coordinate of the last
2447
+ record the cycle decoded, and it bounds the covered CLAIM below the raw
2448
+ cursor target whenever the two differ. All three default to None, which
2449
+ advances nothing — the ingest cycle is the only caller that knows the range,
2450
+ and a test calling this directly must not mint coverage it cannot justify.
1726
2451
  """
1727
2452
  quota_idx = [i for i, (rec, _s, _o) in enumerate(decoded)
1728
2453
  if _is_codex_quota_obs(rec)]
@@ -1730,6 +2455,10 @@ def _cache_applier(decoded) -> int | None:
1730
2455
  if _is_codex_file_account_op(rec)]
1731
2456
  if not quota_idx and not file_idx:
1732
2457
  return None
2458
+ # BEFORE the flocks, for the reason `_coverage_advance_plan` states, and
2459
+ # before them for a second reason too: it is journal file I/O, and the leg's
2460
+ # whole purpose is to hold the global cache writer lock as briefly as it can.
2461
+ plan = _coverage_advance_plan(cursor, covered_to, decoded_end)
1733
2462
  # All-or-nothing across the two families: one stop, the earliest of either.
1734
2463
  stop_idx = min(quota_idx[0] if quota_idx else file_idx[0],
1735
2464
  file_idx[0] if file_idx else quota_idx[0])
@@ -1755,7 +2484,25 @@ def _cache_applier(decoded) -> int | None:
1755
2484
  print(f"[ingest] cache leg connect failed: {exc}", file=sys.stderr)
1756
2485
  return stop_idx
1757
2486
  try:
2487
+ import _cctally_cache
1758
2488
  cache.execute("PRAGMA busy_timeout=15000")
2489
+ # Read and check the predecessor BEFORE the transaction opens
2490
+ # (spec §4.3, as corrected). `prior_is_extendable` checks contiguity
2491
+ # against this cycle's STARTING cursor — a predecessor applied
2492
+ # through less leaves a gap nobody applied, and one applied through
2493
+ # more was written by a pass that saw records this batch does not
2494
+ # carry — AND the two version fields, because `advance` re-stamps
2495
+ # the current module constants and discards `prior`, so extending a
2496
+ # certificate written under an older `interpretationVersion` would
2497
+ # launder it into a current-version one. It deliberately does not
2498
+ # run the full `certificate_is_valid`: an advance's predecessor
2499
+ # necessarily describes an older, smaller journal.
2500
+ prior = _cctally_cache.load_codex_journal_coverage_certificate(cache)
2501
+ if plan is not None:
2502
+ extendable, _why = _lib_cache_coverage.prior_is_extendable(
2503
+ prior, applied_through=(str(cursor[0]), int(cursor[1])))
2504
+ if not extendable:
2505
+ prior = None
1759
2506
  cache.execute("BEGIN IMMEDIATE")
1760
2507
  # Decisions FIRST: §3.5 makes the file/range decision authoritative
1761
2508
  # over the observation stamp, so a decision arriving in this batch
@@ -1768,8 +2515,15 @@ def _cache_applier(decoded) -> int | None:
1768
2515
  # #457: this path is independent of the fused rollout writer,
1769
2516
  # but its quota rows feed the same certificate and dashboard
1770
2517
  # signatures. Keep the token atomic with the materialization.
1771
- import _cctally_cache
1772
2518
  _cctally_cache._bump_codex_physical_mutation_seq(cache)
2519
+ if plan is not None:
2520
+ # AFTER the bump, so the certificate carries the post-bump
2521
+ # sequence, and inside this transaction so a rollback leaves the
2522
+ # predecessor standing even when the journal appends survived.
2523
+ vector, covered, applied_through = plan
2524
+ _cctally_cache._advance_codex_journal_coverage(
2525
+ cache, prior=prior, covered=covered,
2526
+ applied_through=applied_through, pinned_vector=vector)
1773
2527
  cache.commit()
1774
2528
  _report_file_account_conflicts(_file_conflicts)
1775
2529
  except sqlite3.Error as exc:
@@ -3084,64 +3838,479 @@ def _fold_order(evt) -> int:
3084
3838
  return (_EVT_SPECS.get(kind) or _UNKNOWN_EVT_SPEC).order
3085
3839
 
3086
3840
 
3087
- def _write_effective_metadata(conn, selection) -> None:
3088
- """Replace the disposable effective-event summary from a pure selection."""
3841
+ def _replace_protocol_violations(conn, rows) -> None:
3842
+ """Replace the disposable structural-violation summary from kernel rows.
3843
+
3844
+ ONE writer, shared by the rebuild's whole-generation write and the live
3845
+ path's full-prefix fallback, because the two must produce IDENTICAL rows.
3846
+ They did not: the fallback wrote four columns and left `available_after`
3847
+ NULL, and it serialized with `ensure_ascii=True` while the kernel uses
3848
+ `ensure_ascii=False`. A fallback tick therefore left rows a fresh derivation
3849
+ would not match — always on `available_after`, and on the JSON bytes for any
3850
+ non-ASCII character in a violation payload — so
3851
+ `stats_index_matches_journal_prefix` answered False afterwards and the
3852
+ incremental path carried the wrong evidence forward verbatim.
3853
+ """
3854
+ conn.execute("DELETE FROM journal_protocol_violations")
3855
+ for row in rows:
3856
+ conn.execute(
3857
+ "INSERT INTO journal_protocol_violations "
3858
+ "(fingerprint, batch_id, kind, violation_json, available_after) "
3859
+ "VALUES (?, ?, ?, ?, ?)",
3860
+ (
3861
+ row.fingerprint,
3862
+ row.batch_id,
3863
+ row.kind,
3864
+ row.violation_json,
3865
+ row.available_after,
3866
+ ),
3867
+ )
3868
+
3869
+
3870
+ def _write_selector_state(conn, rows) -> None:
3871
+ """Replace the whole durable selector generation from pure-kernel rows.
3872
+
3873
+ This took over from `_write_effective_metadata`, which it superseded and
3874
+ which is now deleted: it writes the same `journal_effective_events` and
3875
+ `journal_protocol_violations` content plus the two added columns, the
3876
+ retained violation evidence, and the three selector tables. Everything lands
3877
+ in ONE transaction with the index content, so a generation never publishes
3878
+ selector state describing a different fold.
3879
+ """
3089
3880
  conn.execute("DELETE FROM journal_effective_events")
3090
- for event_id, selected in selection.by_id.items():
3091
- event_json = None
3092
- if selected.record is not None:
3093
- event_json = (
3094
- _lib_journal.encode_line(selected.record)
3095
- .decode("utf-8")
3096
- .rstrip("\n")
3097
- )
3881
+ for row in rows.effective:
3098
3882
  conn.execute(
3099
3883
  "INSERT INTO journal_effective_events "
3100
- "(event_id, rev, status, content_hash, batch_id, event_json) "
3101
- "VALUES (?, ?, ?, ?, ?, ?)",
3884
+ "(event_id, rev, status, content_hash, batch_id, event_json, "
3885
+ " winning_sequence, conflict_hashes_json) "
3886
+ "VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
3102
3887
  (
3103
- event_id,
3104
- selected.rev,
3105
- selected.status,
3106
- selected.content_hash,
3107
- selected.batch_id,
3108
- event_json,
3888
+ row.event_id,
3889
+ row.rev,
3890
+ row.status,
3891
+ row.content_hash,
3892
+ row.batch_id,
3893
+ row.event_json,
3894
+ row.winning_sequence,
3895
+ row.conflict_hashes_json,
3109
3896
  ),
3110
3897
  )
3111
- _write_protocol_violations(
3112
- conn,
3113
- selection.protocol_violations,
3114
- selection.acknowledged_protocol_violations,
3898
+ _replace_protocol_violations(conn, rows.violations)
3899
+ conn.execute("DELETE FROM journal_selector_batch_records")
3900
+ for row in rows.batch_records:
3901
+ conn.execute(
3902
+ "INSERT INTO journal_selector_batch_records "
3903
+ "(batch_id, kind, key, record_digest, identity_digest, sequence, "
3904
+ " action_core_json) VALUES (?, ?, ?, ?, ?, ?, ?)",
3905
+ (
3906
+ row.batch_id,
3907
+ row.kind,
3908
+ row.key,
3909
+ row.record_digest,
3910
+ row.identity_digest,
3911
+ row.sequence,
3912
+ row.action_core_json,
3913
+ ),
3914
+ )
3915
+ conn.execute("DELETE FROM journal_selector_batches")
3916
+ for row in rows.batches:
3917
+ conn.execute(
3918
+ "INSERT INTO journal_selector_batches "
3919
+ "(batch_id, status, action_count, action_set_hash, begin_segment, "
3920
+ " begin_offset, earliest_commit_segment, earliest_commit_offset) "
3921
+ "VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
3922
+ (
3923
+ row.batch_id,
3924
+ row.status,
3925
+ row.action_count,
3926
+ row.action_set_hash,
3927
+ row.begin_segment,
3928
+ row.begin_offset,
3929
+ row.earliest_commit_segment,
3930
+ row.earliest_commit_offset,
3931
+ ),
3932
+ )
3933
+ conn.execute("DELETE FROM journal_selector_state")
3934
+ state = rows.state
3935
+ conn.execute(
3936
+ "INSERT INTO journal_selector_state "
3937
+ "(id, generation_record_path, generation_stamped_at_utc, "
3938
+ " covered_segment, covered_offset, next_sequence, selector_version, "
3939
+ " cutover_seen, cutover_account_key) "
3940
+ "VALUES (1, ?, ?, ?, ?, ?, ?, ?, ?)",
3941
+ (
3942
+ state.generation_record_path,
3943
+ state.generation_stamped_at_utc,
3944
+ state.covered_segment,
3945
+ state.covered_offset,
3946
+ state.next_sequence,
3947
+ state.selector_version,
3948
+ 1 if state.cutover_seen else 0,
3949
+ state.cutover_account_key,
3950
+ ),
3115
3951
  )
3116
3952
 
3117
3953
 
3118
- def _write_protocol_violations(conn, violations, acknowledged=()) -> None:
3119
- """Replace the disposable structural-violation summary."""
3120
- conn.execute("DELETE FROM journal_protocol_violations")
3121
- rows = [*violations, *acknowledged]
3122
- rows.sort(
3123
- key=lambda violation: (
3124
- violation.batch_id,
3125
- violation.kind,
3126
- violation.fingerprint,
3127
- )
3954
+ def _write_selector_delta(conn, before, after) -> None:
3955
+ """Advance durable selector state from ``before`` to ``after`` in place.
3956
+
3957
+ A whole-generation replace is what the rebuild does, and it is the wrong
3958
+ shape for an ingest tick: `journal_effective_events` holds one row per event
3959
+ id 34,644 on the maintainer's production journal — and rewriting all of
3960
+ every status-line tick would trade the read this session removes for a write
3961
+ of the same size. So only the rows that actually changed are written.
3962
+
3963
+ Three of the four groups express upserts only, and the reason is that **the
3964
+ fold emits no removals** for them: `after ⊇ before` in every one. It is NOT
3965
+ that a key missing from ``after`` lies outside the delta's scope — ``after``
3966
+ is computed FROM ``before``, so every key in ``before`` is by construction
3967
+ inside the read scope, and a key the kernel dropped would be a genuine
3968
+ removal. A merged generation keeps every winner and every batch, and a
3969
+ completed batch's action rows keep their keys with a NULL core rather than
3970
+ disappearing.
3971
+
3972
+ Violations are the exception, so that group DOES express removals. A phase-2
3973
+ verdict is re-derived on every pass and a later record can withdraw one — an
3974
+ incomplete action set completed by a late action stops producing
3975
+ `manifest_action_sequence_mismatch` — and `_check_journal_protocol` reads
3976
+ that table, so a stale row makes `doctor` exit 2 and names a fingerprint no
3977
+ fresh derivation reproduces. The removal is safe precisely because it is
3978
+ scoped: ``before.violations`` is the delta's own scoped read, so the
3979
+ difference can only name rows this delta looked at.
3980
+ """
3981
+ state = after.state
3982
+ conn.execute(
3983
+ "INSERT INTO journal_selector_state "
3984
+ "(id, generation_record_path, generation_stamped_at_utc, "
3985
+ " covered_segment, covered_offset, next_sequence, selector_version, "
3986
+ " cutover_seen, cutover_account_key) "
3987
+ "VALUES (1, ?, ?, ?, ?, ?, ?, ?, ?) "
3988
+ "ON CONFLICT(id) DO UPDATE SET "
3989
+ "generation_record_path = excluded.generation_record_path, "
3990
+ "generation_stamped_at_utc = excluded.generation_stamped_at_utc, "
3991
+ "covered_segment = excluded.covered_segment, "
3992
+ "covered_offset = excluded.covered_offset, "
3993
+ "next_sequence = excluded.next_sequence, "
3994
+ "selector_version = excluded.selector_version, "
3995
+ "cutover_seen = excluded.cutover_seen, "
3996
+ "cutover_account_key = excluded.cutover_account_key",
3997
+ (
3998
+ state.generation_record_path,
3999
+ state.generation_stamped_at_utc,
4000
+ state.covered_segment,
4001
+ state.covered_offset,
4002
+ state.next_sequence,
4003
+ state.selector_version,
4004
+ 1 if state.cutover_seen else 0,
4005
+ state.cutover_account_key,
4006
+ ),
3128
4007
  )
3129
- for violation in rows:
3130
- conn.execute(
3131
- "INSERT INTO journal_protocol_violations "
3132
- "(fingerprint, batch_id, kind, violation_json) "
3133
- "VALUES (?, ?, ?, ?)",
3134
- (
3135
- violation.fingerprint,
3136
- violation.batch_id,
3137
- violation.kind,
3138
- json.dumps(
3139
- violation.to_dict(),
3140
- sort_keys=True,
3141
- separators=(",", ":"),
4008
+
4009
+ # Identity, not equality: `advance_counter` returns the row tuples BY
4010
+ # REFERENCE for a delta the fold does not consume, so a tick that only
4011
+ # moved the counters skips these diffs entirely rather than walking every
4012
+ # durable row to conclude that nothing changed.
4013
+ if before.batches is not after.batches:
4014
+ prior_batches = {row.batch_id: row for row in before.batches}
4015
+ for row in after.batches:
4016
+ if prior_batches.get(row.batch_id) == row:
4017
+ continue
4018
+ conn.execute(
4019
+ "INSERT OR REPLACE INTO journal_selector_batches "
4020
+ "(batch_id, status, action_count, action_set_hash, begin_segment, "
4021
+ " begin_offset, earliest_commit_segment, earliest_commit_offset) "
4022
+ "VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
4023
+ (
4024
+ row.batch_id,
4025
+ row.status,
4026
+ row.action_count,
4027
+ row.action_set_hash,
4028
+ row.begin_segment,
4029
+ row.begin_offset,
4030
+ row.earliest_commit_segment,
4031
+ row.earliest_commit_offset,
3142
4032
  ),
3143
- ),
4033
+ )
4034
+
4035
+ if before.batch_records is not after.batch_records:
4036
+ prior_records = {
4037
+ (row.batch_id, row.kind, row.key): row for row in before.batch_records
4038
+ }
4039
+ for row in after.batch_records:
4040
+ key = (row.batch_id, row.kind, row.key)
4041
+ if prior_records.get(key) == row:
4042
+ continue
4043
+ conn.execute(
4044
+ "INSERT OR REPLACE INTO journal_selector_batch_records "
4045
+ "(batch_id, kind, key, record_digest, identity_digest, sequence, "
4046
+ " action_core_json) VALUES (?, ?, ?, ?, ?, ?, ?)",
4047
+ (
4048
+ row.batch_id,
4049
+ row.kind,
4050
+ row.key,
4051
+ row.record_digest,
4052
+ row.identity_digest,
4053
+ row.sequence,
4054
+ row.action_core_json,
4055
+ ),
4056
+ )
4057
+
4058
+ if before.effective is not after.effective:
4059
+ prior_effective = {row.event_id: row for row in before.effective}
4060
+ for row in after.effective:
4061
+ if prior_effective.get(row.event_id) == row:
4062
+ continue
4063
+ conn.execute(
4064
+ "INSERT OR REPLACE INTO journal_effective_events "
4065
+ "(event_id, rev, status, content_hash, batch_id, event_json, "
4066
+ " winning_sequence, conflict_hashes_json) "
4067
+ "VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
4068
+ (
4069
+ row.event_id,
4070
+ row.rev,
4071
+ row.status,
4072
+ row.content_hash,
4073
+ row.batch_id,
4074
+ row.event_json,
4075
+ row.winning_sequence,
4076
+ row.conflict_hashes_json,
4077
+ ),
4078
+ )
4079
+
4080
+ if before.violations is not after.violations:
4081
+ prior_violations = {row.fingerprint: row for row in before.violations}
4082
+ withdrawn = set(prior_violations) - {
4083
+ row.fingerprint for row in after.violations
4084
+ }
4085
+ for fingerprint in sorted(withdrawn):
4086
+ conn.execute(
4087
+ "DELETE FROM journal_protocol_violations WHERE fingerprint = ?",
4088
+ (fingerprint,),
4089
+ )
4090
+ for row in after.violations:
4091
+ if prior_violations.get(row.fingerprint) == row:
4092
+ continue
4093
+ conn.execute(
4094
+ "INSERT OR REPLACE INTO journal_protocol_violations "
4095
+ "(fingerprint, batch_id, kind, violation_json, available_after) "
4096
+ "VALUES (?, ?, ?, ?, ?)",
4097
+ (
4098
+ row.fingerprint,
4099
+ row.batch_id,
4100
+ row.kind,
4101
+ row.violation_json,
4102
+ row.available_after,
4103
+ ),
4104
+ )
4105
+
4106
+
4107
+ def _read_selector_state(conn):
4108
+ """`journal_selector_state`'s single row as a kernel row, or None.
4109
+
4110
+ Split out of `_read_selector_rows` because the three cheap validity checks —
4111
+ `selector_version`, cursor agreement and generation identity — are decided
4112
+ from this row alone. Materializing the four row groups ahead of them made a
4113
+ degraded tick and a counters-only tick read roughly 100,000 rows to conclude
4114
+ that nothing was needed, on the status-line path.
4115
+
4116
+ Returns None when the table does not hold exactly one row, or is unreadable
4117
+ at all: a stats index predating epoch 1009 has no selector tables, and
4118
+ `open_db` returns a legacy one unchanged until its migration or epoch path
4119
+ runs. Both are degraded states that fall back to full selection, and neither
4120
+ may be mistaken for an empty-but-valid generation.
4121
+ """
4122
+ try:
4123
+ state_rows = conn.execute(
4124
+ "SELECT generation_record_path, generation_stamped_at_utc, "
4125
+ "covered_segment, covered_offset, next_sequence, selector_version, "
4126
+ "cutover_seen, cutover_account_key FROM journal_selector_state"
4127
+ ).fetchall()
4128
+ except sqlite3.Error:
4129
+ return None
4130
+ if len(state_rows) != 1:
4131
+ return None
4132
+ row = state_rows[0]
4133
+ return _lib_selector_state.SelectorStateRow(
4134
+ next_sequence=int(row[4]),
4135
+ selector_version=int(row[5]),
4136
+ covered_segment=row[2],
4137
+ covered_offset=None if row[3] is None else int(row[3]),
4138
+ generation_record_path=row[0],
4139
+ generation_stamped_at_utc=row[1],
4140
+ cutover_seen=bool(row[6]),
4141
+ cutover_account_key=row[7],
4142
+ )
4143
+
4144
+
4145
+ def _read_selector_rows(conn, *, batch_ids=None, event_ids=None):
4146
+ """Read one generation's durable selector state back as kernel rows.
4147
+
4148
+ Returns `None` when `journal_selector_state` does not hold exactly one row.
4149
+ A zero-row or unreadable state is one of the degraded cases that falls back
4150
+ to full selection, and it must never be mistaken for an empty-but-valid
4151
+ generation.
4152
+
4153
+ ``batch_ids`` and ``event_ids`` SCOPE the four row groups to what one delta
4154
+ can reach. Both default to None, which reads the whole generation — the
4155
+ shape validation and the rebuild need. The live path always scopes, because
4156
+ the unscoped read materializes 34,644 `journal_effective_events` rows (mean
4157
+ 825 bytes of retained JSON) and 64,248 `journal_selector_batch_records` rows
4158
+ on a production journal, on every status-line tick.
4159
+
4160
+ Under a scope the returned rows are a SUBSET, so the caller must not treat a
4161
+ key absent from them as a key that was removed — `_write_selector_delta`
4162
+ documents that consequence at the write end.
4163
+
4164
+ The five reads run inside ONE deferred read transaction, so they share a
4165
+ snapshot. Without it each takes its own, and a publication landing between
4166
+ two of them would return rows from two generations under one state row's
4167
+ identity. Every current caller holds the ingest or maintenance lock, so that
4168
+ interleaving is not reachable today; the transaction is what keeps it
4169
+ unreachable if a future caller does not.
4170
+ """
4171
+ _ks = _lib_selector_state
4172
+
4173
+ with _deferred_read(conn):
4174
+ state = _read_selector_state(conn)
4175
+ if state is None:
4176
+ return None
4177
+ return _ks.SelectorRows(
4178
+ state=state,
4179
+ batches=_read_selector_batch_rows(conn, batch_ids),
4180
+ batch_records=_read_selector_batch_record_rows(conn, batch_ids),
4181
+ effective=_read_selector_effective_rows(conn, event_ids),
4182
+ violations=_read_selector_violation_rows(conn, batch_ids),
4183
+ )
4184
+
4185
+
4186
+ @contextlib.contextmanager
4187
+ def _deferred_read(conn):
4188
+ """One snapshot across several SELECTs, without disturbing an open txn.
4189
+
4190
+ A caller already inside a transaction keeps it — beginning a second one
4191
+ raises — and the snapshot it holds is the one this block wanted anyway.
4192
+ """
4193
+ if conn.in_transaction:
4194
+ yield
4195
+ return
4196
+ try:
4197
+ conn.execute("BEGIN")
4198
+ except sqlite3.Error:
4199
+ yield
4200
+ return
4201
+ try:
4202
+ yield
4203
+ finally:
4204
+ try:
4205
+ conn.rollback()
4206
+ except sqlite3.Error:
4207
+ pass
4208
+
4209
+
4210
+ #: SQLite's default parameter ceiling is 999, so a scope wider than this is read
4211
+ #: in chunks rather than in one statement.
4212
+ _SCOPE_CHUNK = 500
4213
+
4214
+
4215
+ def _scoped_query(conn, sql, order_by, column, scope):
4216
+ """Run ``sql`` unscoped, or once per chunk of ``scope`` on ``column``."""
4217
+ if scope is None:
4218
+ yield from conn.execute(f"{sql} ORDER BY {order_by}")
4219
+ return
4220
+ keys = sorted(scope)
4221
+ for start in range(0, len(keys), _SCOPE_CHUNK):
4222
+ chunk = keys[start:start + _SCOPE_CHUNK]
4223
+ placeholders = ",".join("?" * len(chunk))
4224
+ yield from conn.execute(
4225
+ f"{sql} WHERE {column} IN ({placeholders}) ORDER BY {order_by}",
4226
+ chunk,
4227
+ )
4228
+
4229
+
4230
+ def _read_selector_batch_rows(conn, scope):
4231
+ _ks = _lib_selector_state
4232
+ return tuple(
4233
+ _ks.SelectorBatchRow(
4234
+ batch_id=item[0],
4235
+ status=item[1],
4236
+ action_count=None if item[2] is None else int(item[2]),
4237
+ action_set_hash=item[3],
4238
+ begin_segment=item[4],
4239
+ begin_offset=None if item[5] is None else int(item[5]),
4240
+ earliest_commit_segment=item[6],
4241
+ earliest_commit_offset=None if item[7] is None else int(item[7]),
4242
+ )
4243
+ for item in _scoped_query(
4244
+ conn,
4245
+ "SELECT batch_id, status, action_count, action_set_hash, "
4246
+ "begin_segment, begin_offset, earliest_commit_segment, "
4247
+ "earliest_commit_offset FROM journal_selector_batches",
4248
+ "batch_id", "batch_id", scope,
4249
+ )
4250
+ )
4251
+
4252
+
4253
+ def _read_selector_batch_record_rows(conn, scope):
4254
+ _ks = _lib_selector_state
4255
+ return tuple(
4256
+ _ks.SelectorBatchRecordRow(
4257
+ batch_id=item[0],
4258
+ kind=item[1],
4259
+ key=item[2],
4260
+ record_digest=item[3],
4261
+ sequence=int(item[5]),
4262
+ identity_digest=item[4],
4263
+ action_core_json=item[6],
3144
4264
  )
4265
+ for item in _scoped_query(
4266
+ conn,
4267
+ "SELECT batch_id, kind, key, record_digest, identity_digest, "
4268
+ "sequence, action_core_json FROM journal_selector_batch_records",
4269
+ "batch_id, kind, key", "batch_id", scope,
4270
+ )
4271
+ )
4272
+
4273
+
4274
+ def _read_selector_effective_rows(conn, scope):
4275
+ _ks = _lib_selector_state
4276
+ return tuple(
4277
+ _ks.SelectorEffectiveRow(
4278
+ event_id=item[0],
4279
+ rev=int(item[1]),
4280
+ status=item[2],
4281
+ content_hash=item[3],
4282
+ batch_id=item[4],
4283
+ event_json=item[5],
4284
+ winning_sequence=None if item[6] is None else int(item[6]),
4285
+ conflict_hashes_json=item[7],
4286
+ )
4287
+ for item in _scoped_query(
4288
+ conn,
4289
+ "SELECT event_id, rev, status, content_hash, batch_id, event_json, "
4290
+ "winning_sequence, conflict_hashes_json "
4291
+ "FROM journal_effective_events",
4292
+ "event_id", "event_id", scope,
4293
+ )
4294
+ )
4295
+
4296
+
4297
+ def _read_selector_violation_rows(conn, scope):
4298
+ _ks = _lib_selector_state
4299
+ return tuple(
4300
+ _ks.SelectorViolationRow(
4301
+ fingerprint=item[0],
4302
+ batch_id=item[1],
4303
+ kind=item[2],
4304
+ violation_json=item[3],
4305
+ available_after=None if item[4] is None else int(item[4]),
4306
+ )
4307
+ for item in _scoped_query(
4308
+ conn,
4309
+ "SELECT fingerprint, batch_id, kind, violation_json, "
4310
+ "available_after FROM journal_protocol_violations",
4311
+ "batch_id, kind, fingerprint", "batch_id", scope,
4312
+ )
4313
+ )
3145
4314
 
3146
4315
 
3147
4316
  def _metadata_row(conn, event_id):
@@ -3463,7 +4632,7 @@ def _validate_excluded_derived_fks(conn, spec, row) -> None:
3463
4632
  f"(re-derived {expected})")
3464
4633
 
3465
4634
 
3466
- def _full_effective_selection(hw):
4635
+ def _full_effective_selection(hw, accumulators=None):
3467
4636
  records = []
3468
4637
  evidence = []
3469
4638
  prior_high_water = None
@@ -3484,32 +4653,434 @@ def _full_effective_selection(hw):
3484
4653
  return _lib_journal.resolve_effective_events(
3485
4654
  records,
3486
4655
  protocol_prefix_evidence=evidence,
4656
+ accumulators=accumulators,
3487
4657
  )
3488
4658
 
3489
4659
 
3490
- def _correction_commit_high_water(batch_id, hw=None):
3491
- """Return the exact end offset of one completed-batch commit marker.
4660
+ def _selector_generation_matches(conn, state) -> bool:
4661
+ """Whether ``state`` names the generation this connection is reading.
3492
4662
 
3493
- The batch was already structurally validated either by the full effective
3494
- selector or by the live metadata row that names it. The earliest matching
3495
- commit is the narrowest complete prefix and remains stable even when later
3496
- journal bytes or crash-replayed duplicate markers exist.
4663
+ `stats_publication_stamp` carries no counter, so this is an IDENTITY
4664
+ comparison rather than an ordering one, and the stamp is re-read on a FRESH
4665
+ read-only connection: `conn` may be sitting on a superseded generation
4666
+ precisely because in-place publication keeps an open reader alive on the one
4667
+ it started with, which is the case this check exists to catch.
3497
4668
 
3498
- Streams rather than materializing (#496 S4): the previous form built the
3499
- whole prefix through `_read_range` before its first-match return, so a
3500
- marker in the first segment still paid for every later one.
4669
+ Absent, duplicated, unreadable or mismatched all answer False, which is the
4670
+ fall-back-to-full-selection direction (spec §3.4). A stamp read from a
4671
+ destination at any other epoch also answers False, for the same reason
4672
+ `read_publication_stamp` returns None there.
3501
4673
  """
3502
- if not batch_id:
3503
- return None
3504
- if hw is None:
3505
- hw = journal_high_water()
3506
- if hw is None:
3507
- return None
3508
- for segment, offset, raw in iter_range(None, hw):
3509
- record = _lib_journal.decode_line(raw)
3510
- if (
3511
- record is not None
3512
- and record.get("t") == "correction_batch"
4674
+ if state.generation_record_path is None or (
4675
+ state.generation_stamped_at_utc is None
4676
+ ):
4677
+ return False
4678
+ try:
4679
+ row = conn.execute(
4680
+ "SELECT file FROM pragma_database_list WHERE name = 'main'"
4681
+ ).fetchone()
4682
+ except sqlite3.Error:
4683
+ return False
4684
+ path = None if row is None else row[0]
4685
+ if not path:
4686
+ return False
4687
+ try:
4688
+ probe = sqlite3.connect(f"file:{path}?mode=ro", uri=True)
4689
+ try:
4690
+ epoch = int(probe.execute("PRAGMA user_version").fetchone()[0])
4691
+ if epoch != _cctally_core.STATS_INDEX_EPOCH:
4692
+ return False
4693
+ rows = probe.execute(
4694
+ "SELECT record_path, stamped_at_utc FROM stats_publication_stamp"
4695
+ ).fetchall()
4696
+ finally:
4697
+ probe.close()
4698
+ except BaseException:
4699
+ return False
4700
+ if len(rows) != 1:
4701
+ return False
4702
+ return (str(rows[0][0]), str(rows[0][1])) == (
4703
+ state.generation_record_path,
4704
+ state.generation_stamped_at_utc,
4705
+ )
4706
+
4707
+
4708
+ def _cutover_for_selection(seen: bool, account_key, hw):
4709
+ """The cutover account a selection over this prefix must normalize with.
4710
+
4711
+ When durable state SAW the op, the recorded value is exact and no scan
4712
+ happens — which is what removes `resolve_cutover_claude_account()`'s
4713
+ whole-journal traversal from the live path (spec §3.6). When it did not, the
4714
+ answer is `_resolve_cutover_for_rebuild`'s, so the live path and a rebuild
4715
+ pinned at the same high-water cannot disagree; on the live path the pinned
4716
+ high-water IS the current one, so that call returns immediately without
4717
+ opening a segment.
4718
+ """
4719
+ if seen:
4720
+ return (
4721
+ account_key if account_key is not None
4722
+ else _lib_accounts.UNATTRIBUTED
4723
+ )
4724
+ return _resolve_cutover_for_rebuild(_CUTOVER_UNSEEN, hw, list_segments())
4725
+
4726
+
4727
+ def _causal_offset_of(sequence, coordinates):
4728
+ """The journal coordinate of the record at ``sequence``, or None.
4729
+
4730
+ A separate function so the fail-closed path has something to exercise. The
4731
+ coordinate is MANDATORY: substituting the pinned high-water is unsafe,
4732
+ because it is `st_size`, `_iter_segment_lines` omits an incomplete trailing
4733
+ line, and `_repair_torn_tail` can truncate below it — so a cursor written
4734
+ there can sit beyond unread data (spec §3.7).
4735
+ """
4736
+ return coordinates.get(sequence)
4737
+
4738
+
4739
+ def _prefix_position(coordinate):
4740
+ """A comparable position for a journal prefix; None is the very start.
4741
+
4742
+ Ordered through `segment_sort_key`, because bootstrap segments sort before
4743
+ observation segments and a plain string comparison would get the order
4744
+ wrong across that boundary.
4745
+ """
4746
+ if not coordinate or coordinate[0] is None or coordinate[1] is None:
4747
+ return None
4748
+ return (_lib_journal.segment_sort_key(coordinate[0]), int(coordinate[1]))
4749
+
4750
+
4751
+ def _coordinate_covers(covered, target) -> bool:
4752
+ """Whether ``covered`` reaches at least as far as ``target``."""
4753
+ left = _prefix_position(covered)
4754
+ right = _prefix_position(target)
4755
+ if left is None or right is None:
4756
+ return False
4757
+ return left >= right
4758
+
4759
+
4760
+ #: The most journal bytes one live tick may re-fold to realign a durable
4761
+ #: selector prefix that fell behind the caller's cursor.
4762
+ #:
4763
+ #: The gap is normally one tick wide and closes on that tick, because the common
4764
+ #: transient — a `database is locked` swallowed by `_selector_generation_matches`
4765
+ #: — is decided BEFORE the gap read and the next successful tick writes the
4766
+ #: realigned prefix. One refusal is not the shape this cap exists for.
4767
+ #:
4768
+ #: The shape it exists for is a refusal that repeats. `merge_delta`'s
4769
+ #: durably-completed-batch shape refusal falls back to
4770
+ #: `_full_effective_selection`, whose loop acts only on entries whose `batch_id`
4771
+ #: is not `None` — and after a taint the winner reverts to the base journal
4772
+ #: event, whose `batch_id` IS `None`. That is issue #510, so no
4773
+ #: `CorrectionRebuildRequired` is raised and no rebuild follows to realign the
4774
+ #: durable prefix. Every later tick would then re-read and re-decode a
4775
+ #: monotonically growing range, which is the whole-prefix read spec §3.3 exists
4776
+ #: to prevent under another name.
4777
+ #:
4778
+ #: Past the cap the tick degrades to full selection like every other degraded
4779
+ #: case, and `_observe_selector_desynchronization` reports the gap and this cap
4780
+ #: in the structured rebuild record, so the state is observable rather than
4781
+ #: silent. Four mebibytes is roughly 4,600 records at the maintainer's measured
4782
+ #: mean journal line length; a genuine one-tick gap is a handful.
4783
+ _GAP_REFOLD_BYTE_CAP = 4 * 1024 * 1024
4784
+
4785
+
4786
+ def _selector_gap_bytes(durable, cursor) -> "int | None":
4787
+ """Journal bytes in ``[durable, cursor)``, from `stat` alone.
4788
+
4789
+ Mirrors `_iter_range_with_segments`' bounds arithmetic without opening a
4790
+ single segment, which is what lets the cap refuse BEFORE the read it caps.
4791
+
4792
+ Returns `None` when the distance cannot be determined — an unreadable
4793
+ segment, or a cursor naming a segment the journal no longer lists. The
4794
+ caller treats that as over the cap, which is the fall-back direction.
4795
+ """
4796
+ if cursor is None or cursor[0] is None or cursor[1] is None:
4797
+ return None
4798
+ segments = list_segments()
4799
+ if cursor[0] not in segments:
4800
+ return None
4801
+ end_idx = segments.index(cursor[0])
4802
+ if (
4803
+ durable is None
4804
+ or durable[0] is None
4805
+ or durable[1] is None
4806
+ or durable[0] not in segments
4807
+ ):
4808
+ start_idx, start_off = 0, 0
4809
+ else:
4810
+ start_idx, start_off = segments.index(durable[0]), int(durable[1])
4811
+ total = 0
4812
+ for idx in range(start_idx, end_idx + 1):
4813
+ lo = start_off if idx == start_idx else 0
4814
+ if idx == end_idx:
4815
+ hi = int(cursor[1])
4816
+ else:
4817
+ try:
4818
+ hi = os.path.getsize(_cctally_core.JOURNAL_DIR / segments[idx])
4819
+ except OSError:
4820
+ return None
4821
+ if hi > lo:
4822
+ total += hi - lo
4823
+ return total
4824
+
4825
+
4826
+ def _selector_gap_entries(durable, cursor):
4827
+ """Decoded records in ``[durable, cursor)`` with each one's end offset.
4828
+
4829
+ This is non-empty only after a tick that could not validate the durable
4830
+ generation. `_write_cursor` runs whether or not a selector delta was
4831
+ written, so such a tick advances `journal_cursor` and leaves
4832
+ `journal_selector_state` where the last rebuild put it — and nothing on the
4833
+ live path ever rewrites that table, so an EQUALITY comparison between the
4834
+ two could never hold again and F20 stayed off until a full rebuild.
4835
+
4836
+ Re-folding the gap is what brings them back together. It is correct because
4837
+ the selector fold is independent of application, so folding records the
4838
+ cycle already applied changes only the selector's own accumulators, and the
4839
+ read is bounded by the gap rather than by the whole prefix.
4840
+ """
4841
+ entries = []
4842
+ start = None if _prefix_position(durable) is None else (
4843
+ durable[0], int(durable[1]))
4844
+ for segment, offset, raw in iter_range(start, cursor):
4845
+ record = _lib_journal.decode_line(raw)
4846
+ if record is None:
4847
+ # A malformed line consumes no sequence number, exactly as the
4848
+ # cycle's own read loop and `decoded_entry_count` treat it.
4849
+ continue
4850
+ entries.append((record, segment, offset + len(raw) + 1))
4851
+ return entries
4852
+
4853
+
4854
+ def _normalized_for_selection(records, cutover_claude):
4855
+ """Copies of ``records`` with the legacy account stamp applied.
4856
+
4857
+ COPIES, because `_normalize_legacy_account_stamp` mutates in place and these
4858
+ same dicts are what the cycle's pipeline folds; injecting a stamp into them
4859
+ would change what step 4b sees. The payload is copied too, since the evt/op
4860
+ branch writes into it.
4861
+ """
4862
+ copies = []
4863
+ for record in records:
4864
+ if not isinstance(record, dict):
4865
+ copies.append(record)
4866
+ continue
4867
+ copy = dict(record)
4868
+ payload = record.get("payload")
4869
+ if isinstance(payload, dict):
4870
+ copy["payload"] = dict(payload)
4871
+ _normalize_legacy_account_stamp(copy, cutover_claude)
4872
+ copies.append(copy)
4873
+ return copies
4874
+
4875
+
4876
+ def _incremental_selection(conn, records, entries, cursor, covered):
4877
+ """Continue the durable fold over one cycle's delta, or return None.
4878
+
4879
+ ``entries`` is positionally parallel to ``records``: each element is the
4880
+ ``(segment, end offset)`` of the record that consumed one sequence number.
4881
+ The map from ABSOLUTE sequence to coordinate is built HERE rather than by the
4882
+ caller, because a durable prefix behind the cursor prepends its unfolded gap
4883
+ and shifts every delta record's number.
4884
+
4885
+ None means every degraded case: absent or unreadable selector state, a
4886
+ `selector_version` mismatch, a durable prefix AHEAD of the caller's cursor, a
4887
+ gap wider than `_GAP_REFOLD_BYTE_CAP`, a generation identity that does not
4888
+ match a freshly read publication stamp, a generation that moved while the
4889
+ gap was being read, a cutover operation the durable prefix has not folded,
4890
+ and anything the pure kernel refuses. Every one of them takes the existing
4891
+ full-prefix path unchanged.
4892
+
4893
+ Read order is load-bearing. The three cheap checks are decided from ONE row
4894
+ and run BEFORE any row group is materialized, a delta the fold does not
4895
+ consume reads no group at all, and a delta that does is scoped to the
4896
+ batches and event ids it names. This function is on `cmd_record_usage`'s
4897
+ path, which runs on every Claude Code status-line tick, and an unscoped read
4898
+ materializes 34,644 effective rows and 64,248 batch-record rows on a
4899
+ production journal.
4900
+
4901
+ There are TWO read snapshots rather than one, and the gap re-fold sits
4902
+ between them deliberately: it is arbitrary journal file I/O, and holding a
4903
+ stats.db read snapshot open across it pins the WAL against checkpointing for
4904
+ the whole read, which is the bloat #297 documents. The second snapshot
4905
+ re-reads the one state row and refuses on any difference, which is what
4906
+ keeps the row groups and the state row from coming out of two generations.
4907
+
4908
+ That re-read covers in-place publication and any live delta writer, because
4909
+ both mutate the state row this connection can see. It does NOT cover
4910
+ PHYSICAL replacement: an `os.replace` between the two blocks leaves this
4911
+ connection on the unlinked old inode, whose state row and row groups are
4912
+ mutually consistent but stale. Physical replacement is excluded by the ingest
4913
+ lock the caller already holds, not by this check.
4914
+ """
4915
+ _ks = _lib_selector_state
4916
+ with _deferred_read(conn):
4917
+ state = _read_selector_state(conn)
4918
+ if state is None:
4919
+ return None
4920
+ if state.selector_version != _ks.SELECTOR_VERSION:
4921
+ return None
4922
+ durable = (state.covered_segment, state.covered_offset)
4923
+ durable_at = _prefix_position(durable)
4924
+ cursor_at = _prefix_position(cursor)
4925
+ if durable_at is not None and (
4926
+ cursor_at is None or durable_at > cursor_at
4927
+ ):
4928
+ # The durable prefix has folded records the caller has not applied,
4929
+ # so numbering the delta from `next_sequence` would leave a hole the
4930
+ # size of the overlap. The other direction is recoverable and is
4931
+ # recovered below.
4932
+ return None
4933
+ if not _selector_generation_matches(conn, state):
4934
+ return None
4935
+
4936
+ if durable_at == cursor_at:
4937
+ gap = ()
4938
+ else:
4939
+ gap_bytes = _selector_gap_bytes(durable, cursor)
4940
+ if gap_bytes is None or gap_bytes > _GAP_REFOLD_BYTE_CAP:
4941
+ # Bounded rather than unbounded: a refusal that repeats leaves the
4942
+ # durable prefix behind forever, and an uncapped re-fold would grow
4943
+ # the per-tick read without limit. See `_GAP_REFOLD_BYTE_CAP`.
4944
+ return None
4945
+ gap = _selector_gap_entries(durable, cursor)
4946
+
4947
+ stream = [item[0] for item in gap] + list(records)
4948
+ coordinates = {}
4949
+ for index, item in enumerate(gap):
4950
+ coordinates[state.next_sequence + index] = (item[1], item[2])
4951
+ for index, coordinate in enumerate(entries or ()):
4952
+ coordinates[state.next_sequence + len(gap) + index] = coordinate
4953
+
4954
+ for record in stream:
4955
+ if isinstance(record, dict) and record.get("id") == CUTOVER_OP_ID:
4956
+ if not state.cutover_seen:
4957
+ # A cutover the durable prefix has not folded re-normalizes
4958
+ # every legacy Claude line in that prefix, which changes those
4959
+ # events' `content_hash` and `event_json`. The durable winners
4960
+ # were folded WITHOUT it, and this path normalizes only the
4961
+ # delta, so carrying them forward would diverge from a full
4962
+ # pass. Falling back is provably equivalent to one.
4963
+ return None
4964
+ break
4965
+
4966
+ if not any(
4967
+ isinstance(record, dict)
4968
+ and (
4969
+ record.get("t") in _ks.FOLD_RECORD_TYPES
4970
+ or (
4971
+ record.get("t") == "op"
4972
+ and isinstance(record.get("payload"), dict)
4973
+ and record["payload"].get("kind")
4974
+ == _lib_journal._PROTOCOL_RESOLUTION_KIND
4975
+ )
4976
+ )
4977
+ for record in stream
4978
+ ):
4979
+ # Nothing here changes the fold, so only the counters move — and the
4980
+ # four row groups are not read at all. `advance_counter` returns them
4981
+ # by reference, so an empty placeholder makes the delta writer skip
4982
+ # every group by identity, exactly as a full read would have.
4983
+ placeholder = _ks.SelectorRows(state=state)
4984
+ return {
4985
+ "before": placeholder,
4986
+ "after": _ks.advance_counter(
4987
+ placeholder,
4988
+ consumed=state.next_sequence + len(stream),
4989
+ covered=covered,
4990
+ ),
4991
+ "transitions": [],
4992
+ "coordinates": coordinates,
4993
+ }
4994
+
4995
+ batch_ids = _ks.delta_batch_scope(stream)
4996
+ with _deferred_read(conn):
4997
+ if _read_selector_state(conn) != state:
4998
+ return None
4999
+ batches = _read_selector_batch_rows(conn, batch_ids)
5000
+ batch_records = _read_selector_batch_record_rows(conn, batch_ids)
5001
+ effective = _read_selector_effective_rows(
5002
+ conn, _ks.delta_event_scope(stream, batch_records))
5003
+ violations = _read_selector_violation_rows(conn, batch_ids)
5004
+ # No coordinate widening beyond `batch_ids`, and the reason is a
5005
+ # reachability argument rather than an economy. `_preflight_live_events`
5006
+ # consults a batch row only for a winner whose four-tuple DIFFERS from
5007
+ # `journal_effective_events`. `_read_selector_effective_rows` reads that
5008
+ # same table, so a winner the merge passed through compares equal and is
5009
+ # skipped; and a winner the merge re-decided took a correction candidate
5010
+ # from a batch the delta named, so its batch row is already in `batches`.
5011
+ # A read over `{winner batches} - batch_ids` therefore returned rows the
5012
+ # caller could not reach, on every tick that named any correction batch.
5013
+ rows = _ks.SelectorRows(
5014
+ state=state,
5015
+ batches=batches,
5016
+ batch_records=batch_records,
5017
+ effective=effective,
5018
+ violations=violations,
5019
+ )
5020
+
5021
+ cutover_claude = _cutover_for_selection(
5022
+ state.cutover_seen, state.cutover_account_key, covered)
5023
+
5024
+ try:
5025
+ merged, transitions = _ks.merge_delta(
5026
+ rows,
5027
+ _normalized_for_selection(stream, cutover_claude),
5028
+ next_sequence=state.next_sequence,
5029
+ coordinates=coordinates,
5030
+ covered=covered,
5031
+ )
5032
+ except _ks.IncrementalSelectionUnavailable:
5033
+ return None
5034
+ return {
5035
+ "before": rows,
5036
+ "after": merged,
5037
+ "transitions": transitions,
5038
+ "coordinates": coordinates,
5039
+ }
5040
+
5041
+
5042
+ def _raise_taint_transition(transition, coordinates) -> None:
5043
+ """Turn one completed-to-tainted transition into its recovery signal."""
5044
+ coordinate = _causal_offset_of(transition.causal_sequence, coordinates)
5045
+ if coordinate is None:
5046
+ raise JournalError(
5047
+ f"correction batch {transition.batch_id} moved completed -> "
5048
+ "tainted but its causal offset could not be resolved; refusing to "
5049
+ "substitute the pinned high-water"
5050
+ )
5051
+ raise CorrectionRebuildRequired(
5052
+ f"completed correction batch {transition.batch_id} was tainted by a "
5053
+ "later record and requires a stats index rebuild through it",
5054
+ batch_id=transition.batch_id,
5055
+ high_water=coordinate,
5056
+ recovery_eligible=True,
5057
+ kind=CORRECTION_KIND_COMPLETED_TO_TAINTED,
5058
+ )
5059
+
5060
+
5061
+ def _correction_commit_high_water(batch_id, hw=None):
5062
+ """Return the exact end offset of one completed-batch commit marker.
5063
+
5064
+ The batch was already structurally validated either by the full effective
5065
+ selector or by the live metadata row that names it. The earliest matching
5066
+ commit is the narrowest complete prefix and remains stable even when later
5067
+ journal bytes or crash-replayed duplicate markers exist.
5068
+
5069
+ Streams rather than materializing (#496 S4): the previous form built the
5070
+ whole prefix through `_read_range` before its first-match return, so a
5071
+ marker in the first segment still paid for every later one.
5072
+ """
5073
+ if not batch_id:
5074
+ return None
5075
+ if hw is None:
5076
+ hw = journal_high_water()
5077
+ if hw is None:
5078
+ return None
5079
+ for segment, offset, raw in iter_range(None, hw):
5080
+ record = _lib_journal.decode_line(raw)
5081
+ if (
5082
+ record is not None
5083
+ and record.get("t") == "correction_batch"
3513
5084
  and record.get("phase") == "commit"
3514
5085
  and record.get("id") == batch_id
3515
5086
  ):
@@ -3517,8 +5088,22 @@ def _correction_commit_high_water(batch_id, hw=None):
3517
5088
  return None
3518
5089
 
3519
5090
 
5091
+ def _has_correction_records(records) -> bool:
5092
+ return any(
5093
+ record.get("t") in {"correction", "correction_batch"}
5094
+ or (
5095
+ record.get("t") == "op"
5096
+ and isinstance(record.get("payload"), dict)
5097
+ and record["payload"].get("kind")
5098
+ == _lib_journal._PROTOCOL_RESOLUTION_KIND
5099
+ )
5100
+ for record in records
5101
+ )
5102
+
5103
+
3520
5104
  def _preflight_live_events(
3521
- conn, records, hw, conflicts=None, protocol_scan=None
5105
+ conn, records, hw, conflicts=None, protocol_scan=None,
5106
+ selector=None, cursor=None, entries=None,
3522
5107
  ):
3523
5108
  """Validate unread evt/correction records before the stats transaction.
3524
5109
 
@@ -3527,7 +5112,22 @@ def _preflight_live_events(
3527
5112
  every cycle over an already-poisoned journal, exactly like the rebuild. The
3528
5113
  divergent evt is DROPPED from the apply set, the prior effective event
3529
5114
  stands, and the group is appended to `conflicts` when a sink is supplied.
3530
- `CorrectionRebuildRequired` stays fatal."""
5115
+ `CorrectionRebuildRequired` stays fatal.
5116
+
5117
+ `selector` is an out-dict (#496 S5b §3.3). When supplied and the durable
5118
+ generation validates, it receives the merged selector rows so the caller can
5119
+ advance them inside its own transaction — the delta and the cursor it
5120
+ describes then commit together. When the generation does not validate the
5121
+ dict is left empty and durable state stays where the last rebuild put it,
5122
+ which is the conservative direction: `stats_index_matches_journal_prefix`
5123
+ then answers False and its one caller rebuilds.
5124
+
5125
+ `cursor` is the caller's applied journal cursor. The durable covered prefix
5126
+ must not be ahead of it; a prefix BEHIND it is re-folded from the journal
5127
+ over exactly that gap. `entries` is positionally parallel to `records` and
5128
+ carries each record's `(segment, end offset)`, from which the incremental
5129
+ step derives the absolute-sequence coordinate map.
5130
+ """
3531
5131
  event_records = [record for record in records if record.get("t") == "evt"]
3532
5132
  selected_new = _lib_journal.resolve_effective_events(event_records)
3533
5133
  if conflicts is not None:
@@ -3577,51 +5177,117 @@ def _preflight_live_events(
3577
5177
  ),
3578
5178
  )
3579
5179
 
3580
- if any(
3581
- record.get("t") in {"correction", "correction_batch"}
3582
- or (
3583
- record.get("t") == "op"
3584
- and isinstance(record.get("payload"), dict)
3585
- and record["payload"].get("kind")
3586
- == _lib_journal._PROTOCOL_RESOLUTION_KIND
3587
- )
3588
- for record in records
3589
- ):
3590
- full = _full_effective_selection(hw)
3591
- if protocol_scan is not None:
3592
- protocol_scan["scanned"] = True
3593
- protocol_scan["violations"] = full.protocol_violations
3594
- protocol_scan["acknowledged"] = (
3595
- full.acknowledged_protocol_violations
3596
- )
3597
- for selected in full.by_id.values():
3598
- if selected.batch_id is None:
5180
+ has_corrections = _has_correction_records(records)
5181
+ incremental = None
5182
+ if selector is not None or has_corrections:
5183
+ incremental = _incremental_selection(conn, records, entries, cursor, hw)
5184
+ if selector is not None and incremental is not None:
5185
+ selector["before"] = incremental["before"]
5186
+ selector["after"] = incremental["after"]
5187
+
5188
+ if incremental is not None:
5189
+ # The incremental path already knows every batch's verdict, so the
5190
+ # whole-prefix read is not performed at all. `protocol_scan` stays unset
5191
+ # deliberately: the merged violation rows are advanced by the selector
5192
+ # delta the caller writes, and running `_write_protocol_violations` on
5193
+ # top of that would replace them with a set derived from a scan that did
5194
+ # not happen.
5195
+ #
5196
+ # This runs even when the DELTA carries no correction record, because a
5197
+ # re-folded gap can: the merged rows are about to be written into
5198
+ # `journal_effective_events`, and installing a corrected winner without
5199
+ # the rebuild the correction demands is exactly what the loop refuses.
5200
+ # Both loops are bounded by the delta's own scope, so the added work on
5201
+ # an ordinary tick is zero rows.
5202
+ for transition in incremental["transitions"]:
5203
+ _raise_taint_transition(transition, incremental["coordinates"])
5204
+ batches = {row.batch_id: row for row in incremental["after"].batches}
5205
+ for row in incremental["after"].effective:
5206
+ if row.batch_id is None:
3599
5207
  continue
3600
- prior = _metadata_row(conn, selected.event_id)
3601
- if prior is not None:
3602
- prior_tuple = (int(prior[0]), prior[1], prior[2], prior[3])
3603
- selected_tuple = (
3604
- selected.rev,
3605
- selected.status,
3606
- selected.content_hash,
3607
- selected.batch_id,
5208
+ prior = _metadata_row(conn, row.event_id)
5209
+ if prior is not None and (
5210
+ int(prior[0]), prior[1], prior[2], prior[3]
5211
+ ) == (row.rev, row.status, row.content_hash, row.batch_id):
5212
+ continue
5213
+ batch = batches.get(row.batch_id)
5214
+ commit = None
5215
+ if batch is not None and batch.earliest_commit_segment is not None:
5216
+ # The durable coordinate, which is what removes
5217
+ # `_correction_commit_high_water`'s traversal from this path.
5218
+ commit = (
5219
+ batch.earliest_commit_segment,
5220
+ int(batch.earliest_commit_offset),
3608
5221
  )
3609
- if prior_tuple == selected_tuple:
3610
- continue
5222
+ if commit is None:
5223
+ # A whole-prefix stream, and a correctness fallback rather than
5224
+ # a fast path: acceptance criteria 6 and 17 ("no whole-journal
5225
+ # read on the interactive ingest path") do NOT hold here.
5226
+ #
5227
+ # It is reached only when no durable batch row carries an
5228
+ # `earliest_commit_*` for this winner, which `marker_coordinates`
5229
+ # and `_carry_coordinates` normally prevent. It is NOT reached
5230
+ # for a winner the delta merely passed through: `_metadata_row`
5231
+ # and `_read_selector_effective_rows` read the same table, so a
5232
+ # passed-through winner compares equal to its stored metadata
5233
+ # above and is skipped before ever arriving here. A winner the
5234
+ # delta RE-DECIDED names a batch the delta itself carried, whose
5235
+ # row this pass is about to write.
5236
+ commit = _correction_commit_high_water(row.batch_id, hw)
3611
5237
  raise CorrectionRebuildRequired(
3612
- f"completed correction batch {selected.batch_id} requires "
5238
+ f"completed correction batch {row.batch_id} requires "
3613
5239
  "stats index rebuild",
3614
- batch_id=selected.batch_id,
3615
- event_id=selected.event_id,
3616
- high_water=_correction_commit_high_water(selected.batch_id, hw),
5240
+ batch_id=row.batch_id,
5241
+ event_id=row.event_id,
5242
+ high_water=commit,
3617
5243
  expected_metadata=(
3618
- selected.rev,
3619
- selected.status,
3620
- selected.content_hash,
3621
- selected.batch_id,
3622
- ),
5244
+ row.rev, row.status, row.content_hash, row.batch_id),
3623
5245
  recovery_eligible=True,
5246
+ kind=CORRECTION_KIND_NEWLY_COMPLETED,
5247
+ )
5248
+ return to_apply
5249
+
5250
+ if not has_corrections:
5251
+ return to_apply
5252
+
5253
+ accumulators: dict = {}
5254
+ full = _full_effective_selection(hw, accumulators)
5255
+ if protocol_scan is not None:
5256
+ protocol_scan["scanned"] = True
5257
+ # The KERNEL's rows, not a second derivation: the rebuild writes these
5258
+ # exact rows, and a fallback that wrote a different shape left the index
5259
+ # unable to match its own journal prefix afterwards.
5260
+ protocol_scan["rows"] = _lib_selector_state.violation_rows(
5261
+ full, accumulators["fold"])
5262
+ for selected in full.by_id.values():
5263
+ if selected.batch_id is None:
5264
+ continue
5265
+ prior = _metadata_row(conn, selected.event_id)
5266
+ if prior is not None:
5267
+ prior_tuple = (int(prior[0]), prior[1], prior[2], prior[3])
5268
+ selected_tuple = (
5269
+ selected.rev,
5270
+ selected.status,
5271
+ selected.content_hash,
5272
+ selected.batch_id,
3624
5273
  )
5274
+ if prior_tuple == selected_tuple:
5275
+ continue
5276
+ raise CorrectionRebuildRequired(
5277
+ f"completed correction batch {selected.batch_id} requires "
5278
+ "stats index rebuild",
5279
+ batch_id=selected.batch_id,
5280
+ event_id=selected.event_id,
5281
+ high_water=_correction_commit_high_water(selected.batch_id, hw),
5282
+ expected_metadata=(
5283
+ selected.rev,
5284
+ selected.status,
5285
+ selected.content_hash,
5286
+ selected.batch_id,
5287
+ ),
5288
+ recovery_eligible=True,
5289
+ kind=CORRECTION_KIND_NEWLY_COMPLETED,
5290
+ )
3625
5291
  return to_apply
3626
5292
 
3627
5293
 
@@ -3645,7 +5311,14 @@ def _run_cycle(conn: sqlite3.Connection, *, reconcile_config=None,
3645
5311
  # advance — any harvested budget evt lands in the freshly-created first
3646
5312
  # segment past the (absent) HW and replays idempotently on the next cycle.
3647
5313
  decoded: list = [] # (record, segment, offset)
5314
+ # End offset per decoded entry, positionally parallel to `decoded`. Kept
5315
+ # separate rather than widened into those tuples because `QUOTA_APPLIER`
5316
+ # consumes them by shape (#496 S5b): the selector needs each record's end
5317
+ # coordinate, and re-encoding a record to recover its length would only be
5318
+ # right for lines this binary wrote.
5319
+ end_offsets: list = []
3648
5320
  malformed = 0
5321
+ cursor = None
3649
5322
  cursor_target = None
3650
5323
  if hw is None:
3651
5324
  if reconcile_config is None and codex_apply is None:
@@ -3664,6 +5337,7 @@ def _run_cycle(conn: sqlite3.Connection, *, reconcile_config=None,
3664
5337
  malformed += 1
3665
5338
  continue
3666
5339
  decoded.append((rec, seg, off))
5340
+ end_offsets.append(off + len(raw) + 1)
3667
5341
 
3668
5342
  # Step 3: cache leg (Codex quota) BEFORE the stats txn (lock-order law).
3669
5343
  # QUOTA_APPLIER attempts the global-then-Codex cache flock NB upsert; on
@@ -3673,11 +5347,23 @@ def _run_cycle(conn: sqlite3.Connection, *, reconcile_config=None,
3673
5347
  # scalar cursor sound).
3674
5348
  cursor_target = (hw_seg, hw_size)
3675
5349
  if QUOTA_APPLIER is not None:
3676
- stop = QUOTA_APPLIER(decoded)
5350
+ # The cycle's own range, so the leg can advance the #496 S5b
5351
+ # coverage certificate over a batch it can prove contiguous. A
5352
+ # prefix-stop advances nothing, because the leg then committed
5353
+ # neither family and `cursor_target` moves to the stop coordinate.
5354
+ # `decoded_end` is the last DECODED record's end coordinate, which
5355
+ # bounds the covered claim below the raw cursor target whenever the
5356
+ # traversal stopped short of it — a torn or malformed trailing line.
5357
+ stop = QUOTA_APPLIER(
5358
+ decoded, cursor=cursor, covered_to=cursor_target,
5359
+ decoded_end=(
5360
+ None if not decoded
5361
+ else (decoded[-1][1], end_offsets[-1])))
3677
5362
  if stop is not None:
3678
5363
  _rec, stop_seg, stop_off = decoded[stop]
3679
5364
  cursor_target = (stop_seg, stop_off)
3680
5365
  decoded = decoded[:stop]
5366
+ end_offsets = end_offsets[:stop]
3681
5367
 
3682
5368
  records = [r for (r, _s, _o) in decoded]
3683
5369
  batch = [r for r in records if r.get("t") in ("obs", "op")]
@@ -3685,12 +5371,27 @@ def _run_cycle(conn: sqlite3.Connection, *, reconcile_config=None,
3685
5371
  # raising; the groups it drops are counted on the cycle summary.
3686
5372
  preflight_conflicts: list = []
3687
5373
  protocol_scan: dict = {}
5374
+ # #496 S5b §3.3: the delta's own sequence numbering starts where the durable
5375
+ # prefix stopped, and every decoded entry consumes one number — the same
5376
+ # numbering the rebuild produces by appending a placeholder for each valid
5377
+ # non-retained record. `_incremental_selection` owns that arithmetic: it
5378
+ # rejects a durable prefix AHEAD of `cursor`, and re-folds the gap when the
5379
+ # prefix is behind it, so an absent or stale state cannot make these numbers
5380
+ # mean something else.
5381
+ selector_state: dict = {}
5382
+ selector_entries = [
5383
+ (segment, end_offsets[index])
5384
+ for index, (_rec, segment, _off) in enumerate(decoded)
5385
+ ]
3688
5386
  journal_evts = _preflight_live_events(
3689
5387
  conn,
3690
5388
  records,
3691
5389
  cursor_target,
3692
5390
  conflicts=preflight_conflicts,
3693
5391
  protocol_scan=protocol_scan,
5392
+ selector=selector_state,
5393
+ cursor=cursor,
5394
+ entries=selector_entries,
3694
5395
  )
3695
5396
 
3696
5397
  # Step 4: ONE BEGIN IMMEDIATE — replay + pipeline + derived-fact journaling +
@@ -3757,11 +5458,18 @@ def _run_cycle(conn: sqlite3.Connection, *, reconcile_config=None,
3757
5458
  # the cursor so shallow Doctor paths observe either the old complete
3758
5459
  # result or the new complete result, never an in-between state.
3759
5460
  if protocol_scan.get("scanned"):
3760
- _write_protocol_violations(
3761
- conn,
3762
- protocol_scan.get("violations", ()),
3763
- protocol_scan.get("acknowledged", ()),
3764
- )
5461
+ _replace_protocol_violations(conn, protocol_scan.get("rows", ()))
5462
+ # 4c''. Advance durable selector state (#496 S5b §3.3), in the SAME
5463
+ # transaction as the cursor it describes, so the durable prefix equals
5464
+ # the applied journal cursor at every commit and a crash rolls both back
5465
+ # together. It runs AFTER step 4a: that step inserts the plain
5466
+ # six-column effective row for a newly applied evt, and the delta
5467
+ # replaces it with the eight-column row carrying the winning sequence.
5468
+ # Absent when the generation did not validate, which leaves durable
5469
+ # state where the last rebuild put it.
5470
+ if selector_state.get("after") is not None:
5471
+ _write_selector_delta(
5472
+ conn, selector_state["before"], selector_state["after"])
3765
5473
  # 4d. Advance the cursor (to HW, or to the cache-leg prefix boundary).
3766
5474
  # `cursor_target is None` ONLY on a reconcile-only cycle over a still-
3767
5475
  # empty journal (§5.2 above): there are no consumed lines to advance
@@ -3999,8 +5707,48 @@ def _correction_error_result(error) -> IngestResult:
3999
5707
  )
4000
5708
 
4001
5709
 
5710
+ def _tainted_batch_converged(signal: CorrectionRebuildRequired) -> bool:
5711
+ """Convergence for a completed-to-tainted signal (#496 S5b §3.7).
5712
+
5713
+ `_recover_completed_correction`'s existing predicate needs an exact
5714
+ `(rev, status, content_hash, batch_id)`, and after a taint withdraws a
5715
+ completed batch the post-rebuild winner may be an OLDER candidate that
5716
+ durable selector state deliberately does not store — §3.2 keeps no losing
5717
+ candidates. So this kind converges on state that is available: the selector
5718
+ batch is tainted, selector coverage includes the causal offset, and no
5719
+ effective winner names that batch.
5720
+ """
5721
+ if signal.batch_id is None or signal.high_water is None:
5722
+ return False
5723
+ path = pathlib.Path(_cctally_core.DB_PATH)
5724
+ if not path.exists():
5725
+ return False
5726
+ conn = sqlite3.connect(f"file:{path}?mode=ro", uri=True, timeout=5.0)
5727
+ try:
5728
+ rows = _read_selector_rows(conn)
5729
+ except sqlite3.Error:
5730
+ return False
5731
+ finally:
5732
+ conn.close()
5733
+ if rows is None:
5734
+ return False
5735
+ batch = {row.batch_id: row for row in rows.batches}.get(signal.batch_id)
5736
+ if batch is None or batch.status != "tainted":
5737
+ return False
5738
+ if not _coordinate_covers(
5739
+ (rows.state.covered_segment, rows.state.covered_offset),
5740
+ signal.high_water,
5741
+ ):
5742
+ return False
5743
+ return not any(
5744
+ row.batch_id == signal.batch_id for row in rows.effective
5745
+ )
5746
+
5747
+
4002
5748
  def _correction_index_converged(signal: CorrectionRebuildRequired) -> bool:
4003
5749
  """Revalidate the triggering effective row without open-time mutation."""
5750
+ if signal.kind == CORRECTION_KIND_COMPLETED_TO_TAINTED:
5751
+ return _tainted_batch_converged(signal)
4004
5752
  if signal.event_id is None or signal.expected_metadata is None:
4005
5753
  return False
4006
5754
  path = pathlib.Path(_cctally_core.DB_PATH)
@@ -4050,7 +5798,20 @@ def _recover_completed_correction(
4050
5798
  timeout_s: float,
4051
5799
  ) -> None:
4052
5800
  """Revalidate and, when still needed, replace through the trigger prefix."""
4053
- if (
5801
+ if signal.kind == CORRECTION_KIND_COMPLETED_TO_TAINTED:
5802
+ # A different contract, deliberately: this kind carries no `event_id`
5803
+ # and no expected metadata tuple, because the post-taint winner may be a
5804
+ # candidate the durable state does not store. The causal offset is
5805
+ # mandatory and there is nothing to substitute for it, so an absent one
5806
+ # is refused rather than widened to the pinned high-water.
5807
+ if signal.batch_id is None or signal.high_water is None:
5808
+ raise CorrectionRecoveryError(
5809
+ _correction_recovery_guidance(
5810
+ "completed-to-tainted correction lacks the causal record "
5811
+ "offset its rebuild must include"
5812
+ )
5813
+ )
5814
+ elif (
4054
5815
  signal.batch_id is None
4055
5816
  or signal.event_id is None
4056
5817
  or signal.high_water is None
@@ -4167,6 +5928,7 @@ def run_stats_ingest(
4167
5928
  high_water=signal.high_water,
4168
5929
  expected_metadata=signal.expected_metadata,
4169
5930
  recovery_eligible=True,
5931
+ kind=signal.kind,
4170
5932
  ) from signal
4171
5933
 
4172
5934
  try:
@@ -4315,7 +6077,12 @@ class RebuildResult:
4315
6077
  rows_by_table: dict # journal-covered table -> row count in the rebuild
4316
6078
  malformed: int # journal lines that failed to decode (spec §4.4)
4317
6079
  duration_s: float # wall time of the whole rebuild
4318
- segments_read: int # journal segments folded
6080
+ # Segments in the pinned prefix, which is NOT the same as segments this
6081
+ # pass opened: #496 S5b's elision skips some of them and contributes their
6082
+ # stored summary instead. `traversal["elision"]["scannedSegments"]` is the
6083
+ # opened count. The value here is unchanged from before elision existed, so
6084
+ # the public `segmentsRead` key keeps meaning what it always meant.
6085
+ segments_read: int # journal segments in the pinned prefix
4319
6086
  lines_folded: int # op + evt lines applied (obs are rederive input)
4320
6087
  # #374: divergent same-revision groups quarantined behind a lowest-sequence
4321
6088
  # provisional winner. The rebuild COMPLETES and exits 0 — reporting them is
@@ -4348,6 +6115,32 @@ class RebuildResult:
4348
6115
  peak_heap_bytes: int = 0
4349
6116
  #: cache writer flock acquisition to release, in seconds.
4350
6117
  quota_lock_hold_seconds: float = 0.0
6118
+ #: #496 S5b — the durable selector prefix of the index this rebuild is about
6119
+ #: to REPLACE, when that prefix was behind the index's own applied journal
6120
+ #: cursor. `None` when the two agreed, when the destination did not exist, or
6121
+ #: when either could not be read. A tick that cannot validate the durable
6122
+ #: generation advances the cursor without advancing the selector, and the
6123
+ #: next tick re-folds the gap silently (§6.3); this is the only surface on
6124
+ #: which a persistent desynchronization is reported.
6125
+ selector_desynchronized: "dict | None" = None
6126
+ #: #496 S5b F11 — what the quota cache leg did: `status` in
6127
+ #: `{skipped, covered, recovered, failed}`, `reason` naming the coverage
6128
+ #: verdict, `coveredHighWater` and `replayedObservations`. `covered` is the
6129
+ #: intact path, where the leg takes no cache writer flock and replays
6130
+ #: nothing. Additive to the `schemaVersion: 1` rebuild record.
6131
+ quota_cache_coverage: dict = field(default_factory=dict)
6132
+ #: #496 S5b §4.7 — TRUE when this generation's stats quota projection was
6133
+ #: materialized from a cache whose recovery left an uncovered remainder.
6134
+ #:
6135
+ #: It is a separate field from `quota_cache_coverage` because the two answer
6136
+ #: different questions and a consumer must not read one as the other:
6137
+ #: coverage describes the CACHE, this describes the PUBLISHED INDEX. The
6138
+ #: quota projection is materialized FROM `cache.db`, so a partial cache
6139
+ #: produces a semantically partial projection inside the generation being
6140
+ #: published, and completing cache recovery later does not by itself
6141
+ #: reconcile that projection. `RebuildResult` still has no success or
6142
+ #: failure boolean, and this is not one — publication proceeds either way.
6143
+ stats_quota_projection_incomplete: bool = False
4351
6144
 
4352
6145
 
4353
6146
  def _remove_db_sidecars_strict(path) -> None:
@@ -4392,6 +6185,9 @@ _REBUILD_REQUIRED_TABLES = frozenset(
4392
6185
  "journal_cursor",
4393
6186
  "journal_effective_events",
4394
6187
  "journal_protocol_violations",
6188
+ "journal_selector_batch_records",
6189
+ "journal_selector_batches",
6190
+ "journal_selector_state",
4395
6191
  "percent_milestones",
4396
6192
  "project_budget_milestones",
4397
6193
  "projected_milestones",
@@ -4405,6 +6201,7 @@ _REBUILD_REQUIRED_TABLES = frozenset(
4405
6201
  "schema_migrations_skipped",
4406
6202
  "stats_open_fixups",
4407
6203
  "stats_publication_stamp",
6204
+ "stats_quota_projection_state",
4408
6205
  "week_reset_events",
4409
6206
  "weekly_cost_snapshots",
4410
6207
  "weekly_credit_floors",
@@ -4429,6 +6226,7 @@ _REBUILD_REQUIRED_INDEXES = frozenset(
4429
6226
  "idx_five_hour_milestones_journal_id_null",
4430
6227
  "idx_five_hour_reset_events_journal_id",
4431
6228
  "idx_five_hour_reset_events_journal_id_null",
6229
+ "idx_journal_protocol_violations_batch",
4432
6230
  "idx_percent_milestones_journal_id",
4433
6231
  "idx_percent_milestones_journal_id_null",
4434
6232
  "idx_project_budget_milestones_journal_id",
@@ -4453,7 +6251,7 @@ _REBUILD_REQUIRED_INDEXES = frozenset(
4453
6251
  # omitted column, constraint, partial predicate, or index definition. An epoch
4454
6252
  # schema change must update this contract alongside STATS_INDEX_EPOCH.
4455
6253
  _REBUILD_SCHEMA_FINGERPRINT = (
4456
- "7dde5a7995f441558d08b0204136824d6ff7208b221e576c79a76854b76aa178"
6254
+ "2b378fc3be1c7bb249bb0c3ddd2111a802f689cf30b3fd42806a116611c799e6"
4457
6255
  )
4458
6256
 
4459
6257
 
@@ -4554,6 +6352,31 @@ def _validate_rebuilt_stats_index(
4554
6352
  )
4555
6353
 
4556
6354
 
6355
+ def _validate_selector_state(conn, expected) -> None:
6356
+ """Refuse an index whose durable selector state is not what the journal says.
6357
+
6358
+ ``expected`` is the kernel rows a full selection over the same pinned
6359
+ traversal produced. Raises `JournalError`; `stats_index_matches_journal_
6360
+ prefix` catches that and answers False, while the rebuild lets it abort the
6361
+ publication.
6362
+
6363
+ Single-row cardinality is part of the contract, so a zero-row state fails
6364
+ here rather than reading as an empty-but-valid generation.
6365
+ """
6366
+ stored = _read_selector_rows(conn)
6367
+ if stored is None:
6368
+ raise JournalError(
6369
+ "durable selector state is absent or not a single row"
6370
+ )
6371
+ if _lib_selector_state.comparable(stored) != _lib_selector_state.comparable(
6372
+ expected
6373
+ ):
6374
+ raise JournalError(
6375
+ "durable selector state does not match the journal selection "
6376
+ "derived from the same pinned prefix"
6377
+ )
6378
+
6379
+
4557
6380
  def stats_index_matches_journal_prefix(
4558
6381
  path: pathlib.Path, high_water: "tuple[str, int] | None"
4559
6382
  ) -> bool:
@@ -4568,6 +6391,26 @@ def stats_index_matches_journal_prefix(
4568
6391
  if not pathlib.Path(path).exists():
4569
6392
  return False
4570
6393
  try:
6394
+ all_segments = list_segments()
6395
+ segments = all_segments
6396
+ elision = None
6397
+ if high_water is not None:
6398
+ if high_water[0] in segments:
6399
+ segments = segments[:segments.index(high_water[0]) + 1]
6400
+ # THE SAME planner the rebuild uses, constructed the same way from
6401
+ # the same inputs (#496 S5b §5.6). An eliding rebuild and a prefix
6402
+ # validation that disagreed about the same journal would make the
6403
+ # validation meaningless.
6404
+ #
6405
+ # Constructed BEFORE the `stats.db` connection, deliberately: the
6406
+ # planner reads `cache.db`, and the lock-order law runs cache before
6407
+ # stats. Building it inside the stats connection's lifetime would
6408
+ # open the cache underneath an already-open stats reader. Neither
6409
+ # read takes a flock and Python's sqlite3 holds no implicit read
6410
+ # transaction across `fetchall()`, so nothing deadlocks today — but
6411
+ # the law is a TOTAL order, and the cost of keeping it is one short
6412
+ # cache read on a path that was about to fail structurally.
6413
+ elision = plan_segment_elision(segments, high_water)
4571
6414
  conn = sqlite3.connect(f"file:{path}?mode=ro", uri=True)
4572
6415
  try:
4573
6416
  _validate_rebuilt_stats_index(conn, high_water)
@@ -4582,17 +6425,28 @@ def stats_index_matches_journal_prefix(
4582
6425
  protocol_evidence = []
4583
6426
  prior_high_water = None
4584
6427
  cutover_captured = _CUTOVER_UNSEEN
4585
- all_segments = list_segments()
4586
- segments = all_segments
6428
+ marker_coordinates: dict = {}
4587
6429
  hasher = _lib_journal_router.PrefixHashAccumulator()
4588
6430
  if high_water is not None:
4589
- if high_water[0] in segments:
4590
- segments = segments[:segments.index(high_water[0]) + 1]
6431
+ def _elide_segment(name, lo, hi, stat_result) -> bool:
6432
+ nonlocal prior_high_water
6433
+ summary = elision.decide(name, hi, stat_result)
6434
+ if summary is None:
6435
+ return False
6436
+ # Only the placeholders and the boundary: this pass builds
6437
+ # no counters, retains no observation bytes and folds no
6438
+ # last-seen map, so the `decoded`-entry count is the whole
6439
+ # contribution.
6440
+ decoded.extend([None] * int(summary.decoded_entry_count))
6441
+ prior_high_water = (name, int(summary.summarized_size))
6442
+ return True
6443
+
4591
6444
  for segment, offset, raw in _iter_range_with_segments(
4592
6445
  None, high_water, segments,
4593
6446
  on_segment=lambda name: hasher.begin_segment(
4594
6447
  name, prior_high_water),
4595
6448
  on_bytes=hasher.extend,
6449
+ elide=_elide_segment,
4596
6450
  ):
4597
6451
  record = _lib_journal.decode_line(raw)
4598
6452
  if record is not None:
@@ -4600,11 +6454,23 @@ def stats_index_matches_journal_prefix(
4600
6454
  record,
4601
6455
  prior_high_water,
4602
6456
  protocol_evidence,
4603
- hasher=hasher,
6457
+ # See the rebuild's twin: a digest cannot be composed
6458
+ # over an elided prefix, so it is recomputed from
6459
+ # disk instead (§5.1).
6460
+ hasher=None if elision.elided else hasher,
4604
6461
  )
6462
+ if (isinstance(record.get("payload"), dict)
6463
+ and record["payload"].get("kind")
6464
+ == _lib_journal._PROTOCOL_RESOLUTION_KIND):
6465
+ elision.resolution_seen = True
4605
6466
  if (cutover_captured is _CUTOVER_UNSEEN
4606
6467
  and record.get("id") == CUTOVER_OP_ID):
4607
6468
  cutover_captured = _cutover_value_of(record)
6469
+ if record.get("t") == "correction_batch":
6470
+ marker_coordinates[len(decoded)] = (
6471
+ segment,
6472
+ offset + len(raw) + 1,
6473
+ )
4608
6474
  decoded.append(
4609
6475
  record
4610
6476
  if record.get("t")
@@ -4621,9 +6487,30 @@ def stats_index_matches_journal_prefix(
4621
6487
  for record in decoded:
4622
6488
  if record is not None:
4623
6489
  _normalize_legacy_account_stamp(record, cutover_claude)
6490
+ selector_accumulators: dict = {}
4624
6491
  selection = _lib_journal.resolve_effective_events(
4625
6492
  decoded,
4626
6493
  protocol_prefix_evidence=protocol_evidence,
6494
+ accumulators=selector_accumulators,
6495
+ )
6496
+ # The same semantic half the rebuild's oracle runs (#496 S5b §6.2).
6497
+ # Both planners derive it from THEIR OWN pinned traversal, so an
6498
+ # eliding rebuild and a prefix validation cannot reach different
6499
+ # answers about the same journal.
6500
+ _validate_selector_state(
6501
+ conn,
6502
+ _lib_selector_state.rows_from_selection(
6503
+ selection,
6504
+ accumulators=selector_accumulators,
6505
+ next_sequence=len(decoded),
6506
+ coordinates=marker_coordinates,
6507
+ covered=high_water,
6508
+ cutover_seen=cutover_captured is not _CUTOVER_UNSEEN,
6509
+ cutover_account_key=(
6510
+ None if cutover_captured is _CUTOVER_UNSEEN
6511
+ else cutover_captured
6512
+ ),
6513
+ ),
4627
6514
  )
4628
6515
  expected = []
4629
6516
  for event_id, selected in selection.by_id.items():
@@ -4726,6 +6613,27 @@ def _prepare_existing_stats_for_cutover(path: pathlib.Path) -> str:
4726
6613
  replacement main file.
4727
6614
  """
4728
6615
  import _cctally_db
6616
+ import _lib_stats_wal
6617
+
6618
+ wal_index = _lib_stats_wal.inspect_wal_index_family(path)
6619
+ wal_verdict = wal_index.get("verdict")
6620
+ if _lib_stats_wal.is_incoherent_wal_index(wal_index):
6621
+ # The caller has already proved whole-family drain and, for a heal,
6622
+ # preserved the complete pre-checkpoint family. Opening SQLite here
6623
+ # would let a stale aPgno[] map direct valid WAL frames to wrong main
6624
+ # pages before quarantine records the original bytes.
6625
+ return "skipped_incoherent_wal_index"
6626
+ if wal_verdict in {"capture_raced", "analysis_truncated"}:
6627
+ raise JournalError(
6628
+ "old stats index WAL/SHM coherence could not be established "
6629
+ f"before cutover ({wal_verdict})"
6630
+ )
6631
+ if wal_verdict not in {"coherent", "wal_absent", "wal_empty"}:
6632
+ # A malformed/non-empty WAL, missing SHM, or another unrecognized raw
6633
+ # shape is not permission to let SQLite reconstruct or checkpoint it.
6634
+ # The caller has already preserved the complete family and can publish
6635
+ # the independently rebuilt index without mutating these old bytes.
6636
+ return "skipped_unproven_wal_index"
4729
6637
 
4730
6638
  try:
4731
6639
  conn = sqlite3.connect(str(path), timeout=15.0)
@@ -5214,6 +7122,7 @@ def _preserve_stats_family_for_cutover(
5214
7122
  "rebuildRecordPath": context.record_path,
5215
7123
  "binaryVersion": _binary_version(),
5216
7124
  "binaryEpoch": _cctally_core.STATS_INDEX_EPOCH,
7125
+ "sqliteRuntimeVersion": sqlite3.sqlite_version,
5217
7126
  "preservedUserVersion": preserved_user_version,
5218
7127
  "familySizes": family_sizes,
5219
7128
  # The retained COPY is described, not the live file, because the copy
@@ -5402,12 +7311,40 @@ def _publish_generation_in_place(
5402
7311
  conn.execute(f"PRAGMA main.user_version={epoch:d}")
5403
7312
  # The publication's own identity, committed atomically with the
5404
7313
  # content and the epoch it describes (#496 S3 §5).
7314
+ stamped_at = _utc_iso_now()
5405
7315
  conn.execute("DELETE FROM main.stats_publication_stamp")
5406
7316
  conn.execute(
5407
7317
  "INSERT INTO main.stats_publication_stamp "
5408
7318
  "(record_path, started_at_utc, stamped_at_utc) VALUES (?, ?, ?)",
5409
- (str(record_path), started_at, _utc_iso_now()),
7319
+ (str(record_path), started_at, stamped_at),
5410
7320
  )
7321
+ # Spec §7 case 5 asks for a failure BETWEEN the publication-stamp
7322
+ # write and the selector-identity write, so the seam sits here
7323
+ # rather than after both. An injection after both still established
7324
+ # the property, but it exercised a different interleaving than the
7325
+ # one the spec names.
7326
+ _stats_rebuild_test_pause("publication_before_commit")
7327
+ # The SAME identity onto the durable selector row, in the SAME
7328
+ # transaction (#496 S5b §3.4). The scratch was built before this
7329
+ # stamp existed, so a row populated there cannot carry the identity
7330
+ # it will publish under; writing it here is what lets a live tick
7331
+ # prove its durable state belongs to the generation it is reading.
7332
+ #
7333
+ # The existence probe is not tolerance for a missing table on a real
7334
+ # generation — `_REBUILD_REQUIRED_TABLES` makes validation refuse a
7335
+ # scratch that lacks it. This function publishes whatever schema its
7336
+ # scratch carries, and the publication-protocol tests build minimal
7337
+ # synthetic generations that legitimately have no selector state.
7338
+ if conn.execute(
7339
+ "SELECT 1 FROM main.sqlite_schema WHERE type='table' "
7340
+ "AND name='journal_selector_state'"
7341
+ ).fetchone() is not None:
7342
+ conn.execute(
7343
+ "UPDATE main.journal_selector_state "
7344
+ "SET generation_record_path = ?, "
7345
+ "generation_stamped_at_utc = ?",
7346
+ (str(record_path), stamped_at),
7347
+ )
5411
7348
  phase = sp.COMMIT_UNKNOWN
5412
7349
  conn.commit()
5413
7350
  phase = sp.COMMITTED
@@ -5451,30 +7388,33 @@ def _checkpoint_after_publication(conn: sqlite3.Connection) -> str:
5451
7388
  return "checkpointed" if int(row[0]) == 0 else "busy"
5452
7389
 
5453
7390
 
5454
- def _remove_empty_db_sidecars(path) -> None:
5455
- """Remove the sidecars a read-only validation open leaves behind.
5456
-
5457
- Unlike the physical path, an in-place publish leaves a LEGITIMATE WAL: it
5458
- belongs to the live database, and deleting a non-empty one would discard
5459
- committed frames. Only a zero-length WAL is removed the state a
5460
- successful TRUNCATE checkpoint and a clean last close leave so the
5461
- documented sidecar-free end state still holds without ever risking data.
5462
- """
5463
- path = pathlib.Path(path)
5464
- wal = pathlib.Path(str(path) + "-wal")
5465
- try:
5466
- if wal.stat().st_size:
5467
- return
5468
- except OSError:
5469
- pass
5470
- for suffix in ("-wal", "-shm"):
5471
- try:
5472
- pathlib.Path(str(path) + suffix).unlink()
5473
- except FileNotFoundError:
5474
- pass
5475
- except OSError:
5476
- return
5477
- _fsync_dir(path.parent)
7391
+ # An in-place publication used to unlink the live `-wal` and `-shm` here once
7392
+ # the TRUNCATE checkpoint had emptied the WAL, to reach a sidecar-free end
7393
+ # state. That is issue #516 and the call is gone: unlinking the sidecars of a
7394
+ # database other connections still hold open is outside SQLite's contract, and
7395
+ # what it cost is a data-correctness fault and a crash, NOT a cosmetic end
7396
+ # state. Two conditions have to combine, and an earlier thirteen-arrangement
7397
+ # run never combined them: something must WRITE after the unlink, and the
7398
+ # reader must already have READ before it. Re-measured on both LAN runners
7399
+ # (macOS, Python 3.13.14, SQLite 3.53.4, byte-identical on each):
7400
+ #
7401
+ # - later writer in the SAME process — a reader that had read before the
7402
+ # unlink, read-write or `mode=ro`, raises `OperationalError: disk I/O
7403
+ # error`; and a connection that keeps writing through the unlinked inodes
7404
+ # breaks the NEXT connection opened in that process the same way;
7405
+ # - later writer in a CHILD process — that reader silently reads a STALE
7406
+ # generation, and so does a freshly opened connection in the parent;
7407
+ # - a reader that had NOT read before the unlink reads the current value,
7408
+ # which is why the earlier run observed nothing;
7409
+ # - a reader holding a pinned read transaction makes the checkpoint busy, so
7410
+ # the unlink is refused before it can happen.
7411
+ #
7412
+ # One mechanism seen from three ends: whichever connection holds an fd on the
7413
+ # stale `-wal` inode while the shared wal-index describes frames in the other
7414
+ # `-wal` inode takes the short read. A clean last close removes both sidecars
7415
+ # by itself, so nothing is leaked; only the case where a handle is open, which
7416
+ # is exactly the case that must not be unlinked, now leaves a zero-length WAL
7417
+ # behind.
5478
7418
 
5479
7419
 
5480
7420
  #: Probe 2 measured the live WAL peaking at 1.01x the main file across five
@@ -5676,7 +7616,6 @@ def _publish_stats_index_in_place(
5676
7616
  post_error = validate_published_stats_index(
5677
7617
  destination, high_water, expected_record_path=str(record_path),
5678
7618
  )
5679
- _remove_empty_db_sidecars(destination)
5680
7619
 
5681
7620
  live["publicationCheckpoint"] = checkpoint_outcome
5682
7621
  live["postPublicationValidation"] = {
@@ -5903,8 +7842,552 @@ def _decoded_quota_stream(quota_raw, cutover_claude, counters=None):
5903
7842
  yield record
5904
7843
 
5905
7844
 
7845
+ #: Every field the rebuild's F11 fast path needs, captured from ONE read-only
7846
+ #: snapshot so the certificate and the sequence it is validated against cannot
7847
+ #: come from two different cache states.
7848
+ #:
7849
+ #: ``conn`` is that snapshot's connection, and it is returned still inside its
7850
+ #: read transaction. Under WAL a read transaction's snapshot is fixed at its
7851
+ #: first read, so a caller that keeps it open sees the SAME cache state for
7852
+ #: every later read — which is what lets §4.4's projection bundle be captured
7853
+ #: from the snapshot the coverage verdict was decided against, rather than from
7854
+ #: a second connection opened later.
7855
+ _CoverageSnapshot = collections.namedtuple(
7856
+ "_CoverageSnapshot", ("certificate", "physical_seq", "conn"))
7857
+
7858
+
7859
+ def _close_coverage_snapshot(snapshot) -> None:
7860
+ """End the snapshot's read transaction and close it. Never raises."""
7861
+ if snapshot is None or snapshot.conn is None:
7862
+ return
7863
+ try:
7864
+ snapshot.conn.rollback()
7865
+ except sqlite3.Error:
7866
+ pass
7867
+ try:
7868
+ snapshot.conn.close()
7869
+ except sqlite3.Error:
7870
+ pass
7871
+
7872
+
7873
+ def _read_coverage_snapshot(cache_path) -> "_CoverageSnapshot | None":
7874
+ """The stored coverage certificate and physical sequence, in one `BEGIN`.
7875
+
7876
+ Read-only and WAL, so it takes NO writer flock and blocks no writer — which
7877
+ is the whole point of the fast path it feeds. Two separate reads could see
7878
+ a certificate from before a writer's transaction and a sequence from after
7879
+ it, which would validate a certificate the writer had just superseded.
7880
+
7881
+ **The transaction is left OPEN and the caller owns closing it.** Committing
7882
+ here would release the snapshot, and §4.4 requires the source roots, the
7883
+ observations and the ledger state to come from the same one. Every caller
7884
+ goes through `_close_coverage_snapshot`.
7885
+
7886
+ Any failure answers None, and None means replay. Every degraded state here
7887
+ falls back silently (spec §6.3).
7888
+ """
7889
+ import _cctally_cache
7890
+ try:
7891
+ conn = sqlite3.connect(
7892
+ f"file:{cache_path}?mode=ro", uri=True, timeout=5.0)
7893
+ except sqlite3.Error:
7894
+ return None
7895
+ try:
7896
+ conn.execute("PRAGMA busy_timeout=5000")
7897
+ conn.execute("BEGIN")
7898
+ certificate = _cctally_cache.load_codex_journal_coverage_certificate(conn)
7899
+ row = conn.execute(
7900
+ "SELECT value FROM cache_meta "
7901
+ "WHERE key='codex_physical_mutation_seq'"
7902
+ ).fetchone()
7903
+ except sqlite3.Error:
7904
+ _close_coverage_snapshot(_CoverageSnapshot(None, 0, conn))
7905
+ return None
7906
+ try:
7907
+ physical_seq = 0 if row is None or row[0] is None else int(row[0])
7908
+ except (TypeError, ValueError):
7909
+ _close_coverage_snapshot(_CoverageSnapshot(None, 0, conn))
7910
+ return None
7911
+ return _CoverageSnapshot(certificate, physical_seq, conn)
7912
+
7913
+
7914
+ def recover_quota_cache_from_journal(high_water=None, *, quiet=False) -> dict:
7915
+ """Replay the journal's Codex quota records into `cache.db`, no stats work.
7916
+
7917
+ This is the cache half of a rebuild, reachable on its own, which is what
7918
+ lets §4.7's open-time reconciliation resume a recovery a previous rebuild
7919
+ could not finish without re-publishing a whole generation. It reads the
7920
+ journal once, retains the observations as RAW BYTES for the same reason the
7921
+ rebuild does, and hands both populations to the same bounded, revalidating
7922
+ leg — so the caps, the per-chunk lock release, the restart rules and the
7923
+ mint are literally the same code, not a second implementation of them.
7924
+
7925
+ ``quiet`` suppresses the leg's `[rebuild]`-prefixed stderr. This function is
7926
+ reachable from an ordinary command, where acceptance criterion 10 requires
7927
+ no new stderr line and where a `[rebuild]` prefix would be a lie about which
7928
+ operation produced it. The rebuild itself passes the default.
7929
+
7930
+ Returns the leg's coverage record. `complete` is what the caller gates on.
7931
+ """
7932
+ hw = high_water if high_water is not None else journal_high_water()
7933
+ coverage: dict = {
7934
+ "status": "skipped", "reason": None, "replayedObservations": 0,
7935
+ "complete": True, "remainder": None,
7936
+ }
7937
+ if hw is None:
7938
+ return coverage
7939
+ segments = list_segments()
7940
+ if hw[0] not in segments:
7941
+ coverage.update({"status": "incomplete", "complete": False,
7942
+ "remainder": {"reason": "missingSegment"}})
7943
+ return coverage
7944
+ segments = segments[:segments.index(hw[0]) + 1]
7945
+
7946
+ quota_raw: list = []
7947
+ file_ops: list = []
7948
+ cutover_captured = _CUTOVER_UNSEEN
7949
+ # Advances for a malformed line too — consumed but not decoded — exactly as
7950
+ # the rebuild's `prior_high_water` does, because `_bounded_covered_offset`
7951
+ # is documented against the last line the pass CONSUMED.
7952
+ consumed_end = None
7953
+ for segment, offset, raw in _iter_range_with_segments(None, hw, segments):
7954
+ consumed_end = (segment, offset + len(raw) + 1)
7955
+ rec = _lib_journal.decode_line(raw)
7956
+ if rec is None:
7957
+ continue
7958
+ if (cutover_captured is _CUTOVER_UNSEEN
7959
+ and rec.get("id") == CUTOVER_OP_ID):
7960
+ cutover_captured = _cutover_value_of(rec)
7961
+ if _is_codex_file_account_op(rec):
7962
+ file_ops.append(rec)
7963
+ elif _is_codex_quota_obs(rec):
7964
+ quota_raw.append(raw)
7965
+ cutover_claude = _resolve_cutover_for_rebuild(
7966
+ cutover_captured, hw, segments)
7967
+ _rebuild_quota_cache_leg_raw(
7968
+ quota_raw, file_ops, cutover_claude, None,
7969
+ high_water=hw, coverage=coverage, decoded_end=consumed_end, quiet=quiet)
7970
+ # An absent cache.db is a clean skip for the REBUILD — the records stay
7971
+ # durable in the journal and a later `cache-sync` re-materializes them — but
7972
+ # it is not a completed recovery. The leg's `complete: True` describes "no
7973
+ # duty", and this caller's gate reads it as "the cache reached its target",
7974
+ # which for an absent cache with records to replay it certainly has not. The
7975
+ # distinction is made here rather than in the leg so the rebuild's
7976
+ # documented missing-cache behaviour is unchanged.
7977
+ if (coverage.get("complete") is True
7978
+ and (quota_raw or file_ops)
7979
+ and not _cctally_core.CACHE_DB_PATH.exists()):
7980
+ coverage.update({
7981
+ "status": "incomplete", "complete": False,
7982
+ "remainder": {
7983
+ "observations": len(quota_raw),
7984
+ "chunksRemaining": None,
7985
+ "reason": "cacheAbsent",
7986
+ },
7987
+ })
7988
+ return coverage
7989
+
7990
+
7991
+ #: How long one process waits before re-attempting a reconciliation that did not
7992
+ #: clear the flag.
7993
+ #:
7994
+ #: Nothing bounded the repetition before this. The states that leave the flag
7995
+ #: set are ordinary — `locksBusy` under a multi-agent hook storm, `restartLimit`
7996
+ #: under a competing `cache-sync --rebuild`, `noCoverageEstablished` and
7997
+ #: `mintRefused` — and in every one of them the next attempt re-read the whole
7998
+ #: journal and failed the same way. The marker is stamped on ATTEMPT rather than
7999
+ #: on outcome, because the cost this bounds is the read, which a failing attempt
8000
+ #: pays in full.
8001
+ _PROJECTION_RECONCILE_RETRY_SECONDS = 300.0
8002
+
8003
+
8004
+ def _projection_reconcile_marker_path():
8005
+ """Where the last reconciliation attempt is recorded.
8006
+
8007
+ Resolved at call time from `APP_DIR`, not captured at import, because the
8008
+ path constants are re-pointed by `_init_paths_from_env` and by every test
8009
+ fixture. It is a marker file rather than a column so the throttle costs no
8010
+ schema change: `stats_quota_projection_state` is part of the epoch-1009
8011
+ fingerprint, and widening it would move the hardcoded literal and every
8012
+ doctor golden with it.
8013
+ """
8014
+ return _cctally_core.APP_DIR / "stats.quota-reconcile.attempt"
8015
+
8016
+
8017
+ def _projection_reconcile_throttled() -> bool:
8018
+ """True when the previous attempt is too recent to repeat."""
8019
+ if _PROJECTION_RECONCILE_RETRY_SECONDS <= 0:
8020
+ return False
8021
+ try:
8022
+ age = time.time() - _projection_reconcile_marker_path().stat().st_mtime
8023
+ except OSError:
8024
+ return False
8025
+ return 0 <= age < _PROJECTION_RECONCILE_RETRY_SECONDS
8026
+
8027
+
8028
+ def _stamp_projection_reconcile_attempt() -> None:
8029
+ """Record an attempt. Never raises — a marker that cannot be written costs
8030
+ only a repeat, and failing an unrelated command over it would be worse."""
8031
+ try:
8032
+ _cctally_core.APP_DIR.mkdir(parents=True, exist_ok=True)
8033
+ _projection_reconcile_marker_path().touch()
8034
+ os.utime(_projection_reconcile_marker_path(), None)
8035
+ except OSError:
8036
+ pass
8037
+
8038
+
8039
+ def _clear_projection_reconcile_attempt() -> None:
8040
+ """Drop the marker after a reconciliation that actually cleared the flag.
8041
+
8042
+ Same never-raises posture as the stamp: a marker that cannot be removed
8043
+ costs only a delayed retry, and failing the caller over it would be worse.
8044
+ """
8045
+ try:
8046
+ _projection_reconcile_marker_path().unlink()
8047
+ except OSError:
8048
+ pass
8049
+
8050
+
8051
+ def _cache_writer_flocks_available() -> bool:
8052
+ """One non-blocking probe of the two cache writer flocks.
8053
+
8054
+ The journal read is the expensive half of a recovery and was paid FIRST: a
8055
+ pass that cannot take these flocks applies no row, so reading 1.64 GB to
8056
+ discover that is pure waste. Taken here, after the maintenance and ingest
8057
+ locks, so the probe respects §4.7's lock order rather than inverting it.
8058
+ """
8059
+ from _lib_cache_writer_lock import (
8060
+ acquire_cache_writer_flocks, release_cache_writer_flocks,
8061
+ )
8062
+ try:
8063
+ held = acquire_cache_writer_flocks(
8064
+ _cctally_core.CACHE_LOCK_PATH,
8065
+ _cctally_core.CACHE_LOCK_CODEX_PATH,
8066
+ timeout=None,
8067
+ )
8068
+ except OSError:
8069
+ return False
8070
+ if held is None:
8071
+ return False
8072
+ release_cache_writer_flocks(held)
8073
+ return True
8074
+
8075
+
8076
+ def reconcile_incomplete_quota_projection(conn) -> bool:
8077
+ """Resume cache recovery and re-materialize a gated quota projection.
8078
+
8079
+ Returns whether the flag was cleared. Called from `open_db` ahead of the
8080
+ current-epoch fast return, because §4.7's "the next open reconciles it" is
8081
+ only enforceable if some open actually does it — `RebuildResult` is
8082
+ process-local and a current-epoch open otherwise returns without any
8083
+ reconciliation at all.
8084
+
8085
+ `open_db` only reaches this for an ARMED process
8086
+ (`_cctally_core.enable_quota_projection_reconciliation`), because the work
8087
+ below reads the journal from zero to the current high water and `open_db`
8088
+ is on the status-line path. Three things bound what an armed process pays:
8089
+ the maintenance acquire is non-blocking rather than a thirty-second wait,
8090
+ the cache flocks are probed BEFORE the journal read, and a durable marker
8091
+ throttles the repeat.
8092
+
8093
+ Lock order is the repository's, stated in §4.7 and taken in this order:
8094
+ maintenance-exclusive, then the ingest lock, then (inside the leg) the
8095
+ global and Codex cache flocks, then the cache snapshot, then the stats
8096
+ transaction. The ingest acquire is OPPORTUNISTIC: an open that loses it
8097
+ leaves the flag set and the projection gated, which is the fail-closed
8098
+ direction, rather than blocking an interactive command behind an ingest
8099
+ cycle.
8100
+
8101
+ A caller that already holds the ingest lock is skipped outright. That
8102
+ context is the serialized writer — a rebuild or an ingest cycle — and it
8103
+ sets or clears this flag itself; reconciling underneath it would run a
8104
+ second recovery inside its transaction.
8105
+ """
8106
+ import _cctally_db
8107
+ import _cctally_store
8108
+ if conn is None or _cctally_store.holds_ingest_lock():
8109
+ return False
8110
+ # #146's rule is literally about advancing `user_version`, and this path
8111
+ # runs after the `_uv == STATS_INDEX_EPOCH` fast return, so it changes no
8112
+ # schema and no version. But it WRITES data rows to the real prod stats.db
8113
+ # and cache.db, and the cutover branch beside it in `open_db` already
8114
+ # refuses that from a dev checkout. `db rebuild` is the only setter of the
8115
+ # flag and already carries the guard, so the exposure is small — small is
8116
+ # not a reason for the two neighbouring write paths to disagree.
8117
+ if _cctally_db._would_block_prod_stats(_cctally_core.DB_PATH):
8118
+ return False
8119
+ try:
8120
+ row = conn.execute(
8121
+ "SELECT incomplete FROM stats_quota_projection_state WHERE id = 1"
8122
+ ).fetchone()
8123
+ except sqlite3.Error:
8124
+ # NOT "the flag must be clear" — that justification is false and
8125
+ # `assert_projection_readable` no longer uses it. This is the opposite
8126
+ # decision on the opposite question: a connection whose probe fails
8127
+ # cannot safely START a reconciliation, because the reconciliation
8128
+ # writes through this very connection and would take the maintenance,
8129
+ # ingest and cache locks to do it. Declining leaves the flag set, and
8130
+ # the gate — which fails CLOSED on the same error — still refuses every
8131
+ # projection read. The two directions agree on the outcome the user
8132
+ # sees: no incomplete projection is served.
8133
+ return False
8134
+ if row is None or not int(row[0] or 0):
8135
+ return False
8136
+ if _projection_reconcile_throttled():
8137
+ return False
8138
+ try:
8139
+ with _cctally_store.stats_open_time_guard(
8140
+ live=True, wait_seconds=0.0):
8141
+ # Re-read under the exclusive: another process may have reconciled
8142
+ # it while this one waited, and re-running the whole recovery to
8143
+ # discover that would cost a journal read for nothing.
8144
+ try:
8145
+ row = conn.execute(
8146
+ "SELECT incomplete FROM stats_quota_projection_state "
8147
+ "WHERE id = 1").fetchone()
8148
+ except sqlite3.Error:
8149
+ return False
8150
+ if row is None or not int(row[0] or 0):
8151
+ return False
8152
+ fd = _acquire_ingest_lock("opportunistic", 0.0)
8153
+ if fd is None:
8154
+ return False
8155
+ try:
8156
+ if not _cache_writer_flocks_available():
8157
+ return False
8158
+ _stamp_projection_reconcile_attempt()
8159
+ coverage = recover_quota_cache_from_journal(quiet=True)
8160
+ if coverage.get("complete") is not True:
8161
+ return False
8162
+ cleared = _rematerialize_and_clear_projection_gate(
8163
+ conn, quiet=True)
8164
+ if cleared:
8165
+ # The marker bounds the cost of a FAILING attempt. A success
8166
+ # leaves nothing to bound, and keeping it makes the throttle
8167
+ # punish the next genuine incompleteness: a flag set again
8168
+ # within the interval — a second interrupted rebuild, which
8169
+ # is exactly the sequence an upgrade under lock contention
8170
+ # produces — would wait the interval out for no reason.
8171
+ _clear_projection_reconcile_attempt()
8172
+ return cleared
8173
+ finally:
8174
+ _release_ingest_lock(fd)
8175
+ except _cctally_db.StatsDbMaintenanceError:
8176
+ # Another process owns stats maintenance. Leaving the flag set is the
8177
+ # fail-closed direction and costs nothing but a later attempt.
8178
+ return False
8179
+
8180
+
8181
+ def _rematerialize_and_clear_projection_gate(conn, *, quiet=False) -> bool:
8182
+ """Re-materialize the projection from a complete cache and clear the flag.
8183
+
8184
+ Both happen in ONE stats transaction, so a crash between them cannot leave a
8185
+ cleared flag over a projection that was never rewritten.
8186
+
8187
+ The bundle is read from a read-only cache snapshot taken AFTER recovery
8188
+ completed, which is the same ordering the rebuild's recovery path uses and
8189
+ for the same reason: a snapshot from before those writes would miss exactly
8190
+ the rows recovery restored.
8191
+
8192
+ **No bundle, no clear.** `rematerialize_quota_projection_for_rebuild`
8193
+ treats an absent or unreadable cache as a clean no-op, so calling it with
8194
+ `bundle=None` returns without touching a single projection row — and
8195
+ clearing the flag afterwards would serve the partial projection the flag
8196
+ exists to refuse over a projection this function never rewrote. Every other
8197
+ degraded path here fails closed and this one must too.
8198
+
8199
+ This is the SECOND of the two guards that close that path, and the two are
8200
+ NESTED rather than disjoint: the `cacheAbsent` remainder in
8201
+ `recover_quota_cache_from_journal` fires on a subset of the states this one
8202
+ covers, because an absent cache also yields no bundle. Both still earn their
8203
+ place. The first names the state in the coverage record, which is what the
8204
+ `db rebuild --json` `cacheRecovery.remainder.reason` reports and what an
8205
+ operator reads; this one additionally covers a cache that EXISTS but cannot
8206
+ be read, which the first sees as present and would let through.
8207
+ """
8208
+ import _cctally_quota as _q
8209
+ cache_path = _cctally_core.CACHE_DB_PATH
8210
+ bundle = None
8211
+ if cache_path.exists():
8212
+ try:
8213
+ cache = sqlite3.connect(
8214
+ f"file:{cache_path}?mode=ro", uri=True, timeout=5.0)
8215
+ except sqlite3.Error:
8216
+ return False
8217
+ try:
8218
+ cache.execute("PRAGMA busy_timeout=5000")
8219
+ cache.execute("BEGIN")
8220
+ bundle = _q.load_quota_projection_bundle(cache)
8221
+ except sqlite3.Error:
8222
+ return False
8223
+ finally:
8224
+ try:
8225
+ cache.rollback()
8226
+ except sqlite3.Error:
8227
+ pass
8228
+ cache.close()
8229
+ if bundle is None:
8230
+ return False
8231
+ try:
8232
+ conn.execute("BEGIN IMMEDIATE")
8233
+ _q.rematerialize_quota_projection_for_rebuild(conn, bundle=bundle)
8234
+ conn.execute(
8235
+ "UPDATE stats_quota_projection_state SET incomplete = 0, "
8236
+ "target_version = 0, recovery_target_json = NULL WHERE id = 1")
8237
+ conn.commit()
8238
+ except sqlite3.Error as exc:
8239
+ try:
8240
+ conn.rollback()
8241
+ except sqlite3.Error:
8242
+ pass
8243
+ if not quiet:
8244
+ print(f"[stats] quota projection reconciliation failed: {exc}",
8245
+ file=sys.stderr)
8246
+ return False
8247
+ return True
8248
+
8249
+
8250
+ #: The version stamped into `stats_quota_projection_state.target_version`.
8251
+ #:
8252
+ #: The target is VERSIONED rather than a bare coordinate so a target written by
8253
+ #: one binary is never misread by another: a later binary that changes what the
8254
+ #: target names reads a version it does not recognize and treats the projection
8255
+ #: as reconcilable-by-full-recovery instead of interpreting fields it would
8256
+ #: misunderstand.
8257
+ PROJECTION_RECOVERY_TARGET_VERSION = 1
8258
+
8259
+
8260
+ def _write_quota_projection_state(conn, *, coverage, high_water):
8261
+ """Record whether this generation's quota projection is complete.
8262
+
8263
+ Returns the flag it wrote, or **None** when it could not write one. The
8264
+ third state is the point: returning `False` for both "wrote clear" and
8265
+ "could not write" is what made a shipped test vacuous, because the caller
8266
+ then reported a complete projection for a generation whose flag says
8267
+ nothing. `None` is fail-closed at the caller — an unwritten flag is
8268
+ reported as incomplete, which costs a reconciliation and never serves a
8269
+ partial projection.
8270
+
8271
+ Runs inside the caller's transaction, which is the one that materialized
8272
+ the projection — the flag and the rows it describes must commit or roll
8273
+ back together.
8274
+
8275
+ A coverage record with no `complete` key is a leg that never ran (a
8276
+ `update_quota_cache=False` rebuild, or one with nothing to do), and that is
8277
+ complete by absence rather than incomplete by ignorance.
8278
+ """
8279
+ incomplete = bool(coverage) and coverage.get("complete") is False
8280
+ target = None
8281
+ if incomplete:
8282
+ target = json.dumps(
8283
+ {
8284
+ "highWater": (
8285
+ None if high_water is None
8286
+ else [str(high_water[0]), int(high_water[1])]
8287
+ ),
8288
+ "coveredHighWater": coverage.get("coveredHighWater"),
8289
+ "remainder": coverage.get("remainder"),
8290
+ },
8291
+ separators=(",", ":"), sort_keys=True,
8292
+ )
8293
+ try:
8294
+ conn.execute(
8295
+ "INSERT INTO stats_quota_projection_state"
8296
+ "(id, incomplete, target_version, recovery_target_json) "
8297
+ "VALUES (1, ?, ?, ?) "
8298
+ "ON CONFLICT(id) DO UPDATE SET incomplete = excluded.incomplete, "
8299
+ "target_version = excluded.target_version, "
8300
+ "recovery_target_json = excluded.recovery_target_json",
8301
+ (1 if incomplete else 0,
8302
+ PROJECTION_RECOVERY_TARGET_VERSION if incomplete else 0,
8303
+ target),
8304
+ )
8305
+ except sqlite3.Error as exc: # pragma: no cover — pre-1009 index
8306
+ print(f"[rebuild] quota projection state not recorded: {exc}",
8307
+ file=sys.stderr)
8308
+ return None
8309
+ return incomplete
8310
+
8311
+
8312
+ def _read_quota_projection_bundle(snapshot_out):
8313
+ """§4.4's projection bundle from a retained coverage snapshot, or None.
8314
+
8315
+ ``snapshot_out`` is the single-element list the leg appends its still-open
8316
+ read transaction to on the intact path. The bundle is read from that same
8317
+ transaction — under WAL its snapshot is fixed at the certificate read, so
8318
+ the certificate, the sequence, the source roots, the observations and the
8319
+ ledger state all describe ONE cache state.
8320
+
8321
+ None means "no snapshot to consume", and the projection then opens its own
8322
+ connection. That is the recovery path, where a snapshot from before the
8323
+ leg's writes would miss the rows recovery restored, and every degraded case,
8324
+ where falling back is the same silent full behaviour §6.3 asks for.
8325
+ """
8326
+ if not snapshot_out:
8327
+ return None
8328
+ snapshot = snapshot_out[0]
8329
+ if snapshot is None or snapshot.conn is None:
8330
+ return None
8331
+ try:
8332
+ import _cctally_quota as _q
8333
+ return _q.load_quota_projection_bundle(snapshot.conn)
8334
+ except (sqlite3.Error, ValueError):
8335
+ return None
8336
+ finally:
8337
+ _close_coverage_snapshot(snapshot)
8338
+
8339
+
8340
+ def _resolve_quota_cache_coverage(cache_path, high_water, decoded_end=None):
8341
+ """``(vector, covered, verdict, snapshot)`` for this coverage decision.
8342
+
8343
+ ``snapshot`` is returned with its read transaction still open, or None when
8344
+ none could be taken. The caller closes it — on the intact path after it has
8345
+ read §4.4's projection bundle from it, and immediately otherwise.
8346
+
8347
+ ``verdict`` is one of `_lib_cache_coverage`'s reason strings, and `REASON_OK`
8348
+ means the intact path: every cache-relevant journal record in the pinned
8349
+ prefix is already materialized, so the leg takes no writer flock at all.
8350
+
8351
+ Two independent checks, and BOTH are required. `certificate_is_valid` is an
8352
+ identity check — it asks whether the certificate describes the journal and
8353
+ cache physically in front of it — and a certificate covering only the first
8354
+ of three segments passes it. Whether coverage REACHES this rebuild's pinned
8355
+ high-water is a separate comparison, made here.
8356
+
8357
+ ``decoded_end`` is the traversal's own last complete-line boundary, and it
8358
+ bounds the covered claim for the reason `_bounded_covered_offset` gives.
8359
+ """
8360
+ vector = coverage_pinned_vector()
8361
+ covered = None
8362
+ for name, _raw_extent, covered_offset in vector:
8363
+ if high_water is not None and name == str(high_water[0]):
8364
+ covered = (name, _bounded_covered_offset(
8365
+ name, int(high_water[1]), covered_offset, decoded_end))
8366
+ if covered is None:
8367
+ # No boundary could be resolved at all. Reporting `identityRoot` here
8368
+ # would tell an operator the journal identity moved when in fact the
8369
+ # certificate was never consulted.
8370
+ return vector, None, _lib_cache_coverage.REASON_NO_BOUNDARY, None
8371
+ snapshot = _read_coverage_snapshot(cache_path)
8372
+ if snapshot is None:
8373
+ return vector, covered, _lib_cache_coverage.REASON_ABSENT, None
8374
+ ok, reason = _lib_cache_coverage.certificate_is_valid(
8375
+ snapshot.certificate,
8376
+ pinned_vector=vector,
8377
+ physical_seq=snapshot.physical_seq,
8378
+ )
8379
+ if not ok:
8380
+ return vector, covered, reason, snapshot
8381
+ if not _coordinate_covers(snapshot.certificate["coveredHighWater"], covered):
8382
+ return (vector, covered,
8383
+ _lib_cache_coverage.REASON_COVERED_HIGH_WATER, snapshot)
8384
+ return vector, covered, _lib_cache_coverage.REASON_OK, snapshot
8385
+
8386
+
5906
8387
  def _rebuild_quota_cache_leg_raw(
5907
- quota_raw, decoded, cutover_claude, counters=None
8388
+ quota_raw, decoded, cutover_claude, counters=None, *,
8389
+ high_water=None, coverage=None, decoded_end=None, snapshot_out=None,
8390
+ quiet=False, elision_gaps=None,
5908
8391
  ) -> float:
5909
8392
  """Re-materialize cache.db `quota_window_snapshots` AND the #416 Codex
5910
8393
  attribution map from the journal (spec §5.4 + #416 spec §3.4), fed RAW
@@ -5942,58 +8425,407 @@ def _rebuild_quota_cache_leg_raw(
5942
8425
  file_accounts = [
5943
8426
  r for r in decoded if r is not None and _is_codex_file_account_op(r)
5944
8427
  ]
5945
- if not quota_raw and not file_accounts:
8428
+ if coverage is not None:
8429
+ # `complete` is TRUE for a skip, and that is not a slip. Spec §4.7
8430
+ # distinguishes stats publication success from cache-recovery
8431
+ # completeness, and a leg with nothing to recover has no shortfall to
8432
+ # report — the incompleteness this flag exists for is an uncovered
8433
+ # REMAINDER, not an absent duty.
8434
+ coverage.update({
8435
+ "status": "skipped", "reason": None, "replayedObservations": 0,
8436
+ "chunks": 0, "plannedChunks": 0, "restarts": 0,
8437
+ "concurrentWriter": False, "complete": True, "remainder": None,
8438
+ })
8439
+ # `elision_gaps` keeps this from returning early on an empty stream: a pass
8440
+ # that elided every quota-bearing segment has an empty `quota_raw` and a
8441
+ # cache that may still need those observations if the certificate stopped
8442
+ # being valid while the pass ran.
8443
+ if not quota_raw and not file_accounts and not elision_gaps:
5946
8444
  return 0.0
5947
8445
  cache_path = _cctally_core.CACHE_DB_PATH
5948
8446
  if not cache_path.exists():
5949
8447
  return 0.0
8448
+
8449
+ # F11's intact path (spec §4.4). Decided from a read-only WAL snapshot
8450
+ # BEFORE any flock is requested, so a covered cache costs this rebuild zero
8451
+ # writer-lock hold rather than the measured 23.0 s it costs today. The
8452
+ # vector is pinned here for the same reason `_coverage_advance_plan` pins
8453
+ # its own before the flocks: one captured after a concurrent append would
8454
+ # match the current journal while a record nobody applied sat inside it.
8455
+ vector, covered, verdict, snapshot = _resolve_quota_cache_coverage(
8456
+ cache_path, high_water, decoded_end)
8457
+ if coverage is not None:
8458
+ coverage["reason"] = verdict
8459
+ if verdict == _lib_cache_coverage.REASON_OK:
8460
+ if coverage is not None:
8461
+ coverage.update({
8462
+ "status": "covered",
8463
+ "coveredHighWater": [covered[0], covered[1]],
8464
+ "replayedObservations": 0,
8465
+ "complete": True, "remainder": None,
8466
+ })
8467
+ # The intact path wrote nothing, so this snapshot still describes the
8468
+ # cache the verdict was decided against and §4.4's projection bundle may
8469
+ # be read from it. Hand it to the caller with its read transaction open.
8470
+ if snapshot_out is not None:
8471
+ snapshot_out.append(snapshot)
8472
+ else:
8473
+ _close_coverage_snapshot(snapshot)
8474
+ return 0.0
8475
+
8476
+ # The recovery path is about to WRITE to this cache, and a snapshot taken
8477
+ # before those writes would miss exactly the rows recovery restores. It is
8478
+ # closed here and the projection reads its own afterwards, which is what the
8479
+ # pre-change leg did at the same point.
8480
+ _close_coverage_snapshot(snapshot)
8481
+ if elision_gaps:
8482
+ # The read pass elided on a certificate that was valid when it planned,
8483
+ # and this leg's own verdict disagrees — an ordinary Codex batch landing
8484
+ # mid-pass is enough, because it advances the certificate over a journal
8485
+ # this pass did not pin. Recovery is about to replay, so the elided
8486
+ # segments' observations are re-read now, OUTSIDE both flocks, and the
8487
+ # replay proceeds over the same stream a non-eliding pass would have
8488
+ # built. Elision then costs one read in the racy case rather than an
8489
+ # unmaterialized observation under a certificate claiming coverage.
8490
+ before = len(quota_raw)
8491
+ quota_raw, refilled = _refill_elided_quota_raw(quota_raw, elision_gaps)
8492
+ if not refilled:
8493
+ # A segment this pass elided could not be re-read, so its
8494
+ # observations are absent from the stream about to be replayed.
8495
+ # `_run_bounded_recovery` mints whenever `covered` is not None, and
8496
+ # `covered` is the whole pinned prefix — so leaving it set would
8497
+ # certify coverage over rows nobody applied. Dropping it routes the
8498
+ # pass through `noCoverageEstablished`: it applies what it holds and
8499
+ # certifies nothing.
8500
+ covered = None
8501
+ if coverage is not None:
8502
+ coverage["elisionRefill"] = {
8503
+ "segments": len(elision_gaps),
8504
+ "observations": len(quota_raw) - before,
8505
+ "complete": refilled,
8506
+ }
8507
+ return _run_bounded_recovery(
8508
+ quota_raw, file_accounts, cutover_claude, counters,
8509
+ cache_path=cache_path, vector=vector, covered=covered,
8510
+ high_water=high_water, coverage=coverage, quiet=quiet,
8511
+ )
8512
+
8513
+
8514
+ #: Per-chunk caps for the recovery pass. BOTH are enforced (spec §4.5): capping
8515
+ #: by records alone lets one chunk of large observations blow the memory bound,
8516
+ #: and capping by bytes alone lets a chunk of tiny ones carry far more rows than
8517
+ #: one transaction should.
8518
+ #:
8519
+ #: 8 MiB against the maintainer's 905-byte mean observation is roughly 9,000
8520
+ #: records, so the byte cap binds first on real data and the record cap is the
8521
+ #: backstop for a journal of unusually small lines. The pair keeps peak decoded
8522
+ #: memory at one chunk, which is what makes per-chunk decode satisfy F11 and S4's
8523
+ #: measured 2.09 GB together.
8524
+ _RECOVERY_CHUNK_BYTES = 8 * 1024 * 1024
8525
+ _RECOVERY_CHUNK_RECORDS = 20_000
8526
+
8527
+ #: How many times one pass may restart from zero before giving up and reporting
8528
+ #: an uncovered remainder. A restart is triggered by a destructive writer, and a
8529
+ #: writer that keeps clearing the cache would otherwise make this pass loop for
8530
+ #: as long as it kept doing so. Three is enough to ride out a single competing
8531
+ #: `cache-sync --rebuild`; past that the honest answer is an incomplete pass,
8532
+ #: which §4.7 already has a contract for.
8533
+ _RECOVERY_MAX_RESTARTS = 3
8534
+
8535
+ #: Test-only seam, called with the number of chunks committed so far AFTER both
8536
+ #: flocks are released and BEFORE the next chunk requests them. That is exactly
8537
+ #: the window a competing writer can use, so a test proving a hold was RELEASED
8538
+ #: rather than merely shortened writes here rather than racing a thread.
8539
+ _RECOVERY_BETWEEN_CHUNKS = None
8540
+
8541
+
8542
+ def _recovery_pass_identity():
8543
+ """``(pass_id, started_at)`` for one recovery pass.
8544
+
8545
+ ``started_at`` is a coarse wall clock in microseconds, and it exists only to
8546
+ ORDER two passes for the monotonic compare-and-swap. It is never compared for
8547
+ equality and never used as a deadline.
8548
+ """
8549
+ return (
8550
+ hashlib.sha256(
8551
+ f"{os.getpid()}:{time.time_ns()}:{id(object())}".encode("utf-8")
8552
+ ).hexdigest()[:16],
8553
+ int(time.time() * 1_000_000),
8554
+ )
8555
+
8556
+
8557
+ def _recovery_state(cache):
8558
+ """``(physical_seq, source_roots_digest)`` read inside the caller's txn."""
8559
+ import _cctally_quota
8560
+ seq = _cctally_quota.codex_physical_mutation_seq(cache)
8561
+ digest = _lib_cache_coverage.source_roots_digest(
8562
+ _cctally_quota._cache_root_keys(cache))
8563
+ return seq, digest
8564
+
8565
+
8566
+ def _run_bounded_recovery(
8567
+ quota_raw, file_accounts, cutover_claude, counters, *,
8568
+ cache_path, vector, covered, high_water, coverage, quiet=False,
8569
+ ) -> float:
8570
+ """Recovery as resumable chunks, each capped by bytes AND record count.
8571
+
8572
+ Per chunk: decode that chunk's raw lines, acquire the global then the Codex
8573
+ flock, `BEGIN IMMEDIATE`, revalidate, apply, persist progress, commit,
8574
+ release both locks, discard the decoded chunk. The decode is DELIBERATELY
8575
+ per chunk and not once up front: decoding everything first would
8576
+ re-materialize the roughly six gigabytes of observation dictionaries S4
8577
+ removed to bring peak heap from 6.50 GB to 2.09 GB.
8578
+
8579
+ **Every chunk revalidates after reacquiring the locks, because releasing
8580
+ them admits a destructive writer.** A concurrent `cache-sync --rebuild` can
8581
+ take the locks between two chunks, run `_clear_codex_derived_rows`, and
8582
+ delete both the materialized quota state and the certificate. A pass that
8583
+ resumed from an in-memory cursor would continue over cleared state and mint
8584
+ a certificate claiming coverage the cache does not have. Progress is
8585
+ therefore persisted separately from the certificate, every destructive clear
8586
+ deletes it in the same transaction (`_invalidate_codex_journal_coverage_
8587
+ certificate` does both), and a pass that finds it gone restarts from zero.
8588
+ Restarting is always sound, because every apply is idempotent on its natural
8589
+ key.
8590
+
8591
+ Two pieces of per-record state survive the chunking (spec §4.6). The anchor
8592
+ resolver is RECONSTRUCTED per chunk — `_apply_quota_records` builds one per
8593
+ call, and it seeds lazily from the rows already stored, so each chunk's
8594
+ anchor decisions are made against the state committed at that moment rather
8595
+ than against a cache another writer may have mutated underneath a
8596
+ long-lived resolver. The conflict-report set is THREADED across the chunks
8597
+ of one pass, so a single recovery reports at most what a single unchunked
8598
+ call reports today.
8599
+
8600
+ Returns the total measured flock hold across every chunk.
8601
+ """
8602
+ import _cctally_cache
5950
8603
  from _lib_cache_writer_lock import (
5951
8604
  acquire_cache_writer_flocks,
5952
8605
  release_cache_writer_flocks,
5953
8606
  )
5954
8607
 
8608
+ pass_id, started_at = _recovery_pass_identity()
8609
+ identity_root = _lib_cache_coverage.identity_root(vector)
8610
+ spans = _lib_cache_coverage.chunk_spans(
8611
+ [len(raw) + 1 for raw in quota_raw],
8612
+ byte_cap=_RECOVERY_CHUNK_BYTES, record_cap=_RECOVERY_CHUNK_RECORDS,
8613
+ )
8614
+ # NOT cleared on a restart, deliberately. §4.6 requires that one recovery
8615
+ # report at most what a single unchunked call reports today, and a restart
8616
+ # replays records this pass has already reported a conflict for.
8617
+ reported_conflicts: set = set()
8618
+ # The replay counters are cumulative across every decode this pass makes, so
8619
+ # a restart would count the re-decoded prefix twice and inflate
8620
+ # `traversal["quota_replay"]` against what one unchunked call reports. They
8621
+ # are snapshotted here and restored on each restart.
8622
+ counter_baseline = dict(counters) if counters is not None else None
8623
+ hold_seconds = 0.0
8624
+ restarts = 0
8625
+ chunk_index = 0
8626
+ applied = 0
8627
+ committed_chunks = 0
8628
+ outcome = "recovered"
8629
+ stop_reason = None
8630
+ concurrent_writer = False
8631
+
5955
8632
  _cctally_core.APP_DIR.mkdir(parents=True, exist_ok=True)
5956
- try:
5957
- held = acquire_cache_writer_flocks(
5958
- _cctally_core.CACHE_LOCK_PATH,
5959
- _cctally_core.CACHE_LOCK_CODEX_PATH,
5960
- timeout=15.0,
5961
- )
5962
- except OSError as exc:
5963
- print(f"[rebuild] quota cache leg lock failed: {exc}", file=sys.stderr)
5964
- return 0.0
5965
- if held is None:
5966
- print("[rebuild] quota cache leg locks busy; skipping", file=sys.stderr)
5967
- return 0.0
5968
- held_from = time.monotonic()
5969
- try:
5970
- try:
5971
- cache = sqlite3.connect(str(cache_path), timeout=15.0)
5972
- except sqlite3.Error as exc: # pragma: no cover — cache.db unopenable
5973
- print(f"[rebuild] quota cache leg connect failed: {exc}", file=sys.stderr)
5974
- return time.monotonic() - held_from
8633
+
8634
+ def _record(status):
8635
+ if coverage is None:
8636
+ return
8637
+ coverage.update({
8638
+ "status": status,
8639
+ "coveredHighWater": (
8640
+ None if covered is None else [covered[0], covered[1]]),
8641
+ "replayedObservations": applied,
8642
+ "chunks": committed_chunks,
8643
+ "plannedChunks": len(plan),
8644
+ "restarts": restarts,
8645
+ "concurrentWriter": concurrent_writer,
8646
+ "complete": status == "recovered",
8647
+ "remainder": (
8648
+ None if status == "recovered"
8649
+ else {
8650
+ "observations": max(0, len(quota_raw) - applied),
8651
+ "chunksRemaining": max(
8652
+ 0, len(plan) - committed_chunks),
8653
+ "reason": stop_reason,
8654
+ }
8655
+ ),
8656
+ })
8657
+
8658
+ # Chunk 0 carries EVERY file-account decision, because §3.5 makes the
8659
+ # file/range decision authoritative over the observation stamp and a
8660
+ # decision must already govern the observations it covers. That population
8661
+ # is the retained decision records — 5.08% of a production journal — and it
8662
+ # is bounded by the decisions rather than by the observations, so it does
8663
+ # not reintroduce the unbounded hold the chunking exists to remove.
8664
+ #
8665
+ # It shares chunk 0's transaction with the first observation span rather
8666
+ # than taking a lock cycle of its own, so a recovery small enough to fit one
8667
+ # chunk takes exactly ONE hold — byte-identical in shape to the unchunked
8668
+ # form this replaces.
8669
+ if spans:
8670
+ plan = [(True, spans[0])] + [(False, span) for span in spans[1:]]
8671
+ else:
8672
+ plan = [(True, None)]
8673
+
8674
+ while chunk_index < len(plan):
8675
+ with_decisions, span = plan[chunk_index]
8676
+ decoded_chunk = None
8677
+ if span is not None:
8678
+ start, stop, _bytes = span
8679
+ decoded_chunk = list(_decoded_quota_stream(
8680
+ quota_raw[start:stop], cutover_claude, counters))
5975
8681
  try:
5976
- cache.execute("PRAGMA busy_timeout=15000")
5977
- cache.execute("BEGIN IMMEDIATE")
5978
- # Decisions FIRST — same §3.5 precedence ordering as `_cache_applier`.
5979
- _, _file_conflicts = _apply_file_account_records(cache, file_accounts)
5980
- _apply_quota_records(
5981
- cache,
5982
- _decoded_quota_stream(quota_raw, cutover_claude, counters),
8682
+ held = acquire_cache_writer_flocks(
8683
+ _cctally_core.CACHE_LOCK_PATH,
8684
+ _cctally_core.CACHE_LOCK_CODEX_PATH,
8685
+ timeout=15.0,
5983
8686
  )
5984
- cache.commit()
5985
- _report_file_account_conflicts(_file_conflicts)
5986
- except sqlite3.Error as exc:
8687
+ except OSError as exc:
8688
+ if not quiet:
8689
+ print(f"[rebuild] quota cache leg lock failed: {exc}",
8690
+ file=sys.stderr)
8691
+ outcome, stop_reason = "incomplete", "lockFailed"
8692
+ break
8693
+ if held is None:
8694
+ if not quiet:
8695
+ print("[rebuild] quota cache leg locks busy; skipping",
8696
+ file=sys.stderr)
8697
+ outcome, stop_reason = "incomplete", "locksBusy"
8698
+ break
8699
+ held_from = time.monotonic()
8700
+ cache = None
8701
+ try:
5987
8702
  try:
5988
- cache.rollback()
5989
- except sqlite3.Error:
5990
- pass
5991
- print(f"[rebuild] quota cache leg write failed: {exc}", file=sys.stderr)
8703
+ cache = sqlite3.connect(str(cache_path), timeout=15.0)
8704
+ cache.execute("PRAGMA busy_timeout=15000")
8705
+ cache.execute("BEGIN IMMEDIATE")
8706
+ if chunk_index > 0:
8707
+ seq, digest = _recovery_state(cache)
8708
+ verdict, why, saw_writer = (
8709
+ _lib_cache_coverage.resume_verdict(
8710
+ _cctally_cache.load_codex_recovery_progress(cache),
8711
+ pass_id=pass_id, started_at=started_at,
8712
+ identity_root=identity_root, physical_seq=seq,
8713
+ source_roots_digest=digest,
8714
+ )
8715
+ )
8716
+ concurrent_writer = concurrent_writer or saw_writer
8717
+ if verdict == _lib_cache_coverage.YIELD:
8718
+ cache.rollback()
8719
+ outcome, stop_reason = "incomplete", why
8720
+ break
8721
+ if verdict == _lib_cache_coverage.RESTART:
8722
+ cache.rollback()
8723
+ restarts += 1
8724
+ if restarts > _RECOVERY_MAX_RESTARTS:
8725
+ outcome, stop_reason = "incomplete", "restartLimit"
8726
+ break
8727
+ chunk_index = 0
8728
+ applied = 0
8729
+ committed_chunks = 0
8730
+ if counter_baseline is not None:
8731
+ counters.clear()
8732
+ counters.update(counter_baseline)
8733
+ continue
8734
+ file_conflicts = 0
8735
+ if with_decisions:
8736
+ # Decisions FIRST inside this transaction — the same §3.5
8737
+ # precedence ordering `_cache_applier` keeps.
8738
+ _restored, file_conflicts = _apply_file_account_records(
8739
+ cache, file_accounts)
8740
+ if decoded_chunk is not None:
8741
+ _apply_quota_records(
8742
+ cache, decoded_chunk,
8743
+ reported_conflicts=reported_conflicts, quiet=quiet)
8744
+ applied += len(decoded_chunk)
8745
+ last = chunk_index == len(plan) - 1
8746
+ if last and covered is None:
8747
+ # No boundary was resolvable at all, so this pass covered
8748
+ # bytes it cannot name and established nothing. Reporting
8749
+ # `complete` here would say "cache recovery complete" for a
8750
+ # pass that certified no prefix. The progress record goes
8751
+ # too: it describes a run that will never be finished.
8752
+ cache.execute(
8753
+ "DELETE FROM cache_meta WHERE key = ?",
8754
+ (_lib_cache_coverage.PROGRESS_KEY,))
8755
+ cache.commit()
8756
+ committed_chunks += 1
8757
+ chunk_index += 1
8758
+ _report_file_account_conflicts(file_conflicts, quiet=quiet)
8759
+ outcome, stop_reason = "incomplete", "noCoverageEstablished"
8760
+ break
8761
+ if last and covered is not None:
8762
+ # The pass has now replayed the whole pinned prefix, so it
8763
+ # is the one caller allowed to ESTABLISH coverage rather
8764
+ # than only extend it — it is the only writer that reads the
8765
+ # journal. Inside the same transaction as the last rows it
8766
+ # certifies, so a rollback leaves no certificate over rows
8767
+ # that never landed. The progress record goes away with it:
8768
+ # a certificate supersedes progress, and leaving both would
8769
+ # let a later pass resume a run that already finished.
8770
+ minted = _cctally_cache._advance_codex_journal_coverage(
8771
+ cache, prior=None, covered=covered,
8772
+ applied_through=(
8773
+ str(high_water[0]), int(high_water[1])),
8774
+ pinned_vector=vector, allow_mint=True)
8775
+ cache.execute(
8776
+ "DELETE FROM cache_meta WHERE key = ?",
8777
+ (_lib_cache_coverage.PROGRESS_KEY,))
8778
+ if not minted:
8779
+ # The mint refused — a stored certificate already reaches
8780
+ # further than this pass consumed to. Nothing false was
8781
+ # certified, but no coverage was established either, so
8782
+ # the honest report is an incomplete pass rather than
8783
+ # `complete: True` over an absent certificate.
8784
+ cache.commit()
8785
+ committed_chunks += 1
8786
+ chunk_index += 1
8787
+ _report_file_account_conflicts(file_conflicts, quiet=quiet)
8788
+ outcome, stop_reason = "incomplete", "mintRefused"
8789
+ break
8790
+ else:
8791
+ seq, digest = _recovery_state(cache)
8792
+ _cctally_cache._store_codex_recovery_progress(
8793
+ cache,
8794
+ _lib_cache_coverage.make_progress(
8795
+ pass_id=pass_id, started_at=started_at,
8796
+ chunks=chunk_index + 1,
8797
+ identity_root=identity_root, physical_seq=seq,
8798
+ source_roots_digest=digest,
8799
+ covered=(covered if covered is not None
8800
+ else ("", 0))),
8801
+ )
8802
+ cache.commit()
8803
+ committed_chunks += 1
8804
+ chunk_index += 1
8805
+ _report_file_account_conflicts(file_conflicts, quiet=quiet)
8806
+ except sqlite3.Error as exc:
8807
+ if cache is not None:
8808
+ try:
8809
+ cache.rollback()
8810
+ except sqlite3.Error:
8811
+ pass
8812
+ if not quiet:
8813
+ print(f"[rebuild] quota cache leg write failed: {exc}",
8814
+ file=sys.stderr)
8815
+ outcome, stop_reason = "failed", "writeFailed"
8816
+ break
8817
+ finally:
8818
+ if cache is not None:
8819
+ cache.close()
5992
8820
  finally:
5993
- cache.close()
5994
- finally:
5995
- release_cache_writer_flocks(held)
5996
- return time.monotonic() - held_from
8821
+ hold_seconds += time.monotonic() - held_from
8822
+ release_cache_writer_flocks(held)
8823
+ decoded_chunk = None
8824
+ if _RECOVERY_BETWEEN_CHUNKS is not None:
8825
+ _RECOVERY_BETWEEN_CHUNKS(committed_chunks)
8826
+
8827
+ _record(outcome)
8828
+ return hold_seconds
5997
8829
 
5998
8830
 
5999
8831
  #: `_resolve_cutover_for_rebuild` distinguishes "the streaming pass never saw the
@@ -6055,6 +8887,61 @@ def _resolve_cutover_for_rebuild(captured, hw, segments, counters=None) -> str:
6055
8887
  return _lib_accounts.UNATTRIBUTED
6056
8888
 
6057
8889
 
8890
+ def _observe_selector_desynchronization(path) -> "dict | None":
8891
+ """The selector prefix of the index at ``path``, when it is behind its cursor.
8892
+
8893
+ Returns `None` for the healthy case and for every unreadable one — an absent
8894
+ destination, a pre-1009 index without the selector tables, a missing cursor
8895
+ row. This is diagnostic reporting over a re-derivable artifact, so it must
8896
+ never be the reason a rebuild fails.
8897
+ """
8898
+ import _lib_stats_wal
8899
+
8900
+ path = pathlib.Path(path)
8901
+ if not path.exists():
8902
+ return None
8903
+ wal_index = _lib_stats_wal.inspect_wal_index_family(path)
8904
+ if wal_index.get("verdict") not in {"coherent", "wal_absent", "wal_empty"}:
8905
+ # This observation is optional. Opening an unproven WAL/SHM family can
8906
+ # rewrite its headers before the cutover path preserves the incident
8907
+ # bytes, so only let SQLite see a raw-classified safe family (#514).
8908
+ return None
8909
+ try:
8910
+ conn = sqlite3.connect(f"file:{path}?mode=ro", uri=True, timeout=5.0)
8911
+ except sqlite3.Error:
8912
+ return None
8913
+ try:
8914
+ state = _read_selector_state(conn)
8915
+ if state is None:
8916
+ return None
8917
+ cursor = _read_cursor(conn)
8918
+ except sqlite3.Error:
8919
+ return None
8920
+ finally:
8921
+ conn.close()
8922
+ covered = (state.covered_segment, state.covered_offset)
8923
+ covered_at = _prefix_position(covered)
8924
+ cursor_at = _prefix_position(cursor)
8925
+ if cursor_at is None or (covered_at is not None and covered_at >= cursor_at):
8926
+ return None
8927
+ # A gap the live path REFUSES to re-fold is a different state from one it
8928
+ # will close on the next tick, and only the rebuild record makes either
8929
+ # visible. `gapBytes` is `None` when the distance could not be determined,
8930
+ # which the live path also treats as over the cap.
8931
+ gap_bytes = _selector_gap_bytes(covered, cursor)
8932
+ return {
8933
+ "coveredSegment": covered[0],
8934
+ "coveredOffset": covered[1],
8935
+ "cursorSegment": cursor[0],
8936
+ "cursorOffset": int(cursor[1]),
8937
+ "gapBytes": gap_bytes,
8938
+ "gapByteCap": _GAP_REFOLD_BYTE_CAP,
8939
+ "gapExceedsCap": (
8940
+ gap_bytes is None or gap_bytes > _GAP_REFOLD_BYTE_CAP
8941
+ ),
8942
+ }
8943
+
8944
+
6058
8945
  def rebuild_stats_index(
6059
8946
  *,
6060
8947
  context: RebuildContext,
@@ -6128,6 +9015,13 @@ def rebuild_stats_index(
6128
9015
  )
6129
9016
  segments = segments[:segments.index(hw[0]) + 1]
6130
9017
 
9018
+ # Observed BEFORE the scratch is built, because it describes the index this
9019
+ # rebuild is about to replace (#496 S5b). It is the only report a persistent
9020
+ # selector desynchronization gets: the live path's own recovery is silent by
9021
+ # §6.3's uniform policy, so without this a repeatedly degrading generation
9022
+ # check would leave no trace anywhere.
9023
+ selector_desync = _observe_selector_desynchronization(dest)
9024
+
6131
9025
  stamp = dt.datetime.now(dt.timezone.utc).strftime("%Y%m%dT%H%M%S_%f")
6132
9026
  scratch = dest.with_name(dest.name + f".rebuilding-{stamp}")
6133
9027
  _remove_db_family(scratch)
@@ -6145,6 +9039,25 @@ def rebuild_stats_index(
6145
9039
  "quota_replay")
6146
9040
  }
6147
9041
  quota_lock_hold = 0.0
9042
+ #: Whether this generation's quota projection was materialized from a cache
9043
+ #: whose recovery left an uncovered remainder. Declared out here because the
9044
+ #: record and the result are assembled outside the try that sets it.
9045
+ stats_projection_incomplete = False
9046
+ #: §4.4's single read-only WAL snapshot, carried from the quota cache leg to
9047
+ #: the projection pass. The leg appends it ONLY on the intact path, where it
9048
+ #: wrote nothing and its snapshot therefore still describes the cache the
9049
+ #: coverage verdict was decided against. A destructive clear landing between
9050
+ #: the verdict and the projection would otherwise publish a generation whose
9051
+ #: quota projection was materialized from a cleared cache while the verdict
9052
+ #: already read `covered`. Declared out here so the `finally` that closes it
9053
+ #: cannot meet an unbound name.
9054
+ quota_snapshot: list = []
9055
+ #: #496 S5b §4.7 "recorded, not silent": the coverage verdict, the boundary
9056
+ #: it reached and how many observations it had to replay. Additive to a
9057
+ #: `schemaVersion: 1` record under S4's rule, and the ONLY surface reporting
9058
+ #: it — §6.3 makes every degraded coverage state a silent full replay.
9059
+ quota_coverage: dict = {
9060
+ "status": "skipped", "reason": None, "replayedObservations": 0}
6148
9061
  tracing = tracemalloc.is_tracing()
6149
9062
  if tracing:
6150
9063
  tracemalloc.reset_peak()
@@ -6160,23 +9073,70 @@ def rebuild_stats_index(
6160
9073
  protocol_evidence = []
6161
9074
  prior_high_water = None
6162
9075
  cutover_captured = _CUTOVER_UNSEEN
9076
+ # Sequence -> journal coordinate, for correction-batch MARKERS only
9077
+ # (#496 S5b §3.5). That is what removes `_correction_commit_high_water`'s
9078
+ # separate traversal from the live fast path, and it is bounded to the
9079
+ # markers because nothing else is ever looked up: a production journal's
9080
+ # 64,248 correction records carry far fewer markers than lines.
9081
+ marker_coordinates: dict = {}
6163
9082
  last_seen = _lib_journal_router.LastSeenAccumulator()
9083
+ summaries = _SegmentSummaryCollector()
9084
+ # F12 (#496 S5b §5). Planned before the first segment is reached, from
9085
+ # the sidecar this pass's predecessor wrote and the coverage certificate
9086
+ # Stage 3 mints. A plan that elides nothing is exactly today's pass.
9087
+ elision = plan_segment_elision(segments, hw)
6164
9088
  hasher = _lib_journal_router.PrefixHashAccumulator()
6165
9089
  evidence_seconds = 0.0
6166
9090
  prefix = traversal["stats_prefix"]
6167
9091
  read_started = time.monotonic()
9092
+
9093
+ def _elide_segment(name, lo, hi, stat_result) -> bool:
9094
+ """Contribute an elided segment's share instead of reading it."""
9095
+ nonlocal prior_high_water
9096
+ nonlocal malformed
9097
+ summary = elision.decide(name, hi, stat_result)
9098
+ if summary is None:
9099
+ return False
9100
+ prefix["lines"] += summary.lines
9101
+ prefix["bytes"] += summary.bytes
9102
+ prefix["decodes"] += summary.decodes
9103
+ malformed += summary.malformed
9104
+ # EXACTLY the segment's `decoded`-entry count. A quota-only segment
9105
+ # contributes only placeholders, and `resolve_effective_events`
9106
+ # numbers candidates with `enumerate(records)` — three of the seven
9107
+ # structural violation kinds hash that number into a DURABLE
9108
+ # fingerprint that `journal_protocol_resolution` ops reference by
9109
+ # name, so contributing the wrong count makes an acknowledged
9110
+ # violation unresolvable (#496 S5b §5.4).
9111
+ decoded.extend([None] * int(summary.decoded_entry_count))
9112
+ last_seen.merge(
9113
+ summary.last_seen_stamped,
9114
+ summary.last_seen_legacy_claude_at,
9115
+ summary.last_seen_legacy_codex_at,
9116
+ )
9117
+ summaries.adopt(summary)
9118
+ elision.quota_gaps.append(
9119
+ (name, len(quota_raw), summary.summarized_size, summary.lines))
9120
+ prior_high_water = (name, int(summary.summarized_size))
9121
+ return True
9122
+
6168
9123
  if hw is not None:
6169
9124
  for segment, offset, raw in _iter_range_with_segments(
6170
9125
  None, hw, segments,
6171
9126
  on_segment=lambda name: hasher.begin_segment(
6172
9127
  name, prior_high_water),
6173
9128
  on_bytes=hasher.extend,
9129
+ on_extent=lambda name, lo, hi, st: summaries.begin(
9130
+ name, lo, hi, st, last_seen),
9131
+ elide=_elide_segment,
6174
9132
  ):
6175
9133
  prefix["lines"] += 1
6176
9134
  prefix["bytes"] += len(raw) + 1
9135
+ summaries.line(raw, offset + len(raw) + 1)
6177
9136
  rec = _lib_journal.decode_line(raw)
6178
9137
  if rec is None:
6179
9138
  malformed += 1
9139
+ summaries.malformed_line()
6180
9140
  prior_high_water = (
6181
9141
  segment,
6182
9142
  offset + len(raw) + 1,
@@ -6195,16 +9155,49 @@ def rebuild_stats_index(
6195
9155
  rec,
6196
9156
  prior_high_water,
6197
9157
  protocol_evidence,
6198
- hasher=hasher,
9158
+ # OPTIMISTIC ELISION, RE-READ FALLBACK (#496 S5b §5.1).
9159
+ # `PrefixHashAccumulator` absorbs completed segments
9160
+ # into ONE sequential sha256 and `hashlib` can neither
9161
+ # export nor restore midstate, so a digest over a prefix
9162
+ # containing an elided segment is not computable from
9163
+ # the bytes this pass read. Handing `None` routes the
9164
+ # digest to `journal_prefix_hash`, which re-reads the
9165
+ # prefix from disk — including the elided segments — and
9166
+ # produces the byte-identical durable digest. The op's
9167
+ # own claimed hash is never accepted without this
9168
+ # recomputation.
9169
+ hasher=None if elision.elided else hasher,
6199
9170
  )
6200
9171
  evidence_seconds += time.monotonic() - evidence_started
9172
+ if (isinstance(rec.get("payload"), dict)
9173
+ and rec["payload"].get("kind")
9174
+ == _lib_journal._PROTOCOL_RESOLUTION_KIND):
9175
+ # Nothing further is elided in this pass. The
9176
+ # accumulator stays non-composable for the rest of it —
9177
+ # a gap already read is still a gap — so every later
9178
+ # evidence point also recomputes from disk; what this
9179
+ # stops is a NEW gap opening after an op that will
9180
+ # certainly be followed by more evidence points.
9181
+ elision.resolution_seen = True
6201
9182
  # First cutover op wins, exactly as `find_accounts_cutover_op`
6202
9183
  # scans — captured here so the rebuild reads the journal once.
6203
9184
  if (cutover_captured is _CUTOVER_UNSEEN
6204
9185
  and rec.get("id") == CUTOVER_OP_ID):
6205
9186
  cutover_captured = _cutover_value_of(rec)
6206
- last_seen.observe(rec, classify_legacy_provider)
6207
- if rec.get("t") in _lib_journal_router.RETAINED_RECORD_TYPES:
9187
+ # Folded into the OPEN SEGMENT's accumulator, which
9188
+ # `_SegmentSummaryCollector.close` merges into `last_seen` at the
9189
+ # boundary. The merge is a per-key maximum, so the resolved map
9190
+ # is identical to a single whole-pass fold (#496 S5b §5.4).
9191
+ summaries.last_seen.observe(rec, classify_legacy_provider)
9192
+ if rec.get("t") == "correction_batch":
9193
+ marker_coordinates[len(decoded)] = (
9194
+ segment,
9195
+ offset + len(raw) + 1,
9196
+ )
9197
+ retained = (
9198
+ rec.get("t") in _lib_journal_router.RETAINED_RECORD_TYPES)
9199
+ summaries.decoded(retained)
9200
+ if retained:
6208
9201
  decoded.append(rec)
6209
9202
  else:
6210
9203
  if update_quota_cache and _is_codex_quota_obs(rec):
@@ -6226,12 +9219,26 @@ def rebuild_stats_index(
6226
9219
  segment,
6227
9220
  offset + len(raw) + 1,
6228
9221
  )
9222
+ summaries.close(last_seen)
9223
+ # §6.3 "recorded, not silent": every elision refusal is a SILENT
9224
+ # fallback, and this block is the only surface that says which one.
9225
+ traversal["elision"] = elision.counters()
6229
9226
  phase_seconds["journal_read_decode"] = round(
6230
9227
  max(0.0, time.monotonic() - read_started - evidence_seconds), 6)
6231
9228
  phase_seconds["protocol_evidence"] = round(evidence_seconds, 6)
6232
9229
  traversal["protocol_evidence"]["bytes"] = hasher.bytes_hashed
6233
9230
  traversal["protocol_evidence"]["lines"] = hasher.digests_computed
6234
9231
  hasher = None
9232
+ # The sidecar is a pure re-derivable cache, so it is refreshed on the
9233
+ # way past rather than guarded by anything: this pass just pinned every
9234
+ # extent it describes, and `write_sidecar` swallows its own I/O failures
9235
+ # because a pass that could not write it is still a correct pass.
9236
+ if summaries.summaries:
9237
+ _lib_segment_summary.write_sidecar(
9238
+ segment_summary_sidecar_path(),
9239
+ [summaries.summaries[name] for name in segments
9240
+ if name in summaries.summaries],
9241
+ )
6235
9242
 
6236
9243
  # Legacy account normalisation (#341, spec §2 / handoff item 2): a
6237
9244
  # pre-#341 real-account line lacks an account stamp — inject the cutover
@@ -6254,9 +9261,29 @@ def rebuild_stats_index(
6254
9261
  # malformed revision, divergent same-revision candidate, or invalid
6255
9262
  # committed manifest leaves the existing destination untouched.
6256
9263
  selection_started = time.monotonic()
9264
+ selector_accumulators: dict = {}
6257
9265
  effective = _lib_journal.resolve_effective_events(
6258
9266
  decoded,
6259
9267
  protocol_prefix_evidence=protocol_evidence,
9268
+ accumulators=selector_accumulators,
9269
+ )
9270
+ # Durable selector state comes from THIS pinned traversal (#496 S5b
9271
+ # §3.3), so publication carries the fold and the index content together
9272
+ # and nothing has to read the journal a second time to derive it. The
9273
+ # generation identity is deliberately absent here: `stats_publication_
9274
+ # stamp` is written after the scratch is built, so a row populated now
9275
+ # cannot carry the identity it will publish under.
9276
+ selector_rows = _lib_selector_state.rows_from_selection(
9277
+ effective,
9278
+ accumulators=selector_accumulators,
9279
+ next_sequence=len(decoded),
9280
+ coordinates=marker_coordinates,
9281
+ covered=hw,
9282
+ cutover_seen=cutover_captured is not _CUTOVER_UNSEEN,
9283
+ cutover_account_key=(
9284
+ None if cutover_captured is _CUTOVER_UNSEEN
9285
+ else cutover_captured
9286
+ ),
6260
9287
  )
6261
9288
  phase_seconds["effective_selection"] = round(
6262
9289
  time.monotonic() - selection_started, 6)
@@ -6268,7 +9295,10 @@ def rebuild_stats_index(
6268
9295
  leg_started = time.monotonic()
6269
9296
  if update_quota_cache:
6270
9297
  quota_lock_hold = _rebuild_quota_cache_leg_raw(
6271
- quota_raw, decoded, cutover_claude, traversal["quota_replay"])
9298
+ quota_raw, decoded, cutover_claude, traversal["quota_replay"],
9299
+ high_water=hw, coverage=quota_coverage,
9300
+ decoded_end=prior_high_water, snapshot_out=quota_snapshot,
9301
+ elision_gaps=elision.quota_gaps)
6272
9302
  quota_raw = []
6273
9303
  phase_seconds["quota_cache_leg"] = round(
6274
9304
  time.monotonic() - leg_started, 6)
@@ -6297,7 +9327,7 @@ def rebuild_stats_index(
6297
9327
  # snapshots, resets+suppression, block_close, arming, credit effects.
6298
9328
  conn.execute("BEGIN IMMEDIATE")
6299
9329
  try:
6300
- _write_effective_metadata(conn, effective)
9330
+ _write_selector_state(conn, selector_rows)
6301
9331
  for _order, _seq, kind, rec in structural:
6302
9332
  if kind == "op":
6303
9333
  FOLD_APPLIERS[(rec.get("payload") or {}).get("kind")](conn, rec)
@@ -6311,11 +9341,44 @@ def rebuild_stats_index(
6311
9341
  except Exception:
6312
9342
  pass
6313
9343
  raise
9344
+ # Reported separately from `stats_fold` because this span is PART of how
9345
+ # long the retained cache read snapshot stays open past the point it
9346
+ # could have been consumed — see the WAL-pinning note in
9347
+ # docs/journal-gotchas.md. Measured at 0.56 s over a 1.6 GB journal, and
9348
+ # that figure is a LOWER BOUND on the pin, not the pin itself: this
9349
+ # timer starts at `fold_started`, which is set after the fold stream is
9350
+ # built and sorted above, and the snapshot opened earlier still inside
9351
+ # `_rebuild_quota_cache_leg_raw`. The full pin is the leg's post-snapshot
9352
+ # span plus the stream build and sort plus this fold. `quota_cache_leg`
9353
+ # is reported separately, so an operator can bound the first term.
9354
+ phase_seconds["structural_fold"] = round(
9355
+ time.monotonic() - fold_started, 6)
9356
+
9357
+ # §4.4's bundle is read from the retained snapshot here, BEFORE phase 2a
9358
+ # and before the stats transaction, so the cache read transaction closes
9359
+ # as early as it can rather than being held across a stats write. An
9360
+ # absent or unreadable snapshot leaves the bundle None, and the
9361
+ # projection reads its own connection exactly as it did before.
9362
+ #
9363
+ # An open WAL read transaction holds a read mark, so
9364
+ # `wal_checkpoint(TRUNCATE)` from any other process returns busy for as
9365
+ # long as this snapshot lives — which disables both of issue #297's
9366
+ # persistent defences. Reading it here rather than after phase 2a is
9367
+ # what bounds that window: measured on a 1.6 GB journal, the structural
9368
+ # fold above is 0.56 s and the open-block projection below is 27.7 s, so
9369
+ # this placement removes about 98% of the pin. The RELATIVE claim is
9370
+ # what those two figures support; neither is the absolute pin, because
9371
+ # the snapshot opens inside the quota cache leg and both timers start
9372
+ # later (see the note on `structural_fold` above). It cannot move any
9373
+ # earlier without holding the bundle's 254 MiB of observations (measured
9374
+ # on the same store, 232,466 rows) across the structural fold as well.
9375
+ quota_bundle = _read_quota_projection_bundle(quota_snapshot)
6314
9376
 
6315
9377
  # Phase 2a — OPEN 5h block projection (own txn; block-only). Closed blocks
6316
9378
  # came from block_close evts; this materializes the never-closed window(s)
6317
9379
  # so the five_hour_milestone block_id derived_fk resolves. Best-effort
6318
9380
  # (the open block is a projection, §5.3).
9381
+ projection_started = time.monotonic()
6319
9382
  try:
6320
9383
  cctally = sys.modules.get("cctally")
6321
9384
  bf = getattr(cctally, "_backfill_five_hour_blocks", None)
@@ -6324,6 +9387,8 @@ def rebuild_stats_index(
6324
9387
  except Exception as exc: # pragma: no cover — projection is best-effort
6325
9388
  print(f"[rebuild] open 5h block re-materialization failed: {exc}",
6326
9389
  file=sys.stderr)
9390
+ phase_seconds["open_block_projection"] = round(
9391
+ time.monotonic() - projection_started, 6)
6327
9392
 
6328
9393
  # Phase 2b + 3 (txn B) — quota projection re-materialization (after the
6329
9394
  # order-45 arming folds) + milestone/budget folds + cursor advance.
@@ -6331,10 +9396,26 @@ def rebuild_stats_index(
6331
9396
  try:
6332
9397
  try:
6333
9398
  import _cctally_quota as _q
6334
- _q.rematerialize_quota_projection_for_rebuild(conn)
9399
+ _q.rematerialize_quota_projection_for_rebuild(
9400
+ conn, bundle=quota_bundle)
6335
9401
  except Exception as exc: # pragma: no cover — projection best-effort
6336
9402
  print(f"[rebuild] quota projection re-materialization failed: {exc}",
6337
9403
  file=sys.stderr)
9404
+ # §4.7's durable, per-transaction gate. It is set in the SAME
9405
+ # transaction that materializes the projection it describes, so a
9406
+ # rollback leaves neither, and it rides into the live index with the
9407
+ # generation's own content — a process-local `RebuildResult` field
9408
+ # could not survive publication, and in-place publication
9409
+ # deliberately keeps already-open readers alive, so a connection can
9410
+ # observe the incomplete generation without ever calling `open_db`
9411
+ # again.
9412
+ _written = _write_quota_projection_state(
9413
+ conn, coverage=quota_coverage, high_water=hw)
9414
+ # `None` is "could not write the flag at all". Fail CLOSED: report
9415
+ # incomplete, so the generation is reconciled rather than served on
9416
+ # the strength of a flag nobody could store.
9417
+ stats_projection_incomplete = (
9418
+ True if _written is None else _written)
6338
9419
  for _order, _seq, _kind, rec in tail:
6339
9420
  _apply_evt(conn, rec)
6340
9421
  lines_folded += 1
@@ -6371,11 +9452,29 @@ def rebuild_stats_index(
6371
9452
  if checkpoint is not None and int(checkpoint[0]) != 0:
6372
9453
  raise JournalError("rebuilt stats index WAL could not be drained")
6373
9454
  _validate_rebuilt_stats_index(conn, hw)
9455
+ # The SEMANTIC half (#496 S5b §6.2). The structural checks above cover
9456
+ # the new tables' existence and definition; without this, a scratch
9457
+ # carrying correct legacy rows and WRONG selector state would validate,
9458
+ # receive a matching generation stamp, and then be trusted by the fast
9459
+ # path — which is the precise failure shape epic #496 exists to
9460
+ # eliminate, reintroduced one layer up. `selector_rows` is the full
9461
+ # selection this pass already derived from the pinned traversal, so the
9462
+ # comparison costs no second journal read.
9463
+ _validate_selector_state(conn, selector_rows)
6374
9464
  _stats_rebuild_test_pause("rebuild_scratch_complete")
6375
9465
  phase_seconds["scratch_validate"] = round(
6376
9466
  time.monotonic() - validate_started, 6)
6377
9467
  finally:
6378
9468
  conn.close()
9469
+ # A retained coverage snapshot holds an open read transaction on
9470
+ # `cache.db`, which pins the WAL against checkpointing for as long as it
9471
+ # lives. `_read_quota_projection_bundle` closes it on every ordinary
9472
+ # path; this covers the paths that raise before reaching it, which
9473
+ # matters most in a long-lived process such as the dashboard's auto-heal
9474
+ # thread. Closing twice is harmless.
9475
+ for _retained in quota_snapshot:
9476
+ _close_coverage_snapshot(_retained)
9477
+ quota_snapshot = []
6379
9478
 
6380
9479
  # Closed, drained, validated, and durable before the old family is touched.
6381
9480
  _remove_db_sidecars_strict(scratch)
@@ -6427,6 +9526,7 @@ def rebuild_stats_index(
6427
9526
  "forensicsPath": context.forensics_path,
6428
9527
  "binaryVersion": _binary_version(),
6429
9528
  "binaryEpoch": _cctally_core.STATS_INDEX_EPOCH,
9529
+ "sqliteRuntimeVersion": sqlite3.sqlite_version,
6430
9530
  "highWater": [hw[0], hw[1]] if hw is not None else None,
6431
9531
  "destination": str(dest),
6432
9532
  "targetPath": str(target_path) if target_path is not None else None,
@@ -6447,6 +9547,14 @@ def rebuild_stats_index(
6447
9547
  },
6448
9548
  "peakHeapBytes": peak_heap_bytes,
6449
9549
  "quotaLockHoldSeconds": round(quota_lock_hold, 6),
9550
+ "selectorDesynchronized": selector_desync,
9551
+ "quotaCacheCoverage": quota_coverage,
9552
+ # §4.7: distinct from `quotaCacheCoverage`, which describes the
9553
+ # CACHE. This describes the PUBLISHED INDEX, and a consumer must not
9554
+ # read one as the other. The record is written before publication
9555
+ # returns, so a crash afterwards still leaves the remainder
9556
+ # discoverable.
9557
+ "statsQuotaProjectionIncomplete": stats_projection_incomplete,
6450
9558
  },
6451
9559
  )
6452
9560
  phase_seconds["publication"] = round(
@@ -6463,6 +9571,9 @@ def rebuild_stats_index(
6463
9571
  traversal=traversal,
6464
9572
  peak_heap_bytes=peak_heap_bytes,
6465
9573
  quota_lock_hold_seconds=round(quota_lock_hold, 6),
9574
+ selector_desynchronized=selector_desync,
9575
+ quota_cache_coverage=quota_coverage,
9576
+ stats_quota_projection_incomplete=stats_projection_incomplete,
6466
9577
  )
6467
9578
 
6468
9579