cctally 1.117.2 → 1.118.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/CHANGELOG.md +30 -0
  2. package/README.md +7 -3
  3. package/bin/_cctally_cache.py +2783 -175
  4. package/bin/_cctally_completion.py +106 -33
  5. package/bin/_cctally_core.py +69 -4
  6. package/bin/_cctally_dashboard.py +188 -13
  7. package/bin/_cctally_dashboard_conversation.py +5 -1
  8. package/bin/_cctally_dashboard_envelope.py +50 -19
  9. package/bin/_cctally_dashboard_sources.py +23 -1
  10. package/bin/_cctally_db.py +193 -153
  11. package/bin/_cctally_diagnosis_sources.py +63 -7
  12. package/bin/_cctally_doctor.py +865 -93
  13. package/bin/_cctally_five_hour.py +67 -50
  14. package/bin/_cctally_forecast.py +35 -13
  15. package/bin/_cctally_journal.py +186 -18
  16. package/bin/_cctally_milestone_history.py +859 -62
  17. package/bin/_cctally_oauth_samples.py +286 -15
  18. package/bin/_cctally_parser.py +2 -2
  19. package/bin/_cctally_quota.py +104 -12
  20. package/bin/_cctally_record.py +153 -29
  21. package/bin/_cctally_rederive.py +608 -68
  22. package/bin/_cctally_statusline.py +108 -81
  23. package/bin/_cctally_store.py +179 -38
  24. package/bin/_cctally_tui.py +892 -181
  25. package/bin/_cctally_update.py +32 -13
  26. package/bin/_lib_codex_conversation_query.py +52 -14
  27. package/bin/_lib_codex_drain_admission.py +224 -6
  28. package/bin/_lib_codex_metadata.py +39 -0
  29. package/bin/_lib_conversation_anon.py +15 -8
  30. package/bin/_lib_conversation_dispatch.py +7 -0
  31. package/bin/_lib_conversation_query.py +289 -70
  32. package/bin/_lib_dashboard_sources.py +37 -1
  33. package/bin/_lib_detector_access_registry.py +497 -26
  34. package/bin/_lib_doctor.py +85 -20
  35. package/bin/_lib_doctor_scan_index.py +321 -0
  36. package/bin/_lib_ingest_frontier.py +20 -1
  37. package/bin/_lib_oauth_samples.py +23 -7
  38. package/bin/_lib_pricing.py +146 -16
  39. package/bin/_lib_quota.py +19 -0
  40. package/bin/_lib_rederive.py +11 -6
  41. package/bin/_lib_retained_records.py +516 -0
  42. package/bin/_lib_snapshot_cache.py +10 -0
  43. package/bin/_lib_stale_proof.py +272 -43
  44. package/bin/_lib_stats_content_revision.py +831 -0
  45. package/bin/_lib_write_io.py +11 -4
  46. package/bin/cctally +4 -1
  47. package/dashboard/static/assets/ConversationsView-D4Iv4ku0.js +72 -0
  48. package/dashboard/static/assets/{DoctorModal-B0DnU9d9.js → DoctorModal-D4QSq6uz.js} +1 -1
  49. package/dashboard/static/assets/ModalRoot-C9F5dhZi.js +1 -0
  50. package/dashboard/static/assets/{ProjectsDrillPanel-DSxXQSa-.js → ProjectsDrillPanel-WxAdRDMt.js} +1 -1
  51. package/dashboard/static/assets/{SourceDetailModal-BleDpQqG.js → SourceDetailModal-NyyEfXmc.js} +1 -1
  52. package/dashboard/static/assets/{UpdateModal-CMqaAkSK.js → UpdateModal-CFLq9pT1.js} +1 -1
  53. package/dashboard/static/assets/{index-BjQV-0-D.js → index-BKTWH9JE.js} +2 -2
  54. package/dashboard/static/assets/{outlineNavigation-CO-VWBm7.js → outlineNavigation-BnGUwCS3.js} +3 -3
  55. package/dashboard/static/dashboard.html +1 -1
  56. package/package.json +4 -1
  57. package/dashboard/static/assets/ConversationsView-oqNsWzWV.js +0 -72
  58. package/dashboard/static/assets/ModalRoot-CYua2wP_.js +0 -1
@@ -105,6 +105,7 @@ import contextvars
105
105
  import datetime as dt
106
106
  import errno
107
107
  import fcntl
108
+ import hashlib
108
109
  import json
109
110
  import os
110
111
  import pathlib
@@ -117,6 +118,7 @@ import subprocess
117
118
  import sys
118
119
  import tempfile
119
120
  import time
121
+ import uuid
120
122
  from dataclasses import asdict, dataclass, field, replace as _dc_replace
121
123
  from typing import Any, Callable, Iterable, Iterator, NamedTuple
122
124
 
@@ -708,6 +710,14 @@ CODEX_REPLAY_DRAIN_MARKER_NAME = "codex-replay-drain.last-attempt"
708
710
  #: far more often than the `doctor` WARN it produces is actionable.
709
711
  CODEX_REPLAY_DRAIN_SPAWN_THROTTLE_SECONDS = 3600
710
712
 
713
+ #: #982 spec 4.5a: the same spawn's spacing while a well-formed heal record
714
+ #: exists. A pass that ended `contended` or `degraded` leaves the record (in
715
+ #: phase `units`, a global fence over the population and every one-way stamp),
716
+ #: and the executor resumes it per file, so the retry costs one unit's work,
717
+ #: not a whole-tree re-read; waiting the hourly slot would keep the fence up
718
+ #: for an hour after any stall.
719
+ CODEX_HEAL_RETRY_THROTTLE_SECONDS = 300
720
+
711
721
  #: How long the drain worker waits for the cache writer flocks. The default
712
722
  #: ``None`` is a single non-blocking attempt, which spends the whole hourly slot
713
723
  #: on whichever ordinary sync happened to hold the lock at that instant. Nobody
@@ -871,6 +881,22 @@ def _set_cache_meta_if_changed(
871
881
  )
872
882
 
873
883
 
884
+ def _clear_claude_walk_verdicts(conn: sqlite3.Connection) -> None:
885
+ """Delete both Claude walk verdicts inside a destructive wipe's transaction.
886
+
887
+ The walk-complete marker and #916's residual token each vouch for the
888
+ cache's contents, so a rebuild or truncation wipe drops them in the same
889
+ transaction as the rows: a crash between the wipe's commit and the end of
890
+ its walk must not leave either one vouching for an emptied store. The
891
+ caller commits.
892
+ """
893
+ conn.execute(
894
+ "DELETE FROM cache_meta WHERE key IN (?, ?)",
895
+ ("claude_ingest_walk_complete",
896
+ _ingest_frontier.CLAUDE_WALK_ORPHAN_RESIDUAL_KEY),
897
+ )
898
+
899
+
874
900
  def _withdraw_cache_meta_marker(conn: sqlite3.Connection, key: str) -> None:
875
901
  """Remove a walk-completion marker, writing only when it is present.
876
902
 
@@ -1076,6 +1102,29 @@ class ProjectKey:
1076
1102
  is_no_git: bool = field(default=False, compare=False)
1077
1103
 
1078
1104
 
1105
+ def _is_repository_marker(marker: str) -> bool:
1106
+ """Whether a `.git` entry marks a repository for the project walk (#976).
1107
+
1108
+ A bounded structural filter, not a Git validity check: an absent entry (or
1109
+ a dangling symlink) is no marker; a `.git` DIRECTORY (a symlink to one
1110
+ included) counts only when it holds ``HEAD`` and the ``objects/`` and
1111
+ ``refs/`` directories, so a stray non-repository `.git/` in a worktree
1112
+ container is skipped; anything else (a worktree or submodule `gitdir:`
1113
+ file) counts by presence. Metadata calls only: nothing is opened and no
1114
+ `gitdir:` or `commondir` is followed, which keeps the walk inside the
1115
+ frozen-roots stat contract (`bench/write-attribution/frozen_roots.py`).
1116
+ Only the `git-root` walk uses this; `_is_dev_checkout()` and the other
1117
+ `.git` presence predicates keep their own installation-safety contract.
1118
+ """
1119
+ if not os.path.exists(marker):
1120
+ return False
1121
+ if os.path.isdir(marker):
1122
+ return (os.path.exists(os.path.join(marker, "HEAD"))
1123
+ and os.path.isdir(os.path.join(marker, "objects"))
1124
+ and os.path.isdir(os.path.join(marker, "refs")))
1125
+ return True
1126
+
1127
+
1079
1128
  def _resolve_project_key(
1080
1129
  project_path: str | None,
1081
1130
  mode: str, # "git-root" | "full-path"
@@ -1083,8 +1132,9 @@ def _resolve_project_key(
1083
1132
  ) -> ProjectKey:
1084
1133
  """Resolve a raw project_path to its ProjectKey.
1085
1134
 
1086
- Walks parents looking for `.git` (file or dir) to find the canonical
1087
- git-root. Non-git paths fall back to the normalized path. NULL input
1135
+ Walks parents looking for a repository marker (`_is_repository_marker`: a
1136
+ `.git` file, or a `.git` directory holding HEAD, objects/ and refs/) to
1137
+ find the canonical git-root. Non-git paths fall back to the normalized path. NULL input
1088
1138
  becomes a literal `(unknown)` bucket.
1089
1139
  """
1090
1140
  if project_path is None:
@@ -1136,7 +1186,7 @@ def _resolve_project_key(
1136
1186
  while True:
1137
1187
  if cur == home or cur == "/" or os.path.dirname(cur) == cur:
1138
1188
  break
1139
- if os.path.exists(os.path.join(cur, ".git")):
1189
+ if _is_repository_marker(os.path.join(cur, ".git")):
1140
1190
  key = ProjectKey(
1141
1191
  bucket_path=cur,
1142
1192
  display_key=os.path.basename(cur) or cur,
@@ -1708,12 +1758,18 @@ class CodexResetAnchorResolver:
1708
1758
  """
1709
1759
 
1710
1760
  def __init__(self, conn: sqlite3.Connection, *,
1711
- deadline: "float | None" = None, clock=None):
1761
+ deadline: "float | None" = None, clock=None,
1762
+ seed_source: "Callable[[tuple[str, str, str, object], str], _lib_quota.ResetAnchorComponents | None] | None" = None):
1712
1763
  self._conn = conn
1713
1764
  # #980 F3: a budgeted walk passes its walk deadline and clock, which
1714
1765
  # bound the group seed read; unbudgeted callers pass neither.
1715
1766
  self._deadline = deadline
1716
1767
  self._clock = clock if clock is not None else time.monotonic
1768
+ # #975 (spec 4.2): `seed_source(group, logical_limit_key)` supplies a
1769
+ # group's committed evidence in place of the seed SELECT whenever it
1770
+ # returns components (the provenance drain hands out a private copy of
1771
+ # its committed-evidence base); `None` falls back to the SELECT.
1772
+ self._seed_source = seed_source
1717
1773
  self._groups: "dict[tuple[str, str, str, object], _lib_quota.ResetAnchorComponents]" = {}
1718
1774
  self._pending_merges: list[
1719
1775
  tuple[tuple[str, str, str, object], str, dt.datetime,
@@ -1744,6 +1800,11 @@ class CodexResetAnchorResolver:
1744
1800
  components = self._groups.get(group)
1745
1801
  if components is not None:
1746
1802
  return components
1803
+ if self._seed_source is not None:
1804
+ components = self._seed_source(group, logical_limit_key)
1805
+ if components is not None:
1806
+ self._groups[group] = components
1807
+ return components
1747
1808
  components = _lib_quota.ResetAnchorComponents()
1748
1809
  if not self._seed_failed:
1749
1810
  candidates = _lib_jsonl.codex_snap_equivalent_limit_keys(
@@ -1922,6 +1983,219 @@ def _codex_anchor_iso(value: dt.datetime) -> str:
1922
1983
  return value.astimezone(dt.timezone.utc).isoformat().replace("+00:00", "Z")
1923
1984
 
1924
1985
 
1986
+ class _CodexWalkShare:
1987
+ """History-sized state shared across the admitted units of one pass.
1988
+
1989
+ #982 / #975 spec 4.2. One in-process object per heal-executor or
1990
+ provenance pass, never persisted. Every reuse is guarded by a token read
1991
+ under the writer locks: ``(st_dev, st_ino)`` of cache.db, ``PRAGMA
1992
+ schema_version``, ``quota_window_change_log``'s ``sqlite_sequence`` value
1993
+ and ``codex_physical_mutation_seq``. The ledger's three triggers record
1994
+ every insert, delete and seeded-column update of a Codex quota row, so any
1995
+ other writer's quota mutation between two units changes the token; while
1996
+ the ledger table or a trigger is missing the token is ``None`` and nothing
1997
+ is reused. A bind consumes the seal, so a unit that never sealed (an
1998
+ exception, a rollback, a refusal) leaves the next unit seeding afresh.
1999
+
2000
+ - ``bind_walk`` / ``seal``: the walk memo, one ``CodexResetAnchorResolver``
2001
+ shared across population units, i.e. the single walk's own resolver.
2002
+ - ``identity_resolver`` / ``absorb_identity`` / ``seal``: the provenance
2003
+ committed-evidence seed. Each identity resolves against copies of a
2004
+ per-group base equal to a fresh seed of the committed rows; after the
2005
+ identity commits, the base gains exactly that identity's stored rows.
2006
+ Components and anchors do not depend on insertion order, so base plus
2007
+ absorbed rows equals a fresh seed, and evidence an identity resolved but
2008
+ did not store dies with its copy, exactly as a new resolver per call.
2009
+ - ``ledger_paths``: the accounting ledger's path set, read in full once
2010
+ and then only past the last ``seq`` seen. A positive signal only (I7),
2011
+ so a path whose rows a prune later removed may stay in the set.
2012
+ - ``heal_payload``: the heal record's population and D1 set, parsed once
2013
+ per pass and reused while the fence key keeps the same ``gen`` (a
2014
+ payload is only ever written with a new ``gen``).
2015
+ """
2016
+
2017
+ _LEDGER_TRIGGERS = (
2018
+ "trg_qws_ledger_ins", "trg_qws_ledger_del", "trg_qws_ledger_upd")
2019
+
2020
+ def __init__(self) -> None:
2021
+ self._sealed: "tuple | None" = None
2022
+ self._walk: "CodexResetAnchorResolver | None" = None
2023
+ self._base: "CodexResetAnchorResolver | None" = None
2024
+ self._ledger: "set[str] | None" = None
2025
+ self._ledger_seq = 0
2026
+ self._ledger_identity: "tuple | None" = None
2027
+ self._heal_payload: "CodexHealPayload | None" = None
2028
+
2029
+ def heal_payload(self, conn: sqlite3.Connection,
2030
+ record: "dict | None") -> "CodexHealPayload | None":
2031
+ """The record's parsed payload: the one this pass already parsed
2032
+ while ``record`` keeps its ``gen``, otherwise parsed once now."""
2033
+ if record is None or record.get("malformed"):
2034
+ return None
2035
+ cached = self._heal_payload
2036
+ if cached is not None and cached.gen == record.get("gen"):
2037
+ return cached
2038
+ payload = load_codex_heal_payload(conn, record)
2039
+ if payload is not None:
2040
+ self._heal_payload = payload
2041
+ return payload
2042
+
2043
+ def remember_heal_payload(self, payload: "CodexHealPayload") -> None:
2044
+ """A payload this pass wrote itself (the history start unit)."""
2045
+ self._heal_payload = payload
2046
+
2047
+ def forget_heal_payload(self) -> None:
2048
+ """A pass that ended ``restart`` (spec 4.5a) keeps no payload."""
2049
+ self._heal_payload = None
2050
+
2051
+ @classmethod
2052
+ def token(cls, conn: sqlite3.Connection) -> "tuple | None":
2053
+ """The reuse token, or ``None`` when no reuse may be trusted."""
2054
+ from _cctally_quota import codex_physical_mutation_seq
2055
+ try:
2056
+ main = next((row[2] for row in conn.execute("PRAGMA database_list")
2057
+ if row[1] == "main"), None)
2058
+ if not main:
2059
+ return None
2060
+ st = os.stat(main)
2061
+ present = {row[0] for row in conn.execute(
2062
+ "SELECT name FROM sqlite_master "
2063
+ "WHERE (type='table' AND name='quota_window_change_log') "
2064
+ " OR (type='trigger' AND name IN (?,?,?))",
2065
+ cls._LEDGER_TRIGGERS)}
2066
+ if len(present) != 1 + len(cls._LEDGER_TRIGGERS):
2067
+ return None
2068
+ schema_version = int(
2069
+ conn.execute("PRAGMA schema_version").fetchone()[0])
2070
+ row = conn.execute(
2071
+ "SELECT seq FROM sqlite_sequence "
2072
+ "WHERE name='quota_window_change_log'").fetchone()
2073
+ ledger_seq = 0 if row is None else int(row[0])
2074
+ except (sqlite3.Error, OSError, TypeError, ValueError):
2075
+ return None
2076
+ return (st.st_dev, st.st_ino, schema_version, ledger_seq,
2077
+ codex_physical_mutation_seq(conn))
2078
+
2079
+ def _validate(self, conn: sqlite3.Connection) -> None:
2080
+ token = self.token(conn)
2081
+ if token is None or self._sealed is None or token != self._sealed:
2082
+ self._walk = None
2083
+ self._base = None
2084
+ self._sealed = None
2085
+
2086
+ def bind_walk(self, conn: sqlite3.Connection) -> CodexResetAnchorResolver:
2087
+ """The shared walk resolver rebound to ``conn`` (new on a token change)."""
2088
+ self._validate(conn)
2089
+ if self._walk is None:
2090
+ self._walk = CodexResetAnchorResolver(conn)
2091
+ else:
2092
+ self._walk._conn = conn
2093
+ self._walk._pending_merges.clear()
2094
+ return self._walk
2095
+
2096
+ def seal(self, conn: sqlite3.Connection) -> None:
2097
+ """Record the token after this unit's last commit, under its locks."""
2098
+ self._sealed = self.token(conn)
2099
+
2100
+ def invalidate(self) -> None:
2101
+ """Drop everything: the next unit seeds from the committed store."""
2102
+ self._sealed = None
2103
+ self._walk = None
2104
+ self._base = None
2105
+ self._ledger = None
2106
+ self._ledger_seq = 0
2107
+ self._ledger_identity = None
2108
+
2109
+ def identity_resolver(
2110
+ self, conn: sqlite3.Connection) -> CodexResetAnchorResolver:
2111
+ """A resolver for one provenance identity over copies of the base."""
2112
+ self._validate(conn)
2113
+ if self._base is None:
2114
+ self._base = CodexResetAnchorResolver(conn)
2115
+ else:
2116
+ self._base._conn = conn
2117
+ return CodexResetAnchorResolver(conn, seed_source=self._seed_copy)
2118
+
2119
+ def _seed_copy(self, group, logical_limit_key: str):
2120
+ base = self._base
2121
+ if base is None:
2122
+ return None
2123
+ components = base._components_for(group, logical_limit_key)
2124
+ if base._seed_failed:
2125
+ # Never remember a degraded seed; the identity's own resolver
2126
+ # then degrades exactly as a new resolver per call does.
2127
+ self._base = None
2128
+ return None
2129
+ return components.copy()
2130
+
2131
+ def absorb_identity(
2132
+ self, conn: sqlite3.Connection, source_path: str) -> None:
2133
+ """Add one committed identity's stored quota rows to the base.
2134
+
2135
+ Mirrors the seed SELECT's membership exactly: a row belongs to every
2136
+ seeded group with its root and slot whose snap-equivalent limit keys
2137
+ include the row's key (the seed does not filter on the window), with
2138
+ its real order key ``(source_path, line_offset, id)``. A group not yet
2139
+ seeded is skipped: its first seed reads these rows anyway.
2140
+ """
2141
+ base = self._base
2142
+ if base is None or not base._groups:
2143
+ return
2144
+ by_scope: "dict[tuple[str, str], list]" = {}
2145
+ for group in base._groups:
2146
+ by_scope.setdefault((group[0], group[1]), []).append(group)
2147
+ limit_keys: "dict[tuple, frozenset[str]]" = {}
2148
+ for (slot, limit_key, _window, root, path, offset, row_id,
2149
+ raw_text) in conn.execute(
2150
+ "SELECT observed_slot, logical_limit_key, window_minutes, "
2151
+ "source_root_key, source_path, line_offset, id, resets_at_utc "
2152
+ "FROM quota_window_snapshots "
2153
+ "WHERE source='codex' AND source_path=?", (str(source_path),)):
2154
+ groups = by_scope.get((root, slot))
2155
+ if not groups:
2156
+ continue
2157
+ raw = _parse_anchor_iso(raw_text)
2158
+ if raw is None:
2159
+ continue
2160
+ for group in groups:
2161
+ keys = limit_keys.get(group)
2162
+ if keys is None:
2163
+ keys = limit_keys[group] = frozenset(
2164
+ _lib_jsonl.codex_snap_equivalent_limit_keys(
2165
+ str(group[2])))
2166
+ if limit_key in keys:
2167
+ base._groups[group].add(
2168
+ raw, order_key=(str(path), int(offset), int(row_id)))
2169
+
2170
+ def ledger_paths(self, conn: sqlite3.Connection) -> "frozenset[str]":
2171
+ """``_codex_ledger_paths`` across units: one full read per pass, then
2172
+ only the ledger rows past the last ``seq`` seen. A cache.db file or
2173
+ ``schema_version`` change (or an unusable token) forces a full read."""
2174
+ token = self.token(conn)
2175
+ identity = None if token is None else token[:3]
2176
+ if (identity is None or self._ledger is None
2177
+ or identity != self._ledger_identity):
2178
+ rows = conn.execute(
2179
+ "SELECT seq, source_path FROM codex_accounting_change_log "
2180
+ "WHERE change_kind='path'").fetchall()
2181
+ self._ledger, self._ledger_seq = set(), 0
2182
+ self._ledger_identity = identity
2183
+ else:
2184
+ rows = conn.execute(
2185
+ "SELECT seq, source_path FROM codex_accounting_change_log "
2186
+ "WHERE change_kind='path' AND seq > ?",
2187
+ (self._ledger_seq,)).fetchall()
2188
+ for seq, path in rows:
2189
+ if path:
2190
+ self._ledger.add(path)
2191
+ if seq > self._ledger_seq:
2192
+ self._ledger_seq = seq
2193
+ paths = frozenset(self._ledger)
2194
+ if identity is None:
2195
+ self._ledger = None
2196
+ return paths
2197
+
2198
+
1925
2199
  def _codex_provider_roots() -> list[CodexProviderRoot]:
1926
2200
  """Return configured provider roots with their sessions/direct walk roots.
1927
2201
 
@@ -4258,14 +4532,26 @@ def _codex_barrier_seal_pending(conn, *, priority=(), max_keys=None,
4258
4532
 
4259
4533
  The caller holds the cache writer locks and has committed its own work.
4260
4534
  Reads only cache.db (no stat, no decode) and commits once.
4261
- """
4262
- if allow_init:
4263
- _codex_barrier_init_targets(conn)
4264
4535
 
4536
+ #982 spec 4.3 (seal fence): while a heal fence holds, nothing is sealed
4537
+ (census ``heal-fenced``, no scan needed, so no caller requests the worker
4538
+ for it): a per-file unit could otherwise seal a conversation whose
4539
+ cursorless population contributor is still unread, under a census set
4540
+ before the fence. Sealing resumes after the transition, whose piece sets
4541
+ the census from the whole walk's counters.
4542
+ """
4265
4543
  def remaining() -> int:
4266
4544
  return int(conn.execute(
4267
4545
  "SELECT COUNT(*) FROM codex_barrier_repairs").fetchone()[0])
4268
4546
 
4547
+ if _codex_heal_fence_active(conn):
4548
+ if conn.in_transaction:
4549
+ conn.commit()
4550
+ return BarrierSealResult(0, 0, remaining(), "heal-fenced", False,
4551
+ start_after)
4552
+ if allow_init:
4553
+ _codex_barrier_init_targets(conn)
4554
+
4269
4555
  if not _codex_barrier_census_proven(conn):
4270
4556
  if conn.in_transaction:
4271
4557
  conn.commit()
@@ -4334,7 +4620,10 @@ def _codex_barrier_after_walk(conn, stats, *, targeted: bool, deadline,
4334
4620
  set(stats.drain_requested) | {CODEX_REPLAY_DRAIN_COMMAND}))
4335
4621
 
4336
4622
 
4337
- def _codex_admitted_cache_write(fn, *, wait_for_requests: bool):
4623
+ def _codex_admitted_cache_write(
4624
+ fn, *, wait_for_requests: bool,
4625
+ after_commit: "Callable[[sqlite3.Connection], None] | None" = None,
4626
+ contention_bound: "float | None" = None):
4338
4627
  """Run ``fn(conn)`` through the drain gate under the cache lock domain.
4339
4628
 
4340
4629
  The store is opened BEFORE admission takes the cache flocks (review N5;
@@ -4347,10 +4636,20 @@ def _codex_admitted_cache_write(fn, *, wait_for_requests: bool):
4347
4636
  is busy". A connection opened for a refused admission is closed before
4348
4637
  the backoff, and the next attempt opens a fresh one.
4349
4638
 
4639
+ ``after_commit(conn)`` runs after ``fn``'s transaction committed and
4640
+ before ``adm.finish``, so it still holds both store locks: the provenance
4641
+ drain (#975, spec 4.2) refreshes its committed-evidence seed there, where
4642
+ no other writer can have interleaved. It does not run when ``fn`` raises.
4643
+
4644
+ ``contention_bound`` (a heal unit's, #982): store locks another writer
4645
+ holds continuously (no fresh hook request in between) for that many
4646
+ seconds raise ``_CodexHealContended``, as every other heal unit gives up.
4647
+
4350
4648
  Returns ``(True, result)``, or ``(False, None)`` when a fresh hook request
4351
4649
  made the caller yield and it may not wait.
4352
4650
  """
4353
4651
  import _lib_codex_drain_admission as adm
4652
+ contended_since: "float | None" = None
4354
4653
  while True:
4355
4654
  conn = open_cache_db()
4356
4655
  try:
@@ -4364,12 +4663,23 @@ def _codex_admitted_cache_write(fn, *, wait_for_requests: bool):
4364
4663
  conn.close()
4365
4664
  if not wait_for_requests:
4366
4665
  return False, None
4666
+ if contention_bound is not None:
4667
+ if adm.fresh_requests(_cctally_core.APP_DIR):
4668
+ contended_since = None
4669
+ else:
4670
+ now = time.monotonic()
4671
+ if contended_since is None:
4672
+ contended_since = now
4673
+ if now - contended_since >= contention_bound:
4674
+ raise _CodexHealContended()
4367
4675
  _drain_backoff(adm)
4368
4676
  try:
4369
4677
  try:
4370
4678
  result = fn(conn)
4371
4679
  if conn.in_transaction:
4372
4680
  conn.commit()
4681
+ if after_commit is not None:
4682
+ after_commit(conn)
4373
4683
  return True, result
4374
4684
  except BaseException:
4375
4685
  conn.rollback()
@@ -4608,9 +4918,14 @@ def _delete_codex_file_derived_rows(
4608
4918
  "DELETE FROM codex_session_entries WHERE source_path = ?" + root_clause,
4609
4919
  params,
4610
4920
  )
4921
+ # #971 spec D3: ``+`` keeps the root term out of index selection, so the
4922
+ # statement seeks the UNIQUE (source, source_path, ...) autoindex instead
4923
+ # of walking every quota row of the root through idx_qws_codex_load_order.
4924
+ quota_root_clause = (
4925
+ " AND +source_root_key IS ?" if match_source_root else "")
4611
4926
  conn.execute(
4612
4927
  "DELETE FROM quota_window_snapshots WHERE source = 'codex' "
4613
- "AND source_path = ?" + root_clause,
4928
+ "AND source_path = ?" + quota_root_clause,
4614
4929
  params,
4615
4930
  )
4616
4931
  conn.execute(
@@ -4697,6 +5012,10 @@ def _prune_codex_accounting_change_log(
4697
5012
  if retain_sequences < 1:
4698
5013
  raise ValueError("retain_sequences must be positive")
4699
5014
  try:
5015
+ if _codex_heal_fence_active(conn):
5016
+ # #982 spec 4.3: the heal's shared ledger path set reads these
5017
+ # rows; a due prune stays owed for the executor after the fence.
5018
+ return 0
4700
5019
  cutoff = _codex_change_log_prune_cutoff(conn, retain_sequences)
4701
5020
  if cutoff is None:
4702
5021
  return 0
@@ -4970,16 +5289,14 @@ COVERAGE_WRITER_ACTIONS: "dict[str, str]" = {
4970
5289
  "_cctally_journal.rehydrate_codex_journal_families": "preserve",
4971
5290
  # Spec §4.3's migrations row, enumerated rather than named — and enumerated
4972
5291
  # by the action each one TAKES, not by the action the spec's one-line row
4973
- # assumed. Only `_024` deletes rows the journal still retains, and it is the
4974
- # only one that invalidates. `_038_codex_session_files_ingest_complete` is
4975
- # deliberately ABSENT: the review that asked for this row listed it, but it
4976
- # adds a column to `codex_session_files`, which is not a covered family —
4977
- # which is why the guard scans rather than trusts a hand list.
5292
+ # assumed. No migration deletes rows the journal still retains: since #983
5293
+ # 024 only arms the guarded byte-zero replay, whose clear is the
5294
+ # `_clear_codex_derived_rows` entry above, so it writes no covered family
5295
+ # and is absent. `_038_codex_session_files_ingest_complete` is deliberately
5296
+ # ABSENT too: the review that asked for this row listed it, but it adds a
5297
+ # column to `codex_session_files`, which is not a covered family — which is
5298
+ # why the guard scans rather than trusts a hand list.
4978
5299
  #
4979
- # `DELETE FROM quota_window_snapshots WHERE source = 'codex'` with NO
4980
- # sequence bump. Left standing, the certificate would be stale-VALID over a
4981
- # cache holding zero Codex quota rows.
4982
- "_cctally_db._024_codex_fused_ingest_rebuild": "invalidate",
4983
5300
  # `UPDATE quota_window_snapshots SET observed_model = ?`, and it bumps
4984
5301
  # `codex_physical_mutation_seq` in the same transaction whenever it changes
4985
5302
  # a row. The certificate is left standing and goes invalid on the SEQUENCE
@@ -5222,6 +5539,11 @@ def _codex_conversation_project_attribution(
5222
5539
  label, else git identity, else unassigned) so the browse live-recompute
5223
5540
  fallback matches the stored rollup. Degrades to (None, None) when the S3
5224
5541
  kernel or a valid source root is unavailable — never guesses.
5542
+
5543
+ Each call resolves with a fresh `_resolve_project_key` memo, so the answer
5544
+ never depends on which conversation resolved first: the no-git fallback
5545
+ memoizes the first raw spelling's label under the normalized path, which
5546
+ would hand one symlink alias's basename to another (#972 §5.8).
5225
5547
  """
5226
5548
  if not source_root_key:
5227
5549
  return None, None
@@ -5323,12 +5645,115 @@ def _next_conversation_render_revision(conn: sqlite3.Connection) -> int:
5323
5645
  return revision
5324
5646
 
5325
5647
 
5648
+ def _codex_first_row_root(
5649
+ conn: sqlite3.Connection, conversation_key: str,
5650
+ ) -> "str | None":
5651
+ """The source root of a conversation's first normalized row, in the
5652
+ writer's order (#972 §5.3), without loading any message body.
5653
+
5654
+ Reads ``(timestamp_utc, id)`` index order and stops at the end of the
5655
+ first timestamp tie run, which `in_physical_order` then orders exactly as
5656
+ `_load_codex_normalized_rows` orders it. None when there is no row."""
5657
+ run: list = []
5658
+ cursor = conn.execute(
5659
+ "SELECT source_root_key, timestamp_utc, source_path, line_offset "
5660
+ "FROM codex_conversation_messages WHERE conversation_key=? "
5661
+ "ORDER BY timestamp_utc, id",
5662
+ (conversation_key,),
5663
+ )
5664
+ try:
5665
+ for row in cursor:
5666
+ if run and row[1] != run[0][1]:
5667
+ break
5668
+ run.append(row)
5669
+ finally:
5670
+ cursor.close()
5671
+ if not run:
5672
+ return None
5673
+ return _lib_codex_conversation.in_physical_order(
5674
+ run, timestamp=lambda row: row[1],
5675
+ position=lambda row: (row[2], row[3]))[0][0]
5676
+
5677
+
5678
+ def _codex_rollup_thread_fields(
5679
+ conn: sqlite3.Connection,
5680
+ conversation_key: str,
5681
+ *,
5682
+ first_row_root: "Callable[[], str | None]",
5683
+ attribution_cache: "dict | None" = None,
5684
+ require_thread: bool = False,
5685
+ ) -> "tuple[str | None, str | None, str | None, str | None] | None":
5686
+ """``(source_root_key, parent_thread_id, project_key, project_label)`` of
5687
+ one conversation's rollup, from its thread row (#972 §5.3).
5688
+
5689
+ The one derivation the rollup writer and the per-pass reconcile share, so
5690
+ the reconcile detects exactly what the writer would write. The thread
5691
+ root wins when non-empty; otherwise ``first_row_root()`` supplies the
5692
+ first normalized row's root (called lazily, only then).
5693
+
5694
+ ``attribution_cache`` (#972 §5.8 rule 3) memoizes the project attribution
5695
+ per exact ``(source_root_key, cwd, git_json)`` input; each entry is still
5696
+ computed with a fresh resolver memo, so sharing it across conversations
5697
+ never makes one answer depend on another. With ``require_thread`` a
5698
+ conversation with no thread row answers None instead (rule 2: a missing
5699
+ thread row is no evidence for the reconcile)."""
5700
+ # #845 §4.7. The two columns are read as BLOBs and decoded field by
5701
+ # field, so one undecodable byte no longer raises here and no longer
5702
+ # fails the whole recompute. The table name stays UNQUALIFIED, because
5703
+ # this writer must honor account scoping: under
5704
+ # `scope_conversations_db_to_account` it sees the empty TEMP view on
5705
+ # purpose, and under #968's barrier the last-good view.
5706
+ thread = conn.execute(
5707
+ "SELECT CAST(cwd AS BLOB) AS cwd_blob, "
5708
+ "CAST(git_json AS BLOB) AS git_json_blob, parent_thread_id, "
5709
+ "source_root_key "
5710
+ "FROM codex_conversation_threads WHERE conversation_key = ?",
5711
+ (conversation_key,),
5712
+ ).fetchone()
5713
+ if thread is None and require_thread:
5714
+ return None
5715
+ cwd = git_json = parent_thread_id = thread_root = None
5716
+ thread_malformed = False
5717
+ if thread is not None:
5718
+ cwd_blob, git_json_blob, parent_thread_id, thread_root = thread
5719
+ decoded = _lib_codex_metadata.decode_codex_project_metadata(
5720
+ cwd_blob, git_json_blob)
5721
+ cwd, git_json = decoded.cwd, decoded.git_json
5722
+ # The DIRECT-ONLY predicate: this writer resolves the thread's own
5723
+ # fields through `_codex_conversation_project_attribution`, which
5724
+ # stops at a usable `cwd`, so a valid `cwd` beside an undecodable
5725
+ # `git_json` is still attributed.
5726
+ thread_malformed = _lib_codex_metadata.codex_metadata_is_malformed(
5727
+ decoded, None)
5728
+ source_root_key = thread_root if thread_root else first_row_root()
5729
+ if thread_malformed:
5730
+ # `(unassigned)` is a real answer for a thread with no metadata.
5731
+ # Persisting it for a thread whose metadata could not be READ would
5732
+ # stamp a wrong answer that every later read prefers, which is the
5733
+ # failure class `docs/codex-gotchas.md` records for the materialized
5734
+ # `"(unassigned)"`. NULL is the absence every reader already
5735
+ # handles.
5736
+ return source_root_key, parent_thread_id, None, None
5737
+ if attribution_cache is None:
5738
+ project_key, project_label = _codex_conversation_project_attribution(
5739
+ source_root_key, cwd, git_json)
5740
+ else:
5741
+ inputs = (source_root_key, cwd, git_json)
5742
+ attribution = attribution_cache.get(inputs)
5743
+ if attribution is None:
5744
+ attribution = _codex_conversation_project_attribution(*inputs)
5745
+ attribution_cache[inputs] = attribution
5746
+ project_key, project_label = attribution
5747
+ return source_root_key, parent_thread_id, project_key, project_label
5748
+
5749
+
5326
5750
  def _recompute_codex_rollups(
5327
5751
  conn: sqlite3.Connection,
5328
5752
  conversation_keys,
5329
5753
  *,
5330
5754
  advance_render_revision: bool = True,
5331
5755
  inputs_changed: "bool | set[str] | frozenset[str]" = True,
5756
+ attribution_cache: "dict | None" = None,
5332
5757
  ) -> None:
5333
5758
  """Recompute-affected-or-delete the rollup for each conversation (§3.2).
5334
5759
 
@@ -5346,9 +5771,16 @@ def _recompute_codex_rollups(
5346
5771
  outline memo keyed on it is never stale. Any other conversation's row is
5347
5772
  written only when one of its stored values would change, so a true no-op
5348
5773
  writes nothing (and reserves no revision).
5774
+
5775
+ #972: the thread-derived fields come from `_codex_rollup_thread_fields`,
5776
+ the derivation the per-pass reconcile shares; ``attribution_cache`` is the
5777
+ caller's per-pass attribution memo (none by default: every conversation
5778
+ resolves afresh). §5.8 rule 5: every key's outcome is derived before the
5779
+ first write, because the first write takes the conversations write lock
5780
+ and a key's derivation reads only its own rows and stored rollup.
5349
5781
  """
5350
5782
  kern = _lib_codex_conversation
5351
- keys = {key for key in conversation_keys if key}
5783
+ keys = sorted({key for key in conversation_keys if key})
5352
5784
  if not keys:
5353
5785
  return
5354
5786
  reserved_revision: "int | None" = None
@@ -5361,13 +5793,12 @@ def _recompute_codex_rollups(
5361
5793
  reserved_revision = _next_conversation_render_revision(conn)
5362
5794
  return reserved_revision
5363
5795
 
5796
+ # (conversation_key, None) deletes; (conversation_key, values) upserts.
5797
+ outcomes: list = []
5364
5798
  for conversation_key in keys:
5365
5799
  rows = _load_codex_normalized_rows(conn, conversation_key)
5366
5800
  if not rows:
5367
- conn.execute(
5368
- "DELETE FROM codex_conversation_rollups WHERE conversation_key = ?",
5369
- (conversation_key,),
5370
- )
5801
+ outcomes.append((conversation_key, None))
5371
5802
  continue
5372
5803
  item_count = kern.rollup_item_count(rows)
5373
5804
  title = kern.derive_title(rows)
@@ -5376,46 +5807,11 @@ def _recompute_codex_rollups(
5376
5807
  last_activity = max(timestamps) if timestamps else None
5377
5808
  models = sorted({r.model for r in rows if r.model})
5378
5809
  models_json = json.dumps(models) if models else None
5379
- source_root_key = rows[0].source_root_key
5380
- # #845 §4.7. The two columns are read as BLOBs and decoded field by
5381
- # field, so one undecodable byte no longer raises here and no longer
5382
- # fails the whole recompute. The table name stays UNQUALIFIED, because
5383
- # this writer must honor account scoping: under
5384
- # `scope_conversations_db_to_account` it sees the empty TEMP view on
5385
- # purpose.
5386
- thread = conn.execute(
5387
- "SELECT CAST(cwd AS BLOB) AS cwd_blob, "
5388
- "CAST(git_json AS BLOB) AS git_json_blob, parent_thread_id, "
5389
- "source_root_key "
5390
- "FROM codex_conversation_threads WHERE conversation_key = ?",
5391
- (conversation_key,),
5392
- ).fetchone()
5393
- cwd = git_json = parent_thread_id = None
5394
- thread_malformed = False
5395
- if thread is not None:
5396
- cwd_blob, git_json_blob, parent_thread_id, thread_root = thread
5397
- decoded = _lib_codex_metadata.decode_codex_project_metadata(
5398
- cwd_blob, git_json_blob)
5399
- cwd, git_json = decoded.cwd, decoded.git_json
5400
- # The DIRECT-ONLY predicate: this writer resolves the thread's own
5401
- # fields through `_codex_conversation_project_attribution`, which
5402
- # stops at a usable `cwd`, so a valid `cwd` beside an undecodable
5403
- # `git_json` is still attributed.
5404
- thread_malformed = _lib_codex_metadata.codex_metadata_is_malformed(
5405
- decoded, None)
5406
- if thread_root:
5407
- source_root_key = thread_root
5408
- if thread_malformed:
5409
- # `(unassigned)` is a real answer for a thread with no metadata.
5410
- # Persisting it for a thread whose metadata could not be READ would
5411
- # stamp a wrong answer that every later read prefers, which is the
5412
- # failure class `docs/codex-gotchas.md` records for the materialized
5413
- # `"(unassigned)"`. NULL is the absence every reader already
5414
- # handles.
5415
- project_key = project_label = None
5416
- else:
5417
- project_key, project_label = _codex_conversation_project_attribution(
5418
- source_root_key, cwd, git_json)
5810
+ first_root = rows[0].source_root_key
5811
+ source_root_key, parent_thread_id, project_key, project_label = (
5812
+ _codex_rollup_thread_fields(
5813
+ conn, conversation_key, first_row_root=lambda: first_root,
5814
+ attribution_cache=attribution_cache))
5419
5815
  values = (source_root_key, parent_thread_id, item_count, started,
5420
5816
  last_activity, project_key, project_label, models_json, title)
5421
5817
  if inputs_changed is not True and conversation_key not in (
@@ -5429,6 +5825,14 @@ def _recompute_codex_rollups(
5429
5825
  ).fetchone()
5430
5826
  if stored is not None and tuple(stored) == values:
5431
5827
  continue
5828
+ outcomes.append((conversation_key, values))
5829
+ for conversation_key, values in outcomes:
5830
+ if values is None:
5831
+ conn.execute(
5832
+ "DELETE FROM codex_conversation_rollups WHERE conversation_key = ?",
5833
+ (conversation_key,),
5834
+ )
5835
+ continue
5432
5836
  conn.execute(
5433
5837
  "INSERT INTO codex_conversation_rollups "
5434
5838
  "(conversation_key, source_root_key, parent_thread_id, item_count, "
@@ -5446,6 +5850,254 @@ def _recompute_codex_rollups(
5446
5850
  )
5447
5851
 
5448
5852
 
5853
+ #: #972 §5.4: the most conversations one reconcile transaction rewrites.
5854
+ CODEX_ROLLUP_RECONCILE_BATCH = 64
5855
+
5856
+
5857
+ def _reconcile_codex_rollup_attribution(
5858
+ conn: sqlite3.Connection, stats: "CodexIngestStats",
5859
+ ) -> int:
5860
+ """Rewrite every rollup whose thread-derived fields drifted (#972 §5.4).
5861
+
5862
+ A rollup is materialized when the conversation pass writes it, from the
5863
+ thread metadata and filesystem of that moment. Neither a metadata-only
5864
+ contributor that accounting accepts later (the R4 interleave) nor a
5865
+ worktree removed later rewrites it, so every full pass compares each
5866
+ persisted rollup with what `_codex_rollup_thread_fields` derives now and
5867
+ rewrites the ones that differ through the writer itself.
5868
+
5869
+ Detection runs with no transaction open over a materialized list of
5870
+ `main.codex_conversation_rollups`; repair runs the sorted drifted keys in
5871
+ batches of at most `CODEX_ROLLUP_RECONCILE_BATCH` through
5872
+ `_reconcile_codex_rollup_batch`, each in its own explicit transaction
5873
+ (one snapshot of both stores per batch, one render revision per written
5874
+ batch). A database error rolls back its batch and propagates; earlier
5875
+ batches stay committed and the next pass detects the rest. Never on an
5876
+ account-scoped connection, whose TEMP rollups are a view of the
5877
+ persisted ones. Returns the conversations written, which it also adds to
5878
+ ``stats.rollups_reattributed``.
5879
+
5880
+ Admission (§5.8, revisions 5 and 6) is per conversation and shared by
5881
+ detection and every batch (`_admit_codex_rollup_reconcile`): a
5882
+ conversation is compared only once accounting covers, from the same file
5883
+ generation, every `session_meta` line the conversation pass consumed for
5884
+ it (rule 1; a stale cursor of a replaced or truncated file is not
5885
+ coverage, `_codex_rollup_reconcile_uncovered`), never when it has no thread
5886
+ row (rule 2), and a non-database error deriving one rollup skips it
5887
+ (``stats.rollups_reconcile_skipped``, rule 4). No global condition, such
5888
+ as unrelated prune debt or a pending verification, gates the pass. One
5889
+ attribution cache keyed by the raw thread inputs serves the whole pass,
5890
+ and detection reads the rollups in key order (rule 3).
5891
+ """
5892
+ if conn.execute(
5893
+ "SELECT 1 FROM sqlite_temp_master "
5894
+ "WHERE type='table' AND name='_conversation_account_scope'"
5895
+ ).fetchone() is not None:
5896
+ raise RuntimeError(
5897
+ "the rollup reconcile never runs on an account-scoped connection")
5898
+ if conn.in_transaction:
5899
+ raise RuntimeError("the rollup reconcile needs no open transaction")
5900
+ cursor = conn.execute(
5901
+ "SELECT conversation_key, source_root_key, parent_thread_id, "
5902
+ "project_key, project_label FROM main.codex_conversation_rollups "
5903
+ "ORDER BY conversation_key")
5904
+ try:
5905
+ stored = cursor.fetchall()
5906
+ finally:
5907
+ cursor.close()
5908
+ attributions: dict = {}
5909
+ drifted = _admit_codex_rollup_reconcile(conn, stored, stats, attributions)
5910
+ drifted.sort()
5911
+ written = 0
5912
+ for start in range(0, len(drifted), CODEX_ROLLUP_RECONCILE_BATCH):
5913
+ changed = _reconcile_codex_rollup_batch(
5914
+ conn, drifted[start:start + CODEX_ROLLUP_RECONCILE_BATCH],
5915
+ stats, attributions)
5916
+ written += changed
5917
+ stats.rollups_reattributed += changed
5918
+ return written
5919
+
5920
+
5921
+ def _codex_rollup_reconcile_uncovered(
5922
+ conn: sqlite3.Connection, keys: "list[str] | None" = None,
5923
+ ) -> "set[str]":
5924
+ """Conversations whose consumed `session_meta` accounting has not yet
5925
+ processed from the same file generation (#972 §5.8 rule 1, revision 7).
5926
+
5927
+ A conversation is covered when, for every `(source_path, line_offset)`
5928
+ of its `session_meta` events, (a) `cache_db.codex_session_files` holds
5929
+ that path with `last_byte_offset` beyond that line, and (b) that
5930
+ accounting cursor and the conversation cursor for the same path
5931
+ (`main.codex_conversation_source_files`) agree on `size_bytes` and, when
5932
+ both describe the same physical container (equal `container_kind` and
5933
+ `container_path`), show no inode disagreement under
5934
+ `_lib_ingest_frontier.source_identity_replaced` (the inode decides; a
5935
+ missing or non-integer inode on either side is no evidence).
5936
+
5937
+ A path and an offset are coordinates, not generation evidence (VC-1):
5938
+ the conversation walk resets its cursor and the file's retained events
5939
+ when a different file now occupies the pathname, or when the file
5940
+ shrank, while an accounting walk that has not reached the file yet still
5941
+ holds the earlier incarnation's larger offset; comparing coordinates
5942
+ alone let the reconcile write the other contributor's project from a
5943
+ thread row that never saw the new metadata.
5944
+
5945
+ Only legs both walks converge on for every quiet file are compared, so
5946
+ the rule delays a repair while one walk lags and never withholds a
5947
+ settled conversation forever. Each walk adopts a verified container into
5948
+ its own cursor on its own schedule, so one cursor can describe the
5949
+ adopted `.zst` (its inode) for days while the other still describes the
5950
+ plain file it replaced (26 rows of 9 compressed conversations on a real
5951
+ store, 2026-11-05): inodes of different containers are not compared.
5952
+ Content anchors are not compared at all: after a partial read followed
5953
+ by an in-place shrink the two walks settle at one size on one inode with
5954
+ different anchors. The residual (spec §8, R9): a same-size in-place
5955
+ rewrite, or a same-size replacement under a cursor with no inode, while
5956
+ accounting has not re-read the file can write a project from the earlier
5957
+ generation, which the next full pass after accounting reads the file
5958
+ corrects. Decoded sizes, the logical identity (`_codex_identity_path`)
5959
+ and decoded offsets are shared by both walks for plain, compressed and
5960
+ archived rollouts alike. A conversation with no `session_meta` event is
5961
+ covered: there is nothing for accounting to lag behind.
5962
+
5963
+ One set-based read over the `session_meta` partial index
5964
+ (`idx_codex_events_session_meta_provenance`) joined to both cursors by
5965
+ path: never one query per conversation through the conversation-key
5966
+ index, which loads every event of each conversation (89.6 s for 1,251
5967
+ conversations on a real store, against 0.014 s for this read). For the
5968
+ same reason the statement constrains `conversation_key` only through a
5969
+ unary ``+`` and leaves de-duplication and NULL keys to Python: a
5970
+ ``DISTINCT`` or an ``IS NOT NULL`` there makes SQLite prefer a scan of
5971
+ every event through that index. The statement returns only the rows
5972
+ that fail a leg SQL decides exactly (``hard``) or whose same-container
5973
+ inodes SQL sees as different; the inode verdict itself is the shared
5974
+ Python rule, so this read cannot drift from the walks' replacement test.
5975
+ """
5976
+ hard = (
5977
+ "f.path IS NULL OR c.path IS NULL "
5978
+ "OR f.last_byte_offset <= e.line_offset "
5979
+ "OR f.size_bytes IS NOT c.size_bytes")
5980
+ same_container = (
5981
+ "f.container_kind IS c.container_kind "
5982
+ "AND f.container_path IS c.container_path")
5983
+ sql = (
5984
+ f"SELECT e.conversation_key, CASE WHEN {hard} THEN 1 ELSE 0 END, "
5985
+ "f.device_id, f.inode, c.device_id, c.inode "
5986
+ "FROM main.codex_conversation_events AS e "
5987
+ "LEFT JOIN cache_db.codex_session_files AS f ON f.path = e.source_path "
5988
+ # A conversation cursor exists beside its retained events: the walk
5989
+ # writes a file's events and its cursor in one savepoint, and the
5990
+ # prune drain deletes them together. Should one be missing anyway,
5991
+ # there is no generation to compare, so the line is uncovered.
5992
+ "LEFT JOIN main.codex_conversation_source_files AS c "
5993
+ "ON c.path = e.source_path "
5994
+ f"WHERE e.record_type = 'session_meta' AND ({hard} "
5995
+ f"OR ({same_container} AND f.inode IS NOT NULL "
5996
+ "AND c.inode IS NOT NULL AND f.inode IS NOT c.inode))")
5997
+ params: "tuple | list" = ()
5998
+ if keys is not None:
5999
+ if not keys:
6000
+ return set()
6001
+ sql += (" AND +e.conversation_key IN ("
6002
+ + ",".join("?" * len(keys)) + ")")
6003
+ params = keys
6004
+ uncovered: "set[str]" = set()
6005
+ for key, is_hard, f_device, f_inode, c_device, c_inode in conn.execute(
6006
+ sql, params):
6007
+ if key is None or key in uncovered:
6008
+ continue
6009
+ if is_hard or _ingest_frontier.source_identity_replaced(
6010
+ f_device, f_inode, c_device, c_inode):
6011
+ uncovered.add(key)
6012
+ return uncovered
6013
+
6014
+
6015
+ def _admit_codex_rollup_reconcile(
6016
+ conn: sqlite3.Connection, stored: list,
6017
+ stats: "CodexIngestStats", attributions: dict,
6018
+ *, keys: "list[str] | None" = None,
6019
+ ) -> "list[str]":
6020
+ """The drifted, admitted keys among ``stored`` rows (#972 §5.8 rules 1,
6021
+ 2, 4 and the drift comparison), read on the caller's snapshot.
6022
+
6023
+ ``stored`` holds ``(conversation_key, source_root_key,
6024
+ parent_thread_id, project_key, project_label)`` rows; ``keys`` limits
6025
+ the coverage read to a batch's keys (None: every conversation), which
6026
+ compares both stores' cursors for the generation inside the caller's
6027
+ snapshot. The one admission detection and every repair batch share, so
6028
+ they cannot diverge. Fills ``attributions`` for each admitted
6029
+ derivation."""
6030
+ uncovered = _codex_rollup_reconcile_uncovered(conn, keys)
6031
+ drifted = []
6032
+ for conversation_key, *values in stored:
6033
+ if conversation_key in uncovered:
6034
+ continue
6035
+ try:
6036
+ expected = _codex_rollup_thread_fields(
6037
+ conn, conversation_key,
6038
+ first_row_root=lambda key=conversation_key: (
6039
+ _codex_first_row_root(conn, key)),
6040
+ attribution_cache=attributions, require_thread=True)
6041
+ except (ValueError, OSError, UnicodeError):
6042
+ # e.g. a cwd with an embedded NUL, which `realpath` refuses: this
6043
+ # rollup keeps its stored fields for the pass.
6044
+ stats.rollups_reconcile_skipped += 1
6045
+ continue
6046
+ if expected is not None and expected != tuple(values):
6047
+ drifted.append(conversation_key)
6048
+ return drifted
6049
+
6050
+
6051
+ def _reconcile_codex_rollup_batch(
6052
+ conn: sqlite3.Connection, batch: "list[str]",
6053
+ stats: "CodexIngestStats", attributions: dict,
6054
+ ) -> int:
6055
+ """Repair one batch of detected keys in one transaction (#972 §5.4,
6056
+ §5.8 rule 8); returns the conversations written.
6057
+
6058
+ Accounting writes under its own locks, so a thread row or cursor can
6059
+ change between detection and this batch. After ``BEGIN`` the batch
6060
+ re-admits its keys on the transaction's snapshot of both stores, which
6061
+ fills ``attributions``, and hands only the admitted, still-drifted keys to
6062
+ the writer; the writer reads the same snapshot and takes each attribution
6063
+ from the cache, so it writes nothing the admission did not see and makes
6064
+ no new filesystem call for those keys. A batch with no surviving key
6065
+ ends its read-only transaction and reserves no render revision."""
6066
+ if conn.in_transaction:
6067
+ raise RuntimeError("the rollup reconcile needs no open transaction")
6068
+ marks = ",".join("?" * len(batch))
6069
+ conn.execute("BEGIN")
6070
+ try:
6071
+ rows = conn.execute(
6072
+ "SELECT conversation_key, source_root_key, parent_thread_id, "
6073
+ "project_key, project_label, render_revision "
6074
+ "FROM main.codex_conversation_rollups "
6075
+ f"WHERE conversation_key IN ({marks}) ORDER BY conversation_key",
6076
+ batch).fetchall()
6077
+ before = {row[0]: row[5] for row in rows}
6078
+ admitted = _admit_codex_rollup_reconcile(
6079
+ conn, [row[:5] for row in rows], stats, attributions,
6080
+ keys=list(batch))
6081
+ if not admitted:
6082
+ conn.rollback()
6083
+ return 0
6084
+ _recompute_codex_rollups(
6085
+ conn, admitted, inputs_changed=set(),
6086
+ attribution_cache=attributions)
6087
+ after = dict(conn.execute(
6088
+ "SELECT conversation_key, render_revision "
6089
+ "FROM main.codex_conversation_rollups "
6090
+ f"WHERE conversation_key IN ({','.join('?' * len(admitted))})",
6091
+ admitted).fetchall())
6092
+ conn.commit()
6093
+ except BaseException:
6094
+ if conn.in_transaction:
6095
+ conn.rollback()
6096
+ raise
6097
+ _wal_checkpoint.after_commit(conn)
6098
+ return sum(1 for key in admitted if before.get(key) != after.get(key))
6099
+
6100
+
5449
6101
  def _replay_codex_normalization(
5450
6102
  conn: sqlite3.Connection,
5451
6103
  *,
@@ -5715,8 +6367,11 @@ def _codex_identity_has_rows(
5715
6367
  "WHERE path = ? AND source_root_key IS ? LIMIT 1",
5716
6368
  "SELECT 1 FROM codex_session_entries "
5717
6369
  "WHERE source_path = ? AND source_root_key IS ? LIMIT 1",
6370
+ # #971: ``+`` pins the quota probe to the UNIQUE (source, source_path,
6371
+ # ...) autoindex, as in _delete_codex_file_derived_rows; on the root
6372
+ # index an absent identity walked every quota row of the root.
5718
6373
  "SELECT 1 FROM quota_window_snapshots WHERE source = 'codex' "
5719
- "AND source_path = ? AND source_root_key IS ? LIMIT 1",
6374
+ "AND source_path = ? AND +source_root_key IS ? LIMIT 1",
5720
6375
  "SELECT 1 FROM codex_conversation_threads "
5721
6376
  "WHERE source_path = ? AND source_root_key IS ? LIMIT 1",
5722
6377
  )
@@ -6350,9 +7005,14 @@ def _release_cache_store_locks(token) -> None:
6350
7005
  def _drain_backoff(adm) -> None:
6351
7006
  """After a refused admission: wait out every fresh request; when none was
6352
7007
  fresh (the gate or a store lock was busy), sleep one poll step so a
6353
- waiting worker never spins on the gate."""
7008
+ waiting worker never spins on the gate. The step is short when this
7009
+ process observed a fresh Claude waiter within the last
7010
+ ``CLAUDE_WAITER_RECENT_SECONDS`` (#1055 spec 4.3: the waiter can vanish
7011
+ between the refusal and this check); the hint selects the interval only."""
6354
7012
  if adm.fresh_requests(_cctally_core.APP_DIR):
6355
7013
  adm.wait_until_no_fresh_requests(_cctally_core.APP_DIR)
7014
+ elif adm.claude_waiter_recently_observed(_cctally_core.APP_DIR):
7015
+ adm._sleep(adm.SHORT_POLL_SECONDS)
6356
7016
  else:
6357
7017
  adm._sleep(adm.POLL_SECONDS)
6358
7018
 
@@ -7479,6 +8139,10 @@ _CODEX_W11_EAGER_PREPARE_FOR_TESTS = False
7479
8139
  _CODEX_W11_TRACE = None
7480
8140
  #: Test seam: called immediately before each W11 commit (crash injection).
7481
8141
  _CODEX_BATCH_PRECOMMIT_HOOK = None
8142
+ #: Test seam (#982 R1): called inside the locked Codex sync immediately before
8143
+ #: its own store-lock acquisition, i.e. after a heal unit's ``admit()`` has
8144
+ #: returned and before the sync's ``_after_locks`` recheck.
8145
+ _CODEX_PRE_STORE_LOCK_HOOK = None
7482
8146
  _CODEX_W11_FILE_SAVEPOINT = "SAVEPOINT codex_w11_file"
7483
8147
  _CODEX_W11_FILE_RELEASE = "RELEASE SAVEPOINT codex_w11_file"
7484
8148
  _CODEX_W11_FILE_ROLLBACK_TO = "ROLLBACK TO SAVEPOINT codex_w11_file"
@@ -7646,9 +8310,14 @@ def _write_codex_file_batch(
7646
8310
  container: "CodexContainerAcceptance | None" = None,
7647
8311
  barrier: "CodexBarrierRead | None" = None,
7648
8312
  commit: bool = True,
8313
+ thread_guard: "frozenset[str]" = frozenset(),
7649
8314
  ) -> int:
7650
8315
  """Write one fully-buffered Codex file atomically and return entry changes.
7651
8316
 
8317
+ ``thread_guard`` (#982 spec 4.3) is the fenced heal population, passed by
8318
+ a sync without the heal executor's share: a thread row whose current
8319
+ ``source_path`` is a population file is left for the executor's walk.
8320
+
7652
8321
  ``commit=False`` (#901 W11) leaves the caller's transaction open: an
7653
8322
  unbudgeted sync applies each file inside its own savepoint and commits the
7654
8323
  batch once.
@@ -7732,6 +8401,19 @@ def _write_codex_file_batch(
7732
8401
  VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)""",
7733
8402
  quota_rows,
7734
8403
  )
8404
+ if thread_rows and thread_guard:
8405
+ # The upsert overwrites every metadata column, so a conversation's row
8406
+ # is its last ingested contributor's. While the heal holds its
8407
+ # population, a row a population file owns is the single walk's to
8408
+ # replace (the timeline in which this write came before the walk).
8409
+ kept = []
8410
+ for row in thread_rows:
8411
+ current = conn.execute(
8412
+ "SELECT source_path FROM codex_conversation_threads "
8413
+ "WHERE conversation_key=?", (row[0],)).fetchone()
8414
+ if current is None or current[0] not in thread_guard:
8415
+ kept.append(row)
8416
+ thread_rows = kept
7735
8417
  if thread_rows:
7736
8418
  conn.executemany(
7737
8419
  """INSERT INTO codex_conversation_threads
@@ -7929,6 +8611,13 @@ class IngestStats:
7929
8611
  assistant_lines_skipped: int = 0
7930
8612
  skip_reasons: dict = field(default_factory=dict)
7931
8613
  full_walk_complete: bool = False
8614
+ # #916: an exhaustive walk that visited every on-disk source cleanly and
8615
+ # whose only defect was orphan rows (tracked transcripts no longer on
8616
+ # disk). It wrote the residual token instead of the walk-complete marker,
8617
+ # so ``full_walk_complete`` stays False; the accounting frontier accepts
8618
+ # either (`_lib_ingest_frontier.provider_sync_certifiable`).
8619
+ full_walk_residual_only: bool = False
8620
+ orphan_residual_count: int = 0
7932
8621
  maintenance_failed: bool = False
7933
8622
  # #944 spec 5.5 step 5: files whose descriptor no longer matched the
7934
8623
  # frozen stat (replaced between stat and open); retried next sync.
@@ -8172,18 +8861,38 @@ def _targeted_has_pending_global_work(conn) -> bool:
8172
8861
  return row is not None
8173
8862
 
8174
8863
 
8175
- def _acquire_cache_flock(lock_fh, *, timeout):
8864
+ def _acquire_cache_flock(lock_fh, *, timeout, poll: float = 0.2,
8865
+ deadline: "float | None" = None):
8176
8866
  """Acquire the exclusive cache flock on ``lock_fh``.
8177
8867
 
8178
- ``timeout is None`` -> a single non-blocking attempt (today's behavior
8179
- for routine auto-syncs): returns False immediately on contention.
8180
- ``timeout > 0`` -> retry ``LOCK_NB`` every ~0.2s until acquired or the
8181
- deadline elapses. Returns True iff the lock is held on return.
8868
+ ``timeout is None`` (or ``0``) -> a single non-blocking attempt (today's
8869
+ behavior for routine auto-syncs): returns False immediately on
8870
+ contention. ``timeout > 0`` -> retry ``LOCK_NB`` every ``poll`` seconds
8871
+ until acquired or the deadline elapses; the first attempt always happens.
8872
+ Returns True iff the lock is held on return.
8873
+
8874
+ An explicit absolute ``deadline`` (``time.monotonic()`` seconds; #1055
8875
+ spec 4.5) replaces ``timeout`` and has its own rule: an attempt is made
8876
+ only while ``monotonic() < deadline``, and each sleep is clamped to the
8877
+ time remaining, so an expired deadline makes no attempt at all.
8182
8878
 
8183
8879
  A retry-with-sleep loop, NOT a SIGALRM-based blocking LOCK_EX: the
8184
8880
  dashboard runs its sync on a background thread, where Python signals
8185
8881
  never fire, so an alarm timeout would silently never trip.
8186
8882
  """
8883
+ if deadline is not None:
8884
+ while True:
8885
+ if deadline - time.monotonic() <= 0:
8886
+ return False
8887
+ try:
8888
+ fcntl.flock(lock_fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
8889
+ return True
8890
+ except BlockingIOError:
8891
+ pass
8892
+ remaining = deadline - time.monotonic()
8893
+ if remaining <= 0:
8894
+ return False
8895
+ time.sleep(min(poll, remaining))
8187
8896
  deadline = None if timeout is None else (time.monotonic() + timeout)
8188
8897
  while True:
8189
8898
  try:
@@ -8192,7 +8901,38 @@ def _acquire_cache_flock(lock_fh, *, timeout):
8192
8901
  except BlockingIOError:
8193
8902
  if deadline is None or time.monotonic() >= deadline:
8194
8903
  return False
8195
- time.sleep(0.2)
8904
+ time.sleep(poll)
8905
+
8906
+
8907
+ def _wait_behind_drain_unit(lock_fh, wait_seconds: float, owner: dict) -> str:
8908
+ """The Claude hook's waiting path after its first refusal (#1055 4.4).
8909
+
8910
+ Waits only when both the Codex provider lock and the drain gate are busy
8911
+ (best-effort evidence of an admitted drain unit), as the one registered
8912
+ Claude waiter, until one absolute deadline. Puts the waiter in
8913
+ ``owner["waiter"]`` as soon as it exists, so the caller's cleanup
8914
+ releases it on every path. Returns the ``drain_wait`` outcome; the lock
8915
+ is held iff it is ``acquired``.
8916
+ """
8917
+ import _lib_codex_drain_admission as adm
8918
+ deadline = time.monotonic() + wait_seconds
8919
+ if not adm.lock_busy(_cctally_core.CACHE_LOCK_CODEX_PATH):
8920
+ return "not_drain" # Claude against Claude, or another writer
8921
+ if not adm.lock_busy(_cctally_core.APP_DIR / adm.GATE_FILE_NAME):
8922
+ return "not_drain" # a Codex hook or an unadmitted Codex sync
8923
+ waiter = adm.ClaudeWaiter.register(
8924
+ _cctally_core.APP_DIR,
8925
+ deadline_ns=adm._monotonic_ns()
8926
+ + round((deadline - time.monotonic()) * 1e9))
8927
+ if waiter is None:
8928
+ return "busy_waiter"
8929
+ owner["waiter"] = waiter
8930
+ if not _acquire_cache_flock(
8931
+ lock_fh, timeout=None, poll=adm.CLAUDE_HOOK_DRAIN_POLL_SECONDS,
8932
+ deadline=deadline):
8933
+ return "timeout"
8934
+ waiter.mark_holding()
8935
+ return "acquired"
8196
8936
 
8197
8937
 
8198
8938
  def _prune_orphaned_cache_entries(conn, *, lock_timeout=None):
@@ -8679,11 +9419,21 @@ def sync_cache(
8679
9419
  rebuild: bool = False,
8680
9420
  only_paths: "set[str] | None" = None,
8681
9421
  lock_timeout: "float | None" = None,
9422
+ drain_wait_seconds: "float | None" = None,
8682
9423
  ) -> IngestStats:
8683
9424
  """Read-through delta ingest. Acquires an exclusive fcntl.flock; if
8684
9425
  another process holds it, returns immediately with lock_contended=True
8685
9426
  and the caller should proceed with whatever data is already cached.
8686
9427
 
9428
+ ``drain_wait_seconds`` (#1055; only the hook-tick Claude leg passes it,
9429
+ and only on its first invocation) applies when ``lock_timeout`` is None:
9430
+ after the first refusal, a hook that sees an admitted Codex drain unit
9431
+ may wait that long as the one Claude waiter the drain workers yield to.
9432
+ It must not exceed ``max(T, CLAUDE_WAITER_HOLD_ALLOWANCE_SECONDS)``: the
9433
+ drains treat a record ending further out as not fresh, so a larger
9434
+ allowance would poll while the drains never yield. Only the hook leg
9435
+ passes it, with ``CLAUDE_HOOK_DRAIN_WAIT_SECONDS``.
9436
+
8687
9437
  When `rebuild=True`, clears the cached rows AFTER acquiring the lock
8688
9438
  so a lost race does not wipe a cache another process is actively
8689
9439
  populating. If the lock is contended on a rebuild, the cache is left
@@ -8699,10 +9449,25 @@ def sync_cache(
8699
9449
  # verdict. Settled by the clean end-of-walk write; every other exit
8700
9450
  # withdraws a present marker in ``finally``.
8701
9451
  walk_marker_owed = False
9452
+ # #1055: the Claude waiter, once registered; released after the writer
9453
+ # unlock on every exit path (the outermost ``finally``).
9454
+ drain_waiter: dict = {}
8702
9455
  try:
8703
9456
  with _perf.phase("flock") as _p_flock:
8704
9457
  _acquired = _acquire_cache_flock(lock_fh, timeout=lock_timeout)
8705
- _p_flock.set_meta(contended=not _acquired)
9458
+ _drain_wait, _waited = "none", 0.0
9459
+ if (not _acquired and lock_timeout is None
9460
+ and drain_wait_seconds is not None
9461
+ and drain_wait_seconds > 0):
9462
+ _refused_at = time.monotonic()
9463
+ try:
9464
+ _drain_wait = _wait_behind_drain_unit(
9465
+ lock_fh, drain_wait_seconds, drain_waiter)
9466
+ finally:
9467
+ _waited = time.monotonic() - _refused_at
9468
+ _acquired = _drain_wait == "acquired"
9469
+ _p_flock.set_meta(contended=not _acquired, drain_wait=_drain_wait,
9470
+ drain_wait_seconds=round(_waited, 3))
8706
9471
  if not _acquired:
8707
9472
  eprint("[cache] sync already in progress; using existing cache")
8708
9473
  stats.lock_contended = True
@@ -8895,8 +9660,9 @@ def sync_cache(
8895
9660
  # Clear the walk-complete sentinel atomically with the wipe
8896
9661
  # (cctally-dev#93, D5/D2): a stale "complete" marker must never
8897
9662
  # survive a destructive rebuild. The end-of-loop write below
8898
- # re-establishes it only after this rebuild's clean walk.
8899
- conn.execute("DELETE FROM cache_meta WHERE key='claude_ingest_walk_complete'")
9663
+ # re-establishes it only after this rebuild's clean walk. #916:
9664
+ # the residual token goes in the same statement.
9665
+ _clear_claude_walk_verdicts(conn)
8900
9666
  # Issue #139: a rebuild walks every file from offset 0, so the
8901
9667
  # per-file fused walk below repopulates the whole message
8902
9668
  # index — that satisfies any deferred existing-install backfill.
@@ -9219,12 +9985,24 @@ def sync_cache(
9219
9985
  # one exists, so a repeated orphaned sync doesn't churn a no-op write
9220
9986
  # txn every tick); merely withholding THIS run's end-of-loop rewrite
9221
9987
  # is not enough, since a stale marker from a previous sync would
9222
- # otherwise survive and keep vouching for completeness. Setting
9223
- # walk_clean=False additionally suppresses the end-of-loop rewrite so
9224
- # the marker stays absent for this run. With the marker gone the
9225
- # upgrade gate DEFERs the 008/009/010 recomputes (rather than
9226
- # certifying aggregates that still include data from files no longer
9227
- # on disk). The safe CLEANUP lives OUT of sync_cache, in
9988
+ # otherwise survive and keep vouching for completeness. The orphan
9989
+ # set is kept in ``residual_orphans`` and the end-of-loop block
9990
+ # withholds the marker for this run whenever it is non-empty. With the
9991
+ # marker gone the upgrade gate DEFERs the 008/009/010 recomputes
9992
+ # (rather than certifying aggregates that still include data from
9993
+ # files no longer on disk).
9994
+ #
9995
+ # #916: orphans no longer set walk_clean=False. walk_clean describes
9996
+ # the on-disk sources only, so a walk whose ONLY defect is orphan rows
9997
+ # ends "residual-only" and writes the
9998
+ # `claude_ingest_walk_orphan_residual` token instead of the marker.
9999
+ # The token is the accounting frontier's completeness signal and
10000
+ # nobody else's: the walk never reads or writes the orphan rows, so
10001
+ # every on-disk source's bytes are still reflected in the cache, which
10002
+ # is all the frontier certifies. Without it a residual the prune gates
10003
+ # refuse kept the dashboard re-walking every transcript on every tick
10004
+ # until a rebuild. The marker keeps its D5a meaning unchanged.
10005
+ # The safe CLEANUP lives OUT of sync_cache, in
9228
10006
  # _prune_orphaned_cache_entries — it re-derives the safe orphan set
9229
10007
  # independently and removes the full derived surface under three
9230
10008
  # gates: A (session-id not shared by a survivor), B (coverage — every
@@ -9251,6 +10029,7 @@ def sync_cache(
9251
10029
  # file for an orphan and nuke the walk-complete marker. Skip it entirely
9252
10030
  # for targeted: the live-tail fast path never prunes orphans (the full
9253
10031
  # background sync owns that).
10032
+ residual_orphans: "tuple[str, ...]" = ()
9254
10033
  if not targeted:
9255
10034
  global _LAST_WARNED_ORPHAN_SET
9256
10035
  on_disk_paths = {str(jp) for jp in paths}
@@ -9283,7 +10062,10 @@ def sync_cache(
9283
10062
  "DELETE FROM cache_meta WHERE key='claude_ingest_walk_complete'"
9284
10063
  )
9285
10064
  conn.commit()
9286
- walk_clean = False # orphaned rows -> cache doesn't mirror disk (D5a)
10065
+ # Orphaned rows -> the cache doesn't mirror disk (D5a): the
10066
+ # end-of-loop block withholds the marker (#916: and writes the
10067
+ # residual token when the on-disk walk was clean).
10068
+ residual_orphans = tuple(sorted(orphaned_tracked_paths))
9287
10069
  else:
9288
10070
  # No orphans this walk: clear the throttle memory so a LATER,
9289
10071
  # distinct orphan episode (even one recreated at the same paths)
@@ -9363,7 +10145,8 @@ def sync_cache(
9363
10145
  # full-reset (cctally-dev#93, D5/D2): the cache is being wiped, so
9364
10146
  # any "complete" marker is now stale. The end-of-loop write below
9365
10147
  # re-establishes it only after this run's clean re-ingest walk.
9366
- conn.execute("DELETE FROM cache_meta WHERE key='claude_ingest_walk_complete'")
10148
+ # #916: the residual token goes in the same statement.
10149
+ _clear_claude_walk_verdicts(conn)
9367
10150
  # Crash-safety: also clear session_files's size/offset tracking
9368
10151
  # so a partial-state recovery on the NEXT sync forces every
9369
10152
  # file's per-file branch to take the fresh-ingest path. Without
@@ -9413,15 +10196,19 @@ def sync_cache(
9413
10196
  # "walk" phase (never per-row — Section 2 rule: volume is a count, not
9414
10197
  # N timed phases). Opened via the context-manager protocol so the hot
9415
10198
  # loop body below is not reindented; counts recorded after the loop.
9416
- # #195: is the cache-write-split re-walk armed? Computed ONCE per
9417
- # sync_cache call, before the file loop, so every file in this walk uses
9418
- # one statement. Cache migration 030 sets this flag and zeroes the
9419
- # per-file cursors; the end-of-walk block below clears it after a clean,
9420
- # non-targeted full walk. While armed, the chained-conflict variant is
9421
- # used so a replayed NULL-key row updates in place instead of raising
9422
- # IntegrityError and rolling back its whole file. Scoped to THIS flag
9423
- # (not marker-absence) so migration 020's loud duplicate-physical-key
9424
- # backstop stays intact on every other ingest path.
10199
+ # #195: is the cache-write-split re-walk armed? Read ONCE per
10200
+ # sync_cache call, before the file loop. Cache migration 030 sets this
10201
+ # flag and resets the per-file cursors to -1; the end-of-walk block
10202
+ # below clears it after a non-targeted full walk that is clean on disk,
10203
+ # either fully clean or residual-only (#916). While armed, every file
10204
+ # uses the chained-conflict variant so a replayed NULL-key row updates
10205
+ # in place instead of raising IntegrityError and rolling back its whole
10206
+ # file. #916: the statement is chosen PER FILE (``use_rewalk`` in the
10207
+ # loop): a file whose own cursor still reads -1 takes the chained
10208
+ # variant even after the flag retired. Scoped to the flag and to -1
10209
+ # cursors (not marker-absence) so migration 020's loud
10210
+ # duplicate-physical-key backstop stays intact on every other ingest
10211
+ # path.
9425
10212
  rewalk_armed = conn.execute(
9426
10213
  "SELECT 1 FROM cache_meta WHERE key=?",
9427
10214
  (_cctally_db_sib.CACHE_CREATION_SPLIT_REWALK_KEY,),
@@ -9663,13 +10450,24 @@ def sync_cache(
9663
10450
  # collides on idx_entries_physical and would roll back the
9664
10451
  # whole per-file transaction. Steady state keeps the
9665
10452
  # single-target SQL and its LOUD physical-key backstop.
10453
+ #
10454
+ # #916: the obligation is also carried per file. Migration
10455
+ # 030 is the only producer of a negative ``size_bytes``,
10456
+ # so a ``-1`` cursor is a file not yet re-walked since the
10457
+ # reset, still owed the chained clause even after a
10458
+ # residual-only walk retired the global flag (an orphan
10459
+ # whose transcript reappears). Read from THIS file's own
10460
+ # ``prev`` row: the ``prev_size`` local is unbound for a
10461
+ # new file and stale for a file after a ``-1`` one.
10462
+ use_rewalk = rewalk_armed or (
10463
+ prev is not None and prev[0] < 0)
9666
10464
  if attribution_binds is not None:
9667
10465
  _sql = (SESSION_ENTRY_UPSERT_SQL_REWALK_LINKED
9668
- if rewalk_armed
10466
+ if use_rewalk
9669
10467
  else SESSION_ENTRY_UPSERT_SQL_LINKED)
9670
10468
  else:
9671
10469
  _sql = (SESSION_ENTRY_UPSERT_SQL_REWALK
9672
- if rewalk_armed else SESSION_ENTRY_UPSERT_SQL)
10470
+ if use_rewalk else SESSION_ENTRY_UPSERT_SQL)
9673
10471
  conn.executemany(_sql, stamped_rows)
9674
10472
  stats.rows_changed += conn.total_changes - before
9675
10473
  # Conversation message ingest (Plan 1). Lands in the SAME
@@ -9851,7 +10649,12 @@ def sync_cache(
9851
10649
  # writes nothing after the first; the value is therefore the instant
9852
10650
  # the current run of complete walks began. Any exit that does not reach
9853
10651
  # here withdraws the marker (``walk_marker_owed`` in ``finally``).
9854
- if walk_clean and applied_at_start and not targeted:
10652
+ #
10653
+ # #916: a walk that is clean on disk but still tracks orphans
10654
+ # (``residual_orphans``) writes the residual token instead. The two
10655
+ # are mutually exclusive: the marker branch deletes the token, and the
10656
+ # orphan leg already deleted the marker before a token is written.
10657
+ if walk_clean and applied_at_start and not targeted and not residual_orphans:
9855
10658
  conn.execute(
9856
10659
  "INSERT INTO cache_meta(key, value) "
9857
10660
  "VALUES('claude_ingest_walk_complete', ?) "
@@ -9866,9 +10669,53 @@ def sync_cache(
9866
10669
  "DELETE FROM cache_meta WHERE key=?",
9867
10670
  (_cctally_db_sib.CACHE_CREATION_SPLIT_REWALK_KEY,),
9868
10671
  )
10672
+ # #916: the residual cleared (pruned, rebuilt or restored); the
10673
+ # marker now vouches, so the token goes in the same transaction.
10674
+ conn.execute(
10675
+ "DELETE FROM cache_meta WHERE key=?",
10676
+ (_ingest_frontier.CLAUDE_WALK_ORPHAN_RESIDUAL_KEY,),
10677
+ )
9869
10678
  conn.commit()
9870
10679
  stats.full_walk_complete = True
9871
10680
  walk_marker_owed = False
10681
+ elif walk_clean and applied_at_start and not targeted:
10682
+ # #916: residual-only. Every on-disk source was visited cleanly,
10683
+ # so the split re-walk flag retires here too: the only files still
10684
+ # owed their re-walk are absent orphans, and each one's own ``-1``
10685
+ # cursor re-selects the chained clause if it reappears. The marker
10686
+ # stays absent (D5a), and ``full_walk_complete`` stays False.
10687
+ conn.execute(
10688
+ "DELETE FROM cache_meta WHERE key=?",
10689
+ (_cctally_db_sib.CACHE_CREATION_SPLIT_REWALK_KEY,),
10690
+ )
10691
+ # #901 W8: an unchanged orphan set writes nothing. ``at`` is
10692
+ # excluded from the comparison, so it records when the current
10693
+ # set was first seen; the value is debug-only and no reader acts
10694
+ # on it.
10695
+ token = {
10696
+ "orphans": len(residual_orphans),
10697
+ "fingerprint": hashlib.sha256(
10698
+ "\n".join(residual_orphans).encode("utf-8")).hexdigest(),
10699
+ }
10700
+ row = conn.execute(
10701
+ "SELECT value FROM cache_meta WHERE key=?",
10702
+ (_ingest_frontier.CLAUDE_WALK_ORPHAN_RESIDUAL_KEY,),
10703
+ ).fetchone()
10704
+ try:
10705
+ held = json.loads(row[0]) if row is not None else None
10706
+ except (TypeError, ValueError):
10707
+ held = None
10708
+ if not (isinstance(held, dict)
10709
+ and held.get("orphans") == token["orphans"]
10710
+ and held.get("fingerprint") == token["fingerprint"]):
10711
+ token["at"] = dt.datetime.now(dt.timezone.utc).isoformat()
10712
+ _set_cache_meta(
10713
+ conn, _ingest_frontier.CLAUDE_WALK_ORPHAN_RESIDUAL_KEY,
10714
+ json.dumps(token))
10715
+ conn.commit()
10716
+ stats.full_walk_residual_only = True
10717
+ stats.orphan_residual_count = len(residual_orphans)
10718
+ walk_marker_owed = False
9872
10719
  # #279 S2 F1: rolling parse-health record. Anomaly-delta-gated so
9873
10720
  # steady-state (incl. targeted live-tail) syncs stay zero-write;
9874
10721
  # targeted syncs still accumulate — they ingest real new bytes.
@@ -9892,15 +10739,29 @@ def sync_cache(
9892
10739
  # stalling the lock under heavy-reader contention.
9893
10740
  _maybe_truncate_wal(conn, _cctally_core.CACHE_DB_PATH)
9894
10741
  finally:
9895
- if walk_marker_owed:
9896
- _withdraw_cache_meta_marker(conn, "claude_ingest_walk_complete")
9897
- # #901 §5.3c (W9): the end of a cache sync is a checkpoint boundary.
9898
- _wal_checkpoint.at_boundary(conn)
10742
+ # #1055 spec 4.5: nested so a cleanup failure still unlocks the
10743
+ # writer, and the Claude waiter is released only after that unlock.
9899
10744
  try:
9900
- fcntl.flock(lock_fh, fcntl.LOCK_UN)
9901
- except OSError:
9902
- pass
9903
- lock_fh.close()
10745
+ try:
10746
+ if walk_marker_owed:
10747
+ _withdraw_cache_meta_marker(
10748
+ conn, "claude_ingest_walk_complete")
10749
+ # #916: an unsettled walk cannot vouch for residual-only
10750
+ # either.
10751
+ _withdraw_cache_meta_marker(
10752
+ conn, _ingest_frontier.CLAUDE_WALK_ORPHAN_RESIDUAL_KEY)
10753
+ # #901 §5.3c (W9): the end of a cache sync is a checkpoint
10754
+ # boundary.
10755
+ _wal_checkpoint.at_boundary(conn)
10756
+ finally:
10757
+ try:
10758
+ fcntl.flock(lock_fh, fcntl.LOCK_UN)
10759
+ except OSError:
10760
+ pass
10761
+ lock_fh.close()
10762
+ finally:
10763
+ if drain_waiter.get("waiter") is not None:
10764
+ drain_waiter["waiter"].release()
9904
10765
  return stats
9905
10766
 
9906
10767
 
@@ -11301,7 +12162,9 @@ def _record_codex_container_status(conn, stats, *, census_complete: bool,
11301
12162
  prior = decoded if isinstance(decoded, dict) else {}
11302
12163
  except (TypeError, ValueError):
11303
12164
  prior = {}
11304
- observed = {k: int(stats.container_skipped.get(k, 0))
12165
+ # #1052 spec 5.3: every skip the pass saw, its unit phase's included.
12166
+ seen_skipped = stats.all_container_skipped()
12167
+ observed = {k: int(seen_skipped.get(k, 0))
11305
12168
  for k in _CONTAINER_SKIP_KEYS}
11306
12169
  # D3: verification debt is pending work only. A terminal `unverified`
11307
12170
  # container is reported (and warned about) on its own and never feeds V
@@ -11356,7 +12219,7 @@ def _record_codex_container_status(conn, stats, *, census_complete: bool,
11356
12219
  # ingestion observation does not pretend that an idle walk probed Python.
11357
12220
  new_capability_observation = bool(
11358
12221
  stats.files_processed or stats.container_pending
11359
- or any(stats.container_skipped.values()) or stats.rebuild_refused)
12222
+ or any(seen_skipped.values()) or stats.rebuild_refused)
11360
12223
  decoder = prior.get("decoder")
11361
12224
  decoder_observed_at = prior.get("decoderObservedAt", prior.get("observedAt"))
11362
12225
  if new_capability_observation or decoder not in (
@@ -11892,9 +12755,57 @@ class CodexIngestStats:
11892
12755
  barrier_deferred_targets: dict = field(default_factory=dict)
11893
12756
  barrier_published_paths: set = field(default_factory=set)
11894
12757
  barrier_retry_causes: set = field(default_factory=set)
12758
+ # #1052 spec 5.4: the conversation pass's terminal deferrals. A unit whose
12759
+ # contributor carries the conversations store's terminal `unverified`
12760
+ # verdict, and none of whose files is blocked, publishes nothing and is
12761
+ # not retried. ``barrier_keys_terminal`` counts its keys (they stay in
12762
+ # ``barrier_keys_deferred`` too) and ``barrier_terminal_keys`` maps each
12763
+ # terminal file to the keys it holds back (a residual field the
12764
+ # transcript worker does not serialize). ``barrier_container_skipped``
12765
+ # counts, by reason, the unit-phase skips of paths phase 1 did not skip:
12766
+ # ``container_skipped`` is phase 1's census only.
12767
+ barrier_keys_terminal: int = 0
12768
+ barrier_terminal_keys: dict = field(default_factory=dict)
12769
+ barrier_container_skipped: dict = field(default_factory=dict)
11895
12770
  # The status writer already observes capability for unresolved work.
11896
12771
  # Explicit adapters reuse it; unchanged accepted generations never probe.
11897
12772
  container_decoder: "str | None" = None
12773
+ # #982 spec 4.1 steps 4 and 7: a heal walk piece's own report. The files
12774
+ # it ingested (or reset), whether its ``_ingest_limit`` stopped it before
12775
+ # the next such file, the census files it examined successfully
12776
+ # (ingested, reset or found unchanged; never a failed, torn-deferred,
12777
+ # held or U6-skipped file), its discovery set (the census when it is a
12778
+ # walk's first piece), whether it left no census file unwalked, and
12779
+ # whether it made the transition (deleted the heal record) in its
12780
+ # end-of-walk transaction.
12781
+ heal_counted_paths: set = field(default_factory=set)
12782
+ heal_limit_stopped: bool = False
12783
+ heal_examined: "frozenset[str]" = frozenset()
12784
+ heal_discovered: "frozenset[str] | None" = None
12785
+ heal_walk_complete: bool = False
12786
+ heal_transitioned: bool = False
12787
+ # Revision 7: the committed cursor row of every census file this piece
12788
+ # visited (read after its last commit, under its locks), and the visited
12789
+ # files of earlier pieces whose rows changed before the walk's last piece.
12790
+ heal_visit_rows: dict = field(default_factory=dict)
12791
+ heal_visit_changed: "frozenset[str]" = frozenset()
12792
+ # #982 spec 4.4: a budgeted call declined `replay_pending` while a replay
12793
+ # heal record exists (the store is cleared and partly refilled), so the
12794
+ # hook must not run its projection or budget leg over it. A budgeted call
12795
+ # that met contention sets it from an unlocked read of the same record
12796
+ # (`_codex_replay_refill_unlocked`), and its hook publishes no request.
12797
+ replay_refill: bool = False
12798
+ # #982 spec 4.5a (Generation): an executor unit whose pass generation is
12799
+ # not the heal record's (or that found a fresh clear only a new pass's
12800
+ # start unit may perform) wrote nothing; the executor ends its pass
12801
+ # `restart`.
12802
+ heal_generation_stale: bool = False
12803
+ # #972 §5.4: conversations whose rollup the full pass's reconcile
12804
+ # rewrote (thread-derived fields drifted from the current answer).
12805
+ rollups_reattributed: int = 0
12806
+ # #972 §5.8 rule 4: rollups the reconcile skipped because deriving their
12807
+ # expected fields raised a non-database error (an unresolvable cwd).
12808
+ rollups_reconcile_skipped: int = 0
11898
12809
 
11899
12810
  @property
11900
12811
  def targeted_clean(self) -> bool:
@@ -11906,7 +12817,29 @@ class CodexIngestStats:
11906
12817
  and self.files_failed == 0
11907
12818
  and not self.container_pending
11908
12819
  and not self.barrier_retry
11909
- and not any(self.container_skipped.values()))
12820
+ and not any(self.all_container_skipped().values()))
12821
+
12822
+ def all_container_skipped(self) -> "dict[str, int]":
12823
+ """Every container skip this pass saw, by reason (#1052 spec 5.3):
12824
+ phase 1's census plus the unit phase's skips of other paths. The two
12825
+ are path-disjoint by construction, so the sum counts each path once.
12826
+ """
12827
+ merged = dict(self.container_skipped)
12828
+ for reason, count in self.barrier_container_skipped.items():
12829
+ merged[reason] = merged.get(reason, 0) + count
12830
+ return merged
12831
+
12832
+
12833
+ def _print_codex_done_line(stats: CodexIngestStats) -> None:
12834
+ """`cache-sync`'s Codex summary line (the first leg's and the heal's)."""
12835
+ eprint(
12836
+ f"[cache-sync] codex done: {stats.files_processed} processed, "
12837
+ f"{stats.files_skipped_unchanged} skipped, "
12838
+ f"{stats.files_reset_truncated} reset, "
12839
+ f"{stats.rows_changed} rows changed, "
12840
+ f"{stats.lines_malformed} malformed, "
12841
+ f"{stats.token_events_skipped} drift-skipped"
12842
+ )
11910
12843
 
11911
12844
 
11912
12845
  def _progress_codex_stderr(stats: CodexIngestStats, *, force: bool = False) -> None:
@@ -12113,7 +13046,7 @@ def _record_codex_replay_deferral(conn: sqlite3.Connection) -> None:
12113
13046
  raise
12114
13047
 
12115
13048
 
12116
- def _defer_codex_replay_drain() -> str:
13049
+ def _defer_codex_replay_drain(*, short: bool = False) -> str:
12117
13050
  """Hand the declined byte-zero replay to a detached worker (public #5).
12118
13051
 
12119
13052
  Returns ``"spawned"``, ``"throttled"`` or ``"failed"`` — for the lifecycle
@@ -12128,6 +13061,10 @@ def _defer_codex_replay_drain() -> str:
12128
13061
  re-read everything", and slicing that corrupts account attribution — so
12129
13062
  there is nothing to fall back to. The record written at the decline site is
12130
13063
  what makes a drain that never succeeds diagnosable.
13064
+
13065
+ ``short`` (#982 spec 4.5a): a heal pass that ended ``restart`` asks for
13066
+ the short retry throttle even when no heal record exists (a replay
13067
+ re-armed after the pass's transition leaves only the marker).
12131
13068
  """
12132
13069
  marker = _cctally_core.APP_DIR / CODEX_REPLAY_DRAIN_MARKER_NAME
12133
13070
  try:
@@ -12135,6 +13072,20 @@ def _defer_codex_replay_drain() -> str:
12135
13072
  except OSError:
12136
13073
  age = None
12137
13074
  if age is not None and 0 <= age < CODEX_REPLAY_DRAIN_SPAWN_THROTTLE_SECONDS:
13075
+ # #982 spec 4.5a: every path that requests this worker comes through
13076
+ # here, so the heal record's short retry throttle is applied once,
13077
+ # for all of them. Probed only inside the window it can change.
13078
+ if (age < CODEX_HEAL_RETRY_THROTTLE_SECONDS
13079
+ or not (short or _codex_heal_record_present())):
13080
+ return "throttled"
13081
+ if _codex_heal_walk_running():
13082
+ # A heal pass is running: hooks request the worker on every tick
13083
+ # while a record exists, and a worker spawned now would only run the
13084
+ # provenance, verification, reset, membership and seal drains beside
13085
+ # the executor. The pass requests the worker itself, after releasing
13086
+ # this lock, when it ends `contended` or `degraded`; on `restart` its
13087
+ # callers re-run it inline. The marker is not stamped, so the first
13088
+ # request after the pass is not throttled by this one.
12138
13089
  return "throttled"
12139
13090
  try:
12140
13091
  _cctally_core.APP_DIR.mkdir(parents=True, exist_ok=True)
@@ -12145,6 +13096,51 @@ def _defer_codex_replay_drain() -> str:
12145
13096
  else "failed")
12146
13097
 
12147
13098
 
13099
+ def _codex_heal_walk_running() -> bool:
13100
+ """Whether a heal pass holds ``codex-heal-walk.lock``: a non-blocking
13101
+ exclusive probe on a separate open file description, released at once.
13102
+ Never creates the file; any other error answers False (spawn as before).
13103
+ """
13104
+ path = _cctally_core.APP_DIR / CODEX_HEAL_WALK_LOCK_NAME
13105
+ try:
13106
+ fd = os.open(path, os.O_RDWR)
13107
+ except OSError:
13108
+ return False
13109
+ try:
13110
+ try:
13111
+ fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
13112
+ except BlockingIOError:
13113
+ return True
13114
+ except OSError:
13115
+ return False
13116
+ fcntl.flock(fd, fcntl.LOCK_UN)
13117
+ return False
13118
+ finally:
13119
+ os.close(fd)
13120
+
13121
+
13122
+ def _codex_heal_record_present() -> bool:
13123
+ """Whether cache.db holds a well-formed heal record, read through a raw
13124
+ ``mode=rw`` connection (no schema apply, migration or store lock; never
13125
+ creates the file). Anything unreadable answers False, which keeps the
13126
+ hourly throttle."""
13127
+ path = pathlib.Path(_cctally_core.CACHE_DB_PATH)
13128
+ try:
13129
+ if not path.is_file():
13130
+ return False
13131
+ conn = sqlite3.connect(f"{path.resolve().as_uri()}?mode=rw",
13132
+ uri=True, timeout=1.0)
13133
+ except (OSError, sqlite3.Error):
13134
+ return False
13135
+ try:
13136
+ record = load_codex_heal_record(conn)
13137
+ except sqlite3.Error:
13138
+ return False
13139
+ finally:
13140
+ conn.close()
13141
+ return record is not None and not record.get("malformed")
13142
+
13143
+
12148
13144
  class ResetDrainResult(NamedTuple):
12149
13145
  outcome: str
12150
13146
  reset: int
@@ -12338,14 +13334,33 @@ def _cmd_codex_replay_drain_body(args) -> int:
12338
13334
  conn = open_cache_db()
12339
13335
  try:
12340
13336
  stats = CodexIngestStats()
12341
- if ((history.outcome == "settled" and history.candidates)
12342
- or conn.execute("SELECT 1 FROM cache_meta WHERE key=? LIMIT 1",
12343
- (CODEX_REPLAY_FROM_ZERO_KEY,)).fetchone()):
12344
- stats = sync_codex_cache(
12345
- conn, maintenance="caller",
12346
- lock_timeout=CODEX_REPLAY_DRAIN_LOCK_TIMEOUT_SECONDS)
13337
+ replay_armed = conn.execute(
13338
+ "SELECT 1 FROM cache_meta WHERE key=? LIMIT 1",
13339
+ (CODEX_REPLAY_FROM_ZERO_KEY,)).fetchone() is not None
13340
+ heal_record = load_codex_heal_record(conn)
12347
13341
  finally:
12348
13342
  conn.close()
13343
+ # #982 spec 4.6: the byte-zero replay and the history heal that
13344
+ # follows a settled census both run as admitted per-file units, never
13345
+ # as one locked walk; a record left by a dead executor resumes from
13346
+ # its phase. The executor picks replay mode itself when the marker or
13347
+ # a replay record exists.
13348
+ heal = HealResult("none", None, 0, 0, 0, 0, 0)
13349
+ if (replay_armed or heal_record is not None
13350
+ or (history.outcome == "settled" and history.candidates)):
13351
+ heal = run_codex_admitted_heal(
13352
+ wait_for_requests=True,
13353
+ population=(None if heal_record is not None or replay_armed
13354
+ else _codex_history_candidates()))
13355
+ if heal.outcome in ("replay_due", "restart"):
13356
+ # Spec 4.5a: a replay armed during the history heal ended
13357
+ # that pass before its fenced walk; this pass replays.
13358
+ heal = run_codex_admitted_heal(wait_for_requests=True)
13359
+ if heal.outcome == "contended":
13360
+ stats = CodexIngestStats(lock_contended=True)
13361
+ elif heal.stats is not None:
13362
+ # Either mode: the heal's own counts, summed over its units.
13363
+ stats = heal.stats
12349
13364
  if stats.rebuild_refused:
12350
13365
  _log("refused", f"rebuild_refused={stats.rebuild_refused}")
12351
13366
  return 0
@@ -12379,7 +13394,9 @@ def _cmd_codex_replay_drain_body(args) -> int:
12379
13394
  f"reset_done={reset_result.reset} reset_unchanged={reset_result.unchanged} "
12380
13395
  f"barrier_proven={scan.proven} barrier_sealed={seal.sealed} "
12381
13396
  f"barrier_remaining={seal.remaining} history={history.outcome} "
12382
- f"history_candidates={history.candidates} history_traced={history.traced}")
13397
+ f"history_candidates={history.candidates} history_traced={history.traced} "
13398
+ f"heal={heal.outcome} heal_units={heal.units} "
13399
+ f"heal_degraded={heal.degraded}")
12383
13400
  return 0
12384
13401
 
12385
13402
 
@@ -12483,6 +13500,15 @@ def _sync_codex_cache_locked(
12483
13500
  _on_first_file_rollback: Callable[[], None] | None = None,
12484
13501
  _on_file_committed: Callable[[str], None] | None = None,
12485
13502
  _on_file_prepared: Callable[[str], None] | None = None,
13503
+ _walk_share: "_CodexWalkShare | None" = None,
13504
+ _ingest_limit: "int | None" = None,
13505
+ _heal_census: "frozenset[str] | None" = None,
13506
+ _heal_visited: "frozenset[str]" = frozenset(),
13507
+ _heal_visited_rows: "dict[str, tuple | None] | None" = None,
13508
+ _defer_finalization: bool = False,
13509
+ _heal_transition: bool = False,
13510
+ _replay_stage: "str | None" = None,
13511
+ _heal_generation: "str | None" = None,
12486
13512
  ) -> CodexIngestStats:
12487
13513
  """Read-through delta ingest of ~/.codex/sessions/**/*.jsonl.
12488
13514
 
@@ -12520,6 +13546,66 @@ def _sync_codex_cache_locked(
12520
13546
  duplicated cost — measured at roughly half of every growing turn. Every
12521
13547
  other caller keeps ``"auto"``, because the sync-internal reconcile is what
12522
13548
  keeps the projection current for the dashboard and ``cache-sync``.
13549
+
13550
+ The private ``_walk_share`` … ``_heal_transition`` arguments belong to the
13551
+ heal executor (#982 spec 4.1–4.3, ``run_codex_admitted_heal``):
13552
+
13553
+ * ``_walk_share``: this call is an executor unit. Its resolver and ledger
13554
+ path set come from the share (bound under the locks, sealed after the
13555
+ last commit), and the heal fence's population hold and thread-row guard
13556
+ do not apply to it. Every other call holds the population.
13557
+ * ``_ingest_limit``: this call is one piece of a heal walk. After that
13558
+ many files were ingested or reset, the walk stops before the next file
13559
+ with bytes to ingest or reset; it and the rest stay unwalked, as at a
13560
+ budget stop (no deadline, no ``delta_append``), so the piece does not
13561
+ certify. A file that fails does not count.
13562
+ * ``_heal_census`` / ``_heal_visited``: the walk's census (the first
13563
+ piece's discovery set, ``stats.heal_discovered``; ``None`` on the first
13564
+ piece) and the census files earlier pieces examined successfully
13565
+ (``stats.heal_examined``). A piece skips visited files and files
13566
+ outside the census without examining them, counting them as walked
13567
+ (``files_skipped_unchanged``), exactly as the single walk treats a file
13568
+ it already visited or one created after its discovery.
13569
+ ``stats.heal_walk_complete`` says the piece left no census file
13570
+ unwalked.
13571
+ * ``_defer_finalization``: the walk's own settlement runs only the
13572
+ ``expand`` and ``model`` stages and the change-log prune stays owed.
13573
+ * ``_heal_visited_rows``: each visited file's committed cursor row as
13574
+ recorded when a piece visited it (``stats.heal_visit_rows``). The piece
13575
+ that completes the walk re-reads them under its own locks (revision
13576
+ 7): a visited file whose row another writer changed since its visit
13577
+ still counts as walked, but stays in the backlog, makes the barrier
13578
+ census ``0`` and withholds the full-walk certificate
13579
+ (``stats.heal_visit_changed``); it is not re-examined.
13580
+ * ``_heal_transition``: the piece that completes the walk deletes a
13581
+ history heal record in its end-of-walk transaction, whether or not it
13582
+ failed a file or deferred one for a torn ``auth.json``: such a file
13583
+ stays owed exactly as the single walk leaves it
13584
+ (``stats.heal_transitioned``). A replay record is ended by the
13585
+ unchanged consumption block instead: deleted with the marker, or moved
13586
+ to phase ``certify`` when the replay is blocked.
13587
+ * ``_replay_stage="start"`` (#982 spec 4.5): the replay's start unit runs
13588
+ the byte-zero replay's preamble (D1 snapshot, preflight refusals, the
13589
+ clear, which writes the replay heal record, and the authoritative
13590
+ rehydration, which marks it ``rehydrated``) and returns before the
13591
+ walk. With a replay record already present nothing is cleared again,
13592
+ unless a later arming site marked it ``reclear``: the start unit of a
13593
+ new pass is the only executor unit that performs that fresh clear.
13594
+ * ``_heal_generation`` (spec 4.5a): the ``gen`` of the record the unit's
13595
+ pass started with, carried by every executor unit but the start unit.
13596
+ Under the writer locks a unit whose generation is not the record's, or
13597
+ that would have to perform a fresh clear (``reclear``, or an armed
13598
+ marker without a replay record), writes nothing and reports
13599
+ ``stats.heal_generation_stale``; the transition and the consumption
13600
+ block act only when the generations match.
13601
+
13602
+ A replay heal record (spec 4.5) changes every other writer: the D1 test
13603
+ and the compressed-history ``known=`` read its persisted ``known`` set,
13604
+ nothing re-clears, a record not yet ``rehydrated`` is rehydrated first,
13605
+ a targeted call that is not an executor unit and every budgeted call
13606
+ decline ``replay_pending`` (the budgeted one records the deferral and
13607
+ sets ``stats.replay_refill``), and an unbudgeted ordinary call completes
13608
+ the remaining refill as an ordinary walk.
12523
13609
  """
12524
13610
  if quota_reconcile not in ("auto", "defer"):
12525
13611
  raise ValueError(
@@ -12575,7 +13661,14 @@ def _sync_codex_cache_locked(
12575
13661
  rebuild_prune_scope: "_CodexPruneScope | None" = None
12576
13662
  ordinary_prune_scope: "_CodexPruneScope | None" = None
12577
13663
  raised = False
13664
+ # #982: True once this call bound the heal executor's share under its
13665
+ # locks; only such a call may seal (or must invalidate) it.
13666
+ share_bound = False
12578
13667
  try:
13668
+ if _CODEX_PRE_STORE_LOCK_HOOK is not None:
13669
+ # Test seam (#982 R1): after admission returned, immediately
13670
+ # before this sync's own store-lock acquisition.
13671
+ _CODEX_PRE_STORE_LOCK_HOOK()
12579
13672
  with _perf.phase("flock") as _p_flock:
12580
13673
  acquired = acquire_cache_writer_flocks(
12581
13674
  _cctally_core.CACHE_LOCK_PATH,
@@ -12588,6 +13681,11 @@ def _sync_codex_cache_locked(
12588
13681
  if acquired is None:
12589
13682
  eprint("[codex-cache] sync already in progress; using existing cache")
12590
13683
  stats.lock_contended = True
13684
+ if deadline is not None:
13685
+ # #982 spec 4.4: tell the hook whether this tick would have
13686
+ # declined as a replay refill, so it publishes no waiting
13687
+ # request that the refill units could only wait out.
13688
+ stats.replay_refill = _codex_replay_refill_unlocked(conn)
12591
13689
  return stats
12592
13690
  if _after_locks is not None and _after_locks():
12593
13691
  stats.lock_contended = stats.drain_yielded = True
@@ -12641,7 +13739,80 @@ def _sync_codex_cache_locked(
12641
13739
  "SELECT 1 FROM cache_meta WHERE key=?",
12642
13740
  (CODEX_REPLAY_FROM_ZERO_KEY,),
12643
13741
  ).fetchone() is not None
12644
- if replay_pending and (targeted or deadline is not None):
13742
+ if _replay_stage not in (None, "start"):
13743
+ raise ValueError(
13744
+ "sync_codex_cache: _replay_stage must be None or 'start'")
13745
+ if _replay_stage is not None and (
13746
+ targeted or rebuild or deadline is not None):
13747
+ raise ValueError(
13748
+ "sync_codex_cache: _replay_stage is an unbudgeted whole-tree "
13749
+ "start unit")
13750
+ if _heal_generation is not None and (
13751
+ _walk_share is None or _replay_stage is not None or rebuild):
13752
+ raise ValueError(
13753
+ "sync_codex_cache: _heal_generation belongs to an executor "
13754
+ "unit that is not the start unit")
13755
+ # #982 spec 4.5: a replay heal record means the clear already
13756
+ # happened (its start unit committed). Every writer then refills
13757
+ # with the persisted D1 set instead of clearing again; only the
13758
+ # executor's units refill out of the single walk's order.
13759
+ heal_record = load_codex_heal_record(conn)
13760
+ replay_record = (heal_record if heal_record is not None
13761
+ and heal_record.get("mode") == "replay" else None)
13762
+ replay_active = replay_pending or replay_record is not None
13763
+ if _replay_stage is not None and not replay_active:
13764
+ # Nothing to start: another writer consumed the replay.
13765
+ return stats
13766
+ # #982 spec 4.5a (Generation): every executor unit but the start unit
13767
+ # belongs to the pass whose record generation it carries. A record
13768
+ # rewritten since (a fresh clear writes a new `gen`), a `reclear`
13769
+ # mark, or an armed marker without a replay record (a fresh clear
13770
+ # only a new pass's start unit may perform) ends that pass: this unit
13771
+ # writes nothing -- no ingest, clear, consumption, transition or
13772
+ # settlement, and no withdrawal of the full-walk certificate -- so no
13773
+ # piece ever skips a file it visited before a clear it did not see.
13774
+ if (_walk_share is not None and _replay_stage is None
13775
+ and _codex_heal_generation_stale(
13776
+ heal_record, replay_pending, _heal_generation)):
13777
+ stats.heal_generation_stale = True
13778
+ walk_marker_owed = False
13779
+ return stats
13780
+ # #982 spec 4.5a: a replay record re-armed by a later arming site
13781
+ # (`reclear`) is cleared again like an armed marker; a malformed one
13782
+ # (its D1 set or rehydration flag unreadable) declines every writer,
13783
+ # because clearing would recapture D1 from a partly refilled store.
13784
+ # The payload (population and D1 set) is parsed only by a call that
13785
+ # will use it, once, or taken from the executor's share. A `reclear`
13786
+ # mark acts only with the marker its arming site wrote in the same
13787
+ # transaction (`_codex_replay_reclear_due`); one whose marker is gone
13788
+ # (only a damaged store has it) is a record that no longer asks for a
13789
+ # replay, refilled with and retired like any other.
13790
+ replay_reclear = _codex_replay_reclear_due(replay_record,
13791
+ replay_pending)
13792
+ replay_payload: "CodexHealPayload | None" = None
13793
+ if replay_record is not None and not replay_record.get("malformed") \
13794
+ and deadline is None and not (targeted and _walk_share is None):
13795
+ replay_payload = (
13796
+ _walk_share.heal_payload(conn, replay_record)
13797
+ if _walk_share is not None
13798
+ else load_codex_heal_payload(conn, replay_record))
13799
+ replay_malformed = replay_record is not None and (
13800
+ bool(replay_record.get("malformed"))
13801
+ or (deadline is None and not (targeted and _walk_share is None)
13802
+ and replay_payload is None))
13803
+ refill_record = (replay_record if replay_record is not None
13804
+ and not replay_reclear and not replay_malformed
13805
+ else None)
13806
+ if replay_active and (
13807
+ # Spec 4.5a (revision 10): every writer, an explicit
13808
+ # `cache-sync --rebuild` included, fails closed on a malformed
13809
+ # record. Its persisted D1 set may be the only evidence for a
13810
+ # rollout not yet refilled, and a fresh capture cannot recover
13811
+ # it, so no clear may run over it.
13812
+ replay_malformed
13813
+ or deadline is not None
13814
+ or (targeted and (_walk_share is None
13815
+ or refill_record is None))):
12645
13816
  # A live-tail tick must DEFER, never raise through the
12646
13817
  # `targeted and rebuild` guard below.
12647
13818
  #
@@ -12668,13 +13839,54 @@ def _sync_codex_cache_locked(
12668
13839
  # and Codex ingest froze permanently. So the budgeted decline
12669
13840
  # RECORDS itself (doctor reads the record) and hands the unbudgeted
12670
13841
  # drain to a detached worker.
13842
+ #
13843
+ # #982 spec 4.5: with a replay heal record (the clear committed)
13844
+ # a targeted call proceeds only as an executor unit, and a
13845
+ # budgeted one still declines: ingesting outside the executor's
13846
+ # order would change thread rows and anchors relative to the
13847
+ # single walk. `replay_refill` tells the hook that the store is
13848
+ # cleared and partly refilled (no projection, no budget leg): a
13849
+ # record in phase `units`, or one too malformed to tell. A
13850
+ # blocked replay's `certify` record ends the fence, and its hooks
13851
+ # project and run the budget leg as for a blocked replay today.
12671
13852
  if deadline is not None:
12672
13853
  _record_codex_replay_deferral(conn)
13854
+ stats.replay_refill = replay_record is not None and (
13855
+ bool(replay_record.get("malformed"))
13856
+ or replay_record.get("phase") == "units")
12673
13857
  stats.deferred_reason = "replay_pending"
12674
13858
  return stats
12675
13859
  # Both incompatible combinations already returned at the decline above,
12676
- # so the marker-OR'd rebuild needs no second guard.
12677
- rebuild = rebuild or replay_pending
13860
+ # so the marker-OR'd rebuild needs no second guard. A replay whose
13861
+ # start unit committed (a replay record) never clears again: the
13862
+ # caller refills with the record's persisted D1 set. An explicit
13863
+ # caller rebuild still clears (it is its own single walk) and keeps
13864
+ # the persisted set in its snapshot (a malformed record declined
13865
+ # above, whatever the caller asked).
13866
+ rebuild = rebuild or (replay_pending and (replay_record is None
13867
+ or replay_reclear))
13868
+ replay_refill = refill_record is not None and not rebuild
13869
+ if _walk_share is not None and deadline is not None:
13870
+ raise ValueError(
13871
+ "sync_codex_cache: _walk_share is an unbudgeted heal unit")
13872
+ # #982 spec 4.3: a history heal record in phase `units` is the fence.
13873
+ # A sync without the executor's share holds its population (skips
13874
+ # those files, exactly as unsettled history is held) and requests the
13875
+ # worker, which resumes a dead executor from the record. A rebuild is
13876
+ # itself the single walk the heal stands in for: it ingests every
13877
+ # file and retires the record in its clear transaction below.
13878
+ heal_history = (heal_record is not None
13879
+ and heal_record.get("mode") == "history")
13880
+ heal_population: "frozenset[str]" = frozenset()
13881
+ if (heal_history and heal_record.get("phase") == "units"
13882
+ and _walk_share is None and not rebuild):
13883
+ # The one parse of the payload in this call. Unreadable holds
13884
+ # nothing more (the fence itself still stands, fail closed).
13885
+ history_payload = load_codex_heal_payload(conn, heal_record)
13886
+ if history_payload is not None:
13887
+ heal_population = history_payload.population
13888
+ stats.drain_requested = tuple(sorted(
13889
+ set(stats.drain_requested) | {CODEX_REPLAY_DRAIN_COMMAND}))
12678
13890
  # #927 spec §4.2 B4: the ordinary prune's protection needs every
12679
13891
  # retained identity at a discovered path the scope refuses, including
12680
13892
  # path-level refusal under a recognized root and child-only
@@ -12715,7 +13927,10 @@ def _sync_codex_cache_locked(
12715
13927
  # design rejects, and it is what breaks acceptance criterion 4. Captured
12716
13928
  # BEFORE the clear because the clear is what erases the evidence. A
12717
13929
  # rollout first seen DURING a rebuild is absent here and takes a normal
12718
- # first-ingest decision.
13930
+ # first-ingest decision. The clear below persists this set in its own
13931
+ # transaction for EVERY clear (the replay record's `known`, #982 4.5;
13932
+ # an explicit rebuild arms the replay there, #1051), so an interrupt
13933
+ # between the clear's commit and the walk's completion never loses it.
12719
13934
  # Keyed on the DURABLE file identity, never on `path`. `path` holds the
12720
13935
  # first configured candidate spelling, which is unstable across
12721
13936
  # `$CODEX_HOME` reordering and symlink respelling (review F12) — a
@@ -12736,6 +13951,20 @@ def _sync_codex_cache_locked(
12736
13951
  except (ValueError, TypeError, OSError):
12737
13952
  continue
12738
13953
  rebuild_known_identities |= _codex_history_identities(conn)
13954
+ if replay_payload is not None:
13955
+ # #982 spec 4.5: an explicit rebuild (or a re-armed replay's
13956
+ # fresh clear, spec 4.5a) over a refill in progress keeps
13957
+ # every identity the replay's own snapshot captured before its
13958
+ # clear (the cursors it erased).
13959
+ rebuild_known_identities |= replay_payload.known
13960
+ elif replay_refill:
13961
+ # #982 spec 4.5: the snapshot the replay's start unit captured
13962
+ # before its clear, persisted in the record. Never recaptured: the
13963
+ # cursors it was taken from are gone. Shared, never copied: a
13964
+ # refill only reads it.
13965
+ rebuild_known_identities = replay_payload.known
13966
+ # Which reads use the D1 snapshot: a clearing rebuild's and a refill's.
13967
+ d1_snapshot = rebuild or replay_refill
12739
13968
  if rebuild and not targeted:
12740
13969
  # #485: rebuild used to clear first and discover second. With an
12741
13970
  # empty/missing/misconfigured root that made the explicit repair
@@ -12748,12 +13977,30 @@ def _sync_codex_cache_locked(
12748
13977
  preflight_sources, _preflight_root_keys = (
12749
13978
  _collect_retained_codex_paths_and_roots(conn)
12750
13979
  )
13980
+ preflight_roots = _codex_provider_roots()
13981
+ # #983 spec 5.2: a pre-#294 S1 row carries no source_root_key. The
13982
+ # snapshot above skipped its cursor, and the partition would never
13983
+ # check the configured root that holds it, so resolve each one to
13984
+ # its owning root before either guard reads it. A discovered file
13985
+ # joins the D1 snapshot here, before the clear and the walk.
13986
+ if any(root_key is None for _path, root_key in preflight_sources):
13987
+ discovered_by_identity = {
13988
+ str(item.physical_path): item for item in preflight_files}
13989
+ resolved_sources: list[tuple[str, str | None]] = []
13990
+ for _path, root_key in preflight_sources:
13991
+ if root_key is None:
13992
+ root_key, owner_file = _codex_legacy_null_root_owner(
13993
+ _path, discovered_by_identity, preflight_roots)
13994
+ if owner_file is not None:
13995
+ rebuild_known_identities.add(
13996
+ codex_file_identity(owner_file))
13997
+ resolved_sources.append((_path, root_key))
13998
+ preflight_sources = resolved_sources
12751
13999
  preflight_source_root_keys = {
12752
14000
  root_key
12753
14001
  for _path, root_key in preflight_sources
12754
14002
  if root_key is not None
12755
14003
  }
12756
- preflight_roots = _codex_provider_roots()
12757
14004
  (
12758
14005
  _safe_sources,
12759
14006
  refused_sources,
@@ -12797,12 +14044,45 @@ def _sync_codex_cache_locked(
12797
14044
  if rebuild:
12798
14045
  # Clear INSIDE the lock — see sync_cache() for the full
12799
14046
  # rationale. Done before the existing SELECT so delta
12800
- # detection sees an empty baseline.
14047
+ # detection sees an empty baseline. The clear, the full
14048
+ # finalization debt, the replay marker and the replay record
14049
+ # with the D1 set commit together in this one transaction
14050
+ # (#1051): there is no intermediate commit.
12801
14051
  if _clear_codex_derived_rows(conn):
12802
14052
  _bump_codex_physical_mutation_seq(conn)
12803
14053
  # #980 D3: full re-derivation restores the unattributed state, so
12804
14054
  # every root's whole range is owed, in the wipe's own transaction.
12805
14055
  add_codex_finalization_debt(conn, full=True)
14056
+ if heal_history:
14057
+ # #982: this walk re-reads every rollout in discovery order.
14058
+ _delete_codex_heal_record(conn)
14059
+ if not replay_pending:
14060
+ # #1051: every clear runs as a byte-zero replay, an explicit
14061
+ # `cache-sync --rebuild` included. Arming in the clear's own
14062
+ # transaction is what persists the D1 snapshot below: an
14063
+ # explicit rebuild interrupted after this commit and before
14064
+ # its walk completes leaves the marker and the replay record,
14065
+ # so every retry refills with the persisted `known` instead of
14066
+ # sending pre-mechanism history to the live `auth.json`. The
14067
+ # predicate is marker absence, not `not replay_active`, so a
14068
+ # replay record that lost its marker gets it back too. From
14069
+ # here on this call runs the replay's lifecycle: the
14070
+ # rehydration marks the record, a clean complete walk consumes
14071
+ # both, a blocked one moves the record to `certify`.
14072
+ arm_codex_replay_from_zero(conn)
14073
+ replay_pending = True
14074
+ replay_active = True
14075
+ if replay_active:
14076
+ # #982 spec 4.5: the clear's own transaction writes the replay
14077
+ # heal record with the D1 snapshot, so no later writer clears
14078
+ # again or recaptures it from the cursors this just erased.
14079
+ # The rehydration's transaction below marks it `rehydrated`.
14080
+ _store_codex_heal_record(
14081
+ conn, {"mode": "replay", "phase": "units",
14082
+ "rehydrated": False},
14083
+ population=[str(item.source_path)
14084
+ for item in preflight_files],
14085
+ known=rebuild_known_identities)
12806
14086
  conn.commit()
12807
14087
  eprint("[cache-sync] rebuild: cleared Codex cached entries")
12808
14088
  # #416 spec §3.4: rehydrate the attribution map from the journal BEFORE
@@ -12847,8 +14127,16 @@ def _sync_codex_cache_locked(
12847
14127
  except sqlite3.DatabaseError:
12848
14128
  _cursor_row = None
12849
14129
  _cursor_text = _cursor_row[0] if _cursor_row else None
14130
+ # #982 spec 4.5: a replay record whose rehydration never committed (a
14131
+ # crash between the clear's commit and the rehydration's) is
14132
+ # rehydrated authoritatively first, under these locks, before any
14133
+ # rollout is read; the same transaction marks it `rehydrated`.
14134
+ mark_rehydrated = replay_active and (
14135
+ rebuild or (refill_record is not None
14136
+ and not refill_record.get("rehydrated")))
14137
+ authoritative_rehydration = bool(rebuild) or mark_rehydrated
12850
14138
  _since = None
12851
- if _cursor_text and not rebuild:
14139
+ if _cursor_text and not authoritative_rehydration:
12852
14140
  _seg, _sep, _off = str(_cursor_text).rpartition(":")
12853
14141
  if _seg and _off.isdigit():
12854
14142
  _since = (_seg, int(_off))
@@ -12869,10 +14157,10 @@ def _sync_codex_cache_locked(
12869
14157
  # `database is locked` trigger #297 documents.
12870
14158
  _rehydration = _jr.rehydrate_codex_journal_families(
12871
14159
  conn,
12872
- authoritative=bool(rebuild),
14160
+ authoritative=authoritative_rehydration,
12873
14161
  file_account_since=_since,
12874
14162
  window_attribution_since=(
12875
- None if rebuild
14163
+ None if authoritative_rehydration
12876
14164
  else load_codex_window_attribution_cursor(conn)),
12877
14165
  caller="sync_codex_cache(rebuild=True)",
12878
14166
  )
@@ -12903,7 +14191,15 @@ def _sync_codex_cache_locked(
12903
14191
  # without the caller having to model that function's write
12904
14192
  # conditions. Getting this wrong strands an open transaction across
12905
14193
  # the whole walk.
12906
- if rebuild or _new_cursor != _cursor_text or conn.in_transaction:
14194
+ if mark_rehydrated:
14195
+ rehydrated_record = load_codex_heal_record(conn)
14196
+ if (rehydrated_record is not None
14197
+ and rehydrated_record.get("mode") == "replay"
14198
+ and not rehydrated_record.get("malformed")):
14199
+ rehydrated_record["rehydrated"] = True
14200
+ _store_codex_heal_fence(conn, rehydrated_record)
14201
+ if (authoritative_rehydration or _new_cursor != _cursor_text
14202
+ or conn.in_transaction):
12907
14203
  conn.commit()
12908
14204
  # Both reports come AFTER the commit (closeout review C5): the
12909
14205
  # `except` below rolls back, so anything printed before it would
@@ -12930,6 +14226,10 @@ def _sync_codex_cache_locked(
12930
14226
  "[cache-sync] could not rehydrate Codex attribution decisions "
12931
14227
  f"or window attributions: {exc}; deferring the Codex walk")
12932
14228
  return stats
14229
+ if _replay_stage == "start":
14230
+ # #982 spec 4.5: the start unit ends with the rehydration; the
14231
+ # executor's units refill in the single walk's order.
14232
+ return stats
12933
14233
 
12934
14234
  # #980 D2: a budgeted call settles the finalization debt it inherited
12935
14235
  # BEFORE walking, under the same admission rules, so older debt cannot
@@ -13225,8 +14525,14 @@ def _sync_codex_cache_locked(
13225
14525
  # #980 F3: a budgeted walk bounds the resolver's group seed read with
13226
14526
  # its walk deadline (late-bound clock, so the walk's test seam drives
13227
14527
  # it); an unbudgeted call is unchanged.
13228
- anchor_resolver = CodexResetAnchorResolver(
13229
- conn, deadline=deadline, clock=lambda: _walk_clock())
14528
+ if _walk_share is not None:
14529
+ # #982 spec 4.2: the single walk's own resolver, shared across the
14530
+ # heal's units; kept only while no other writer mutated quota.
14531
+ anchor_resolver = _walk_share.bind_walk(conn)
14532
+ share_bound = True
14533
+ else:
14534
+ anchor_resolver = CodexResetAnchorResolver(
14535
+ conn, deadline=deadline, clock=lambda: _walk_clock())
13230
14536
  # #901 §5.3c (W11, Q14): every call without the hook's budget prepares
13231
14537
  # its delta with no write transaction open and applies it in one
13232
14538
  # transaction, each file in its own savepoint, at most
@@ -13246,6 +14552,16 @@ def _sync_codex_cache_locked(
13246
14552
  # Whole-transaction losses per file, so a persistent one ends.
13247
14553
  w11_losses: "dict[str, int]" = {}
13248
14554
  walk_queue = collections.deque(enumerate(walk))
14555
+ # #982 spec 4.1 steps 4 and 7: a heal walk piece's census (its own
14556
+ # discovery set on the first piece) and the census files it found
14557
+ # unchanged; ``None`` outside a heal walk.
14558
+ heal_census: "frozenset[str] | None" = None
14559
+ heal_unchanged: "set[str]" = set()
14560
+ if _ingest_limit is not None:
14561
+ stats.heal_discovered = frozenset(
14562
+ str(item.source_path) for item in files)
14563
+ heal_census = (stats.heal_discovered if _heal_census is None
14564
+ else frozenset(_heal_census))
13249
14565
 
13250
14566
  def _w11_batch_bytes() -> int:
13251
14567
  return sum(item.new_input for item in w11_batch)
@@ -13495,7 +14811,9 @@ def _sync_codex_cache_locked(
13495
14811
  strike_marks: list[tuple[str, int]] = []
13496
14812
  barrier_holds: list = []
13497
14813
  barrier_touched: set[str] = set()
13498
- history = _CodexHistoryContext(conn)
14814
+ history = _CodexHistoryContext(
14815
+ conn, ledger_source=(None if _walk_share is None
14816
+ else _walk_share.ledger_paths))
13499
14817
  for _walk_index, discovered in _walk_items():
13500
14818
  # The budget is checked BEFORE a file is opened, so a tick either
13501
14819
  # commits a file whole (or to a recorded partial offset) or does not
@@ -13506,12 +14824,28 @@ def _sync_codex_cache_locked(
13506
14824
  break
13507
14825
  jp = discovered.source_path
13508
14826
  path_str = str(jp)
14827
+ if heal_census is not None and (
14828
+ path_str in _heal_visited or path_str not in heal_census):
14829
+ # #982 spec 4.1: an earlier piece of this walk examined the
14830
+ # file (bytes appended since belong to later syncs), or it was
14831
+ # created after the census. Either way the single walk would
14832
+ # not read it again, so it is walked, never examined.
14833
+ stats.files_skipped_unchanged += 1
14834
+ continue
13509
14835
  if path_str in protected_paths:
13510
14836
  # The file is still visible, but its configured root did not
13511
14837
  # yield recognizable Codex data. Treat that as unknown
13512
14838
  # filesystem state: neither truncation/requalification reset
13513
14839
  # nor append ingestion may replace the retained generation.
13514
14840
  continue
14841
+ if path_str in heal_population:
14842
+ # #982 spec 4.3: only the heal executor ingests population
14843
+ # files, in the single walk's order; held like unsettled
14844
+ # history (the worker was requested above).
14845
+ stats.container_reads_held += 1
14846
+ stats.container_skipped["verification-pending"] = (
14847
+ stats.container_skipped.get("verification-pending", 0) + 1)
14848
+ continue
13515
14849
  selected = _select_codex_container(jp)
13516
14850
  if selected is None:
13517
14851
  eprint(f"[codex-cache] stat failed for {jp}: no container")
@@ -13541,7 +14875,7 @@ def _sync_codex_cache_locked(
13541
14875
  # included, is classified like compressed history.
13542
14876
  verdict = history.classify(
13543
14877
  conn, codex_file_identity(discovered), path_str,
13544
- known=rebuild_known_identities if rebuild else (),
14878
+ known=rebuild_known_identities if d1_snapshot else (),
13545
14879
  archive=selected.location == _rc.ARCHIVE)
13546
14880
  if verdict == "unknown":
13547
14881
  # Untraced compressed history waits for the off-hook
@@ -13721,6 +15055,8 @@ def _sync_codex_cache_locked(
13721
15055
  and size == prev_size
13722
15056
  ):
13723
15057
  stats.files_skipped_unchanged += 1
15058
+ if heal_census is not None:
15059
+ heal_unchanged.add(path_str)
13724
15060
  continue
13725
15061
  elif (
13726
15062
  not requalified and not replaced and not incomplete
@@ -13746,6 +15082,32 @@ def _sync_codex_cache_locked(
13746
15082
  initial_total_tokens = 0
13747
15083
  prev_total_tokens = None
13748
15084
 
15085
+ # #982 spec 4.1 steps 4 and 7: a heal walk piece ingests (or
15086
+ # resets) at most `_ingest_limit` files. Reaching the next file
15087
+ # with bytes to ingest or reset stops the piece BEFORE it is read:
15088
+ # it and the rest stay unwalked, as at a budget stop, and the
15089
+ # piece does not certify.
15090
+ if (_ingest_limit is not None
15091
+ and path_str not in stats.heal_counted_paths
15092
+ and len(stats.heal_counted_paths) >= _ingest_limit):
15093
+ requeue = (_w11_flush(final=False) if w11 and w11_batch
15094
+ else [])
15095
+ # W11 counts a file when it is prepared; one whose batch then
15096
+ # failed ingested nothing, so it does not count (the next
15097
+ # piece re-examines it) and this piece goes on.
15098
+ stats.heal_counted_paths &= set(committed_state)
15099
+ if len(stats.heal_counted_paths) < _ingest_limit:
15100
+ if requeue:
15101
+ walk_queue.extendleft(
15102
+ reversed(requeue + [(_walk_index, discovered)]))
15103
+ continue
15104
+ else:
15105
+ stats.heal_limit_stopped = True
15106
+ owed = ({_walk_index}
15107
+ | {index for index, _item in requeue}
15108
+ | {index for index, _item in walk_queue})
15109
+ unwalked = [walk[index] for index in sorted(owed)]
15110
+ break
13749
15111
  # #901 §5.3c (W11): a file's new input is known before it is read.
13750
15112
  # Preparation stops before the file that would take the batch past
13751
15113
  # either bound; that batch is applied and committed, and
@@ -13870,7 +15232,7 @@ def _sync_codex_cache_locked(
13870
15232
  # an ascending list, so the appended low offset would then shadow
13871
15233
  # the real decision for every later byte in the file.
13872
15234
  file_account_key = None
13873
- elif rebuild and file_identity in rebuild_known_identities:
15235
+ elif d1_snapshot and file_identity in rebuild_known_identities:
13874
15236
  # Spec D1: history that was never durably stamped becomes
13875
15237
  # unattributed; nothing is inferred. A rebuild re-reads bytes
13876
15238
  # that were ingested BEFORE this mechanism existed, and reading
@@ -14463,6 +15825,7 @@ def _sync_codex_cache_locked(
14463
15825
  inode=st.st_ino,
14464
15826
  container=container_acceptance,
14465
15827
  barrier=barrier_read,
15828
+ thread_guard=heal_population,
14466
15829
  ),
14467
15830
  merges=anchor_resolver.take_pending_merges(),
14468
15831
  reset_counted=bool(truncated or requalified),
@@ -14477,6 +15840,8 @@ def _sync_codex_cache_locked(
14477
15840
  skip_reasons=dict(iter_state.skip_reasons),
14478
15841
  ))
14479
15842
  _codex_w11_trace("prepare_end", path_str, new_input)
15843
+ if _ingest_limit is not None:
15844
+ stats.heal_counted_paths.add(path_str)
14480
15845
  # Private test seam: after a file is prepared and before the
14481
15846
  # next file's preparation, which under W11 is where a later
14482
15847
  # target's turn begins (its commit comes with its batch's).
@@ -14535,6 +15900,7 @@ def _sync_codex_cache_locked(
14535
15900
  inode=st.st_ino,
14536
15901
  container=container_acceptance,
14537
15902
  barrier=barrier_read,
15903
+ thread_guard=heal_population,
14538
15904
  )
14539
15905
  except sqlite3.DatabaseError as exc:
14540
15906
  conn.rollback()
@@ -14567,6 +15933,8 @@ def _sync_codex_cache_locked(
14567
15933
  anchor_resolver.discard_uncommitted_file()
14568
15934
  continue
14569
15935
  anchor_resolver.mark_file_committed()
15936
+ if _ingest_limit is not None:
15937
+ stats.heal_counted_paths.add(path_str)
14570
15938
  # #901 §5.3c (W9): the size trigger is evaluated after each commit.
14571
15939
  _wal_checkpoint.after_commit(conn)
14572
15940
  committed_state[path_str] = (
@@ -14625,6 +15993,24 @@ def _sync_codex_cache_locked(
14625
15993
  # points: the pre-open deadline check fills `unwalked`, while a stop
14626
15994
  # INSIDE a file only sets the flag.
14627
15995
  walk_complete = not stats.budget_exhausted and not unwalked
15996
+ # #982 spec 4.1 steps 4 and 7 (revision 7): the piece that completes
15997
+ # a heal walk certifies a file an earlier piece visited only while its
15998
+ # committed cursor row still equals the one recorded at that visit.
15999
+ # A file another writer changed since (a hook that ingested part of an
16000
+ # append and stopped at its budget, say) still counts as walked, but
16001
+ # stays in this piece's backlog, makes the barrier census 0 and
16002
+ # withholds the full-walk certificate, exactly as a single walk
16003
+ # followed by that writer would leave them; it is not re-examined.
16004
+ visit_changed: "list[CodexDiscoveredFile]" = []
16005
+ if heal_census is not None and walk_complete and _heal_visited_rows:
16006
+ current_rows = _load_codex_session_files_rows(
16007
+ conn, sorted(_heal_visited_rows))
16008
+ changed = frozenset(
16009
+ path for path, row in _heal_visited_rows.items()
16010
+ if current_rows.get(path) != row)
16011
+ stats.heal_visit_changed = changed
16012
+ visit_changed = [item for item in files
16013
+ if str(item.source_path) in changed]
14628
16014
  _p_walk.set_count(stats.files_processed)
14629
16015
  _p_walk.set_meta(skipped=stats.files_skipped_unchanged,
14630
16016
  rows=stats.rows_changed)
@@ -14671,7 +16057,8 @@ def _sync_codex_cache_locked(
14671
16057
  else (unwalked[0] if unwalked else None))
14672
16058
  backlog_files, backlog_bytes = _codex_backlog_after(
14673
16059
  partial_files + unwalked
14674
- + [item.discovered for item in withdrawn],
16060
+ + [item.discovered for item in withdrawn]
16061
+ + visit_changed,
14675
16062
  existing, committed_state)
14676
16063
  stats.backlog_files = backlog_files
14677
16064
  stats.backlog_bytes = backlog_bytes
@@ -14750,8 +16137,32 @@ def _sync_codex_cache_locked(
14750
16137
  elif walk_complete:
14751
16138
  conn.execute("DELETE FROM cache_meta WHERE key = ?",
14752
16139
  ("codex_torn_auth_deferred",))
16140
+ # #982 spec 4.1 step 5: the transition. The fenced walk's
16141
+ # piece that leaves no census file unwalked ends a history
16142
+ # heal's fence here, in its own end-of-walk transaction
16143
+ # (before its settlement, which the executor defers to the
16144
+ # finalization units anyway), whether or not it failed a file
16145
+ # or deferred one for a torn `auth.json`: such a file stays
16146
+ # owed exactly as the single walk leaves it, and the census
16147
+ # below, set from this piece's counters, keeps it blocking
16148
+ # new seals. (A replay heal keeps the consumption conditions
16149
+ # of the block after this one.) A run that later withdraws
16150
+ # its last batch repeats this step with `walk_complete` False
16151
+ # after the rollback undid the delete.
16152
+ if _heal_transition:
16153
+ stats.heal_transitioned = False
16154
+ if walk_complete:
16155
+ transition_record = load_codex_heal_record(conn)
16156
+ if (transition_record is not None
16157
+ and transition_record.get("mode") == "history"
16158
+ and transition_record.get("gen")
16159
+ == _heal_generation):
16160
+ _delete_codex_heal_record(conn)
16161
+ stats.heal_transitioned = True
14753
16162
  # Consume the byte-zero replay marker only after a clean full walk,
14754
- # and only when THIS call observed it. A contended call returned
16163
+ # and only when THIS call observed it or armed it itself (#1051:
16164
+ # every clear arms the marker in its own transaction, so an
16165
+ # explicit rebuild consumes what it armed). A contended call returned
14755
16166
  # long before here, and a walk that failed or deferred a file leaves
14756
16167
  # the marker standing — a surviving marker is what makes the repair
14757
16168
  # retry on the next sync, and it is also what keeps
@@ -14769,6 +16180,19 @@ def _sync_codex_cache_locked(
14769
16180
  # file. It is defense in depth today — a budgeted tick declines a
14770
16181
  # pending replay outright above — but the two clears must keep the
14771
16182
  # same shape.
16183
+ # #982 spec 4.5a (Generation): an executor walk consumes the
16184
+ # marker, ends a replay record or moves it to `certify` only
16185
+ # while the record is still its pass's, re-read here under the
16186
+ # same locks (the preamble already returned otherwise).
16187
+ heal_generation_ok = True
16188
+ if _walk_share is not None:
16189
+ heal_generation_ok = not _codex_heal_generation_stale(
16190
+ load_codex_heal_record(conn),
16191
+ conn.execute(
16192
+ "SELECT 1 FROM cache_meta WHERE key=? LIMIT 1",
16193
+ (CODEX_REPLAY_FROM_ZERO_KEY,)).fetchone()
16194
+ is not None,
16195
+ _heal_generation)
14772
16196
  if (
14773
16197
  walk_complete
14774
16198
  and stats.files_failed == 0
@@ -14786,17 +16210,26 @@ def _sync_codex_cache_locked(
14786
16210
  )
14787
16211
  ):
14788
16212
  _clear_codex_prune_refusal(conn)
14789
- if replay_pending:
16213
+ if replay_pending and heal_generation_ok:
14790
16214
  conn.execute("DELETE FROM cache_meta WHERE key = ?",
14791
16215
  (CODEX_REPLAY_FROM_ZERO_KEY,))
14792
- conn.execute("DELETE FROM cache_meta WHERE key = ?",
14793
- (CODEX_REPLAY_BLOCKED_KEY,))
14794
- # The budgeted-decline record is cleared by the same walk, for
14795
- # the same reason: an unbudgeted walk reaching here is exactly
14796
- # the caller whose absence the record was reporting.
14797
- conn.execute("DELETE FROM cache_meta WHERE key = ?",
14798
- (CODEX_REPLAY_DEFERRED_KEY,))
14799
- elif replay_pending and walk_complete:
16216
+ # #982 spec 4.1 step 5: a replay heal record ends with
16217
+ # the marker, in the same transaction.
16218
+ consumed_record = load_codex_heal_record(conn)
16219
+ if (heal_generation_ok and consumed_record is not None
16220
+ and consumed_record.get("mode") == "replay"):
16221
+ _delete_codex_heal_record(conn)
16222
+ stats.heal_transitioned = _heal_transition
16223
+ if heal_generation_ok:
16224
+ conn.execute("DELETE FROM cache_meta WHERE key = ?",
16225
+ (CODEX_REPLAY_BLOCKED_KEY,))
16226
+ # The budgeted-decline record is cleared by the same
16227
+ # walk, for the same reason: an unbudgeted walk
16228
+ # reaching here is exactly the caller whose absence
16229
+ # the record was reporting.
16230
+ conn.execute("DELETE FROM cache_meta WHERE key = ?",
16231
+ (CODEX_REPLAY_DEFERRED_KEY,))
16232
+ elif replay_pending and walk_complete and heal_generation_ok:
14800
16233
  # A full walk ran and could NOT consume the marker, so the
14801
16234
  # replay — and with it every Codex transcript ingest, which
14802
16235
  # defers behind this marker — is stalled rather than merely
@@ -14808,6 +16241,15 @@ def _sync_codex_cache_locked(
14808
16241
  "files_failed": stats.files_failed,
14809
16242
  "files_deferred_torn": stats.files_deferred_torn,
14810
16243
  }, sort_keys=True))
16244
+ # #982 spec 4.1 step 5: a blocked replay ends its fence
16245
+ # (phase `certify`) and keeps `known`; nothing re-clears.
16246
+ blocked_record = load_codex_heal_record(conn)
16247
+ if (blocked_record is not None
16248
+ and blocked_record.get("mode") == "replay"
16249
+ and blocked_record.get("phase") == "units"):
16250
+ blocked_record["phase"] = "certify"
16251
+ _store_codex_heal_fence(conn, blocked_record)
16252
+ stats.heal_transitioned = _heal_transition
14811
16253
  conn.execute(_CODEX_FINALIZATION_RELEASE)
14812
16254
  # #980 D3/D4: settle the persisted finalization debt — the debt
14813
16255
  # this call's batches wrote and any debt inherited — in order:
@@ -14837,15 +16279,21 @@ def _sync_codex_cache_locked(
14837
16279
  # a later call (or the executor) revisits it; a budgeted call
14838
16280
  # admits a unit only while the budget leaves room, and a unit that
14839
16281
  # reaches the deadline rolls back and is never reported as a zero.
16282
+ settle_stages = (
16283
+ {"stages": _CODEX_HEAL_FENCED_STAGES} if _defer_finalization
16284
+ else {})
14840
16285
  outcome = _codex_settle_finalization(
14841
16286
  conn, deadline=budget_deadline, clock=_finalization_clock,
14842
- core_model=True, strike_state=strike_state)
16287
+ core_model=True, strike_state=strike_state, **settle_stages)
14843
16288
  for message in outcome.errors:
14844
16289
  eprint(message)
14845
16290
  if (outcome.model_failed or outcome.adoption_failed
14846
16291
  or outcome.reconcile_failed):
14847
16292
  stats.maintenance_failed = True
14848
16293
  if outcome.cancelled or outcome.deferred_to_worker or (
16294
+ # #982: a fenced or deferred settlement leaves debt owed.
16295
+ outcome.fenced or (
16296
+ _defer_finalization and not outcome.completed)) or (
14849
16297
  budget_deadline is not None and not outcome.completed):
14850
16298
  # Cancelled, handed to the worker, or a unit the remaining
14851
16299
  # budget could not admit: the persisted debt stays owed.
@@ -14870,7 +16318,11 @@ def _sync_codex_cache_locked(
14870
16318
  # prunes; when the prune is due it records it as owed maintenance
14871
16319
  # for the drain executor instead.
14872
16320
  with _codex_finalization_stage(conn):
14873
- if deadline is None:
16321
+ # #982 spec 4.3: during the heal fence (and in a heal round
16322
+ # that defers its settlement) a due prune stays owed for the
16323
+ # executor instead.
16324
+ if (deadline is None and not _defer_finalization
16325
+ and not _codex_heal_fence_active(conn)):
14874
16326
  _prune_codex_accounting_change_log(conn)
14875
16327
  elif _codex_change_log_prune_due(conn):
14876
16328
  record_codex_owed_maintenance(conn, change_log_prune=True)
@@ -14889,6 +16341,13 @@ def _sync_codex_cache_locked(
14889
16341
  stats.maintenance_failed = False
14890
16342
  finalization_notes.clear()
14891
16343
  _finalization_steps(False, w11_withdrawn)
16344
+ if heal_census is not None:
16345
+ # #982 spec 4.1: committed and unchanged census files are
16346
+ # visited; a withdrawn batch's files are not (`committed_state`
16347
+ # forgot them), so the next piece examines them again.
16348
+ stats.heal_examined = frozenset(
16349
+ (heal_unchanged | set(committed_state)) & heal_census)
16350
+ stats.heal_walk_complete = walk_complete and not w11_lost
14892
16351
  # Projection intentionally runs only after this function releases the
14893
16352
  # Codex cache flock in ``finally`` below. cache.db and stats.db are not
14894
16353
  # cross-database atomic: after this committed ingest, a projection
@@ -14975,6 +16434,7 @@ def _sync_codex_cache_locked(
14975
16434
  and not stats.files_deferred_reset
14976
16435
  and not stats.container_pending
14977
16436
  and not stats.container_reads_held
16437
+ and not stats.heal_visit_changed
14978
16438
  and (
14979
16439
  stats.files_processed + stats.files_skipped_unchanged
14980
16440
  + sum(stats.container_skipped.values())
@@ -15016,6 +16476,7 @@ def _sync_codex_cache_locked(
15016
16476
  barrier_census = bool(
15017
16477
  walk_complete and stats.files_failed == 0
15018
16478
  and stats.files_deferred_torn == 0 and not w11_lost
16479
+ and not stats.heal_visit_changed
15019
16480
  and stats.files_processed + stats.files_skipped_unchanged
15020
16481
  + sum(stats.container_skipped.values())
15021
16482
  + stats.container_pending == stats.files_total)
@@ -15083,6 +16544,11 @@ def _sync_codex_cache_locked(
15083
16544
  # one recorded the drain as owed above.
15084
16545
  if deadline is None:
15085
16546
  _maybe_truncate_wal(conn, _cctally_core.CACHE_DB_PATH)
16547
+ if heal_census is not None and stats.heal_examined:
16548
+ # Revision 7: what this piece visited, as committed, read after
16549
+ # its last write and before its locks are released.
16550
+ stats.heal_visit_rows = _load_codex_session_files_rows(
16551
+ conn, sorted(stats.heal_examined))
15086
16552
  except BaseException:
15087
16553
  raised = True
15088
16554
  raise
@@ -15126,7 +16592,16 @@ def _sync_codex_cache_locked(
15126
16592
  raise
15127
16593
  if read_policy is not None and raised:
15128
16594
  read_policy.invalidate("cache_unavailable")
16595
+ if share_bound and not raised:
16596
+ # #982 spec 4.2: after this unit's last commit, before its
16597
+ # locks are released: the next unit reuses the memo only if
16598
+ # nobody mutated quota in between.
16599
+ _walk_share.seal(conn)
16600
+ share_bound = False
15129
16601
  finally:
16602
+ if share_bound:
16603
+ # An exception (or a failure above): the next unit reseeds.
16604
+ _walk_share.invalidate()
15130
16605
  # #981 R4: released on every path, whatever the pin raised.
15131
16606
  release_cache_writer_flocks(held_writer_flocks)
15132
16607
 
@@ -15342,9 +16817,299 @@ class FinalizationOutcome:
15342
16817
  restored: int = 0
15343
16818
  readopted: int = 0
15344
16819
  deferred_to_worker: bool = False
16820
+ #: #982 spec 4.3: the heal fence restricted this settlement to the
16821
+ #: ``expand`` and ``model`` stages; callers treat it like a deferred unit.
16822
+ fenced: bool = False
15345
16823
  errors: list = field(default_factory=list)
15346
16824
 
15347
16825
 
16826
+ # --------------------------------------------------------------------------
16827
+ # #982 / #975 — the Codex heal record and its fence (spec 4.3)
16828
+ # --------------------------------------------------------------------------
16829
+ #
16830
+ # The heal record is two ``cache_meta`` keys written and retired together:
16831
+ #
16832
+ # * ``codex_heal_walk``, the small FENCE key ``{"v", "gen", "mode", "phase"}``
16833
+ # plus, in replay mode, ``"rehydrated"`` and, once a later arming site
16834
+ # re-arms the replay, ``"reclear"``. Every fence check reads only this key.
16835
+ # * ``codex_heal_walk_payload``, the history-sized half ``{"v", "gen",
16836
+ # "population"}`` plus, in replay mode, ``"known"`` (the D1 snapshot). It is
16837
+ # parsed at most once per sync call, or once per executor pass (the walk
16838
+ # share hands it to the units), and bound to the fence key by ``gen``.
16839
+ #
16840
+ # Both keys are written once by the heal executor's start unit (or the
16841
+ # replay's clear) in one transaction and deleted together once by the
16842
+ # transition (the first fenced certification round that walks clean and
16843
+ # complete); they are never rewritten per unit. Only the fence key changes in
16844
+ # between: the rehydration sets ``rehydrated``, a blocked replay moves it to
16845
+ # phase ``certify``, and ``arm_codex_replay_from_zero`` sets ``reclear``.
16846
+ # While the fence key holds phase ``units`` (the fence):
16847
+ #
16848
+ # * only the executor's units (syncs carrying its ``_walk_share``) ingest a
16849
+ # population file; every other sync holds them and requests the worker;
16850
+ # * every one-way stamping path is restricted: the settlement chokepoint runs
16851
+ # only ``expand`` and ``model`` and reports ``fenced``, and the two stamping
16852
+ # kernels stamp nothing (or raise ``CodexHealFenced`` when strict);
16853
+ # * a non-executor thread-row upsert never replaces a population file's row;
16854
+ # * the accounting change-log prune deletes nothing and stays owed.
16855
+
16856
+ CODEX_HEAL_WALK_KEY = "codex_heal_walk"
16857
+ CODEX_HEAL_PAYLOAD_KEY = "codex_heal_walk_payload"
16858
+ CODEX_HEAL_WALK_LOCK_NAME = "codex-heal-walk.lock"
16859
+ CODEX_HEAL_FENCED_TOKEN = "deferred_heal"
16860
+ _CODEX_HEAL_RECORD_VERSION = 1
16861
+ _CODEX_HEAL_FENCED_STAGES = ("expand", "model")
16862
+
16863
+
16864
+ class CodexHealFenced(sqlite3.DatabaseError):
16865
+ """A strict stamping kernel refused while the heal fence holds.
16866
+
16867
+ A ``sqlite3.DatabaseError`` whose text is the status token
16868
+ ``deferred_heal``, mirroring ``deferred_model``: ``account attribute``'s
16869
+ all-or-nothing apply rolls its cache transaction back and reaches its
16870
+ existing ``recordedPending`` outcome, and a rerun after the heal
16871
+ completes it."""
16872
+
16873
+
16874
+ class CodexHealPayload(NamedTuple):
16875
+ """The history-sized half of a heal record, parsed once."""
16876
+
16877
+ gen: str
16878
+ population: "frozenset[str]"
16879
+ known: "frozenset[str] | None" # replay mode only (the D1 snapshot)
16880
+
16881
+
16882
+ def _codex_heal_meta_value(conn: sqlite3.Connection, key: str):
16883
+ try:
16884
+ row = conn.execute("SELECT value FROM cache_meta WHERE key=?",
16885
+ (key,)).fetchone()
16886
+ except sqlite3.OperationalError as exc:
16887
+ if "no such table" in str(exc):
16888
+ return None
16889
+ raise
16890
+ return None if row is None else row[0]
16891
+
16892
+
16893
+ def load_codex_heal_record(conn: sqlite3.Connection) -> "dict | None":
16894
+ """The heal record's small fence key, or None. Never the payload.
16895
+
16896
+ Unreadable fails closed. A record nobody can parse is a fence over an
16897
+ unknown population marked ``malformed``: in history mode (the next
16898
+ executor's transition removes it), or in replay mode when the record
16899
+ says so or the byte-zero replay marker is armed, because a replay record
16900
+ that lost its D1 set must never be cleared again with a recaptured one
16901
+ (spec 4.5a): writers decline and the executor ends ``degraded``."""
16902
+ text = _codex_heal_meta_value(conn, CODEX_HEAL_WALK_KEY)
16903
+ if text is None:
16904
+ return None
16905
+ try:
16906
+ record = json.loads(str(text))
16907
+ except (TypeError, ValueError):
16908
+ record = None
16909
+ if (isinstance(record, dict)
16910
+ and record.get("v") == _CODEX_HEAL_RECORD_VERSION
16911
+ and isinstance(record.get("gen"), str)
16912
+ and record.get("mode") in ("history", "replay")
16913
+ and record.get("phase") in ("units", "certify")
16914
+ and isinstance(record.get("reclear", False), bool)
16915
+ and (record.get("mode") != "replay"
16916
+ or isinstance(record.get("rehydrated"), bool))):
16917
+ return record
16918
+ replay = (isinstance(record, dict) and record.get("mode") == "replay")
16919
+ if not replay:
16920
+ try:
16921
+ replay = conn.execute(
16922
+ "SELECT 1 FROM cache_meta WHERE key=? LIMIT 1",
16923
+ (CODEX_REPLAY_FROM_ZERO_KEY,)).fetchone() is not None
16924
+ except sqlite3.OperationalError:
16925
+ replay = False
16926
+ return {"v": _CODEX_HEAL_RECORD_VERSION, "gen": None,
16927
+ "mode": "replay" if replay else "history", "phase": "units",
16928
+ "malformed": True}
16929
+
16930
+
16931
+ def _codex_replay_refill_unlocked(conn: sqlite3.Connection) -> bool:
16932
+ """Would a budgeted sync decline as a replay refill (#982 spec 4.4)?
16933
+
16934
+ Read without the store locks by a budgeted call that met contention: a
16935
+ replay heal record in phase ``units``, or one too malformed to tell, is
16936
+ the state in which no budgeted tick ingests, projects or evaluates a
16937
+ budget (the predicate the decline sets ``replay_refill`` from). A waiting
16938
+ request could then only make the refill units idle for its lifetime.
16939
+ A marker without a record (the start unit has not committed) and a
16940
+ blocked replay's ``certify`` record answer False: their hooks still
16941
+ project, so their requests are kept. Unreadable answers False, so the
16942
+ hook publishes its request, as today."""
16943
+ try:
16944
+ record = load_codex_heal_record(conn)
16945
+ except sqlite3.Error:
16946
+ return False
16947
+ return (record is not None and record.get("mode") == "replay"
16948
+ and (bool(record.get("malformed"))
16949
+ or record.get("phase") == "units"))
16950
+
16951
+
16952
+ def load_codex_heal_payload(
16953
+ conn: sqlite3.Connection,
16954
+ record: "dict | None") -> "CodexHealPayload | None":
16955
+ """Parse the record's payload: the population and, in replay mode, the
16956
+ D1 set. None when the record is malformed or the payload is missing,
16957
+ malformed or another record's (its ``gen`` differs); callers fail closed
16958
+ (a history fence over an empty population; a replay that declines)."""
16959
+ if record is None or record.get("malformed"):
16960
+ return None
16961
+ text = _codex_heal_meta_value(conn, CODEX_HEAL_PAYLOAD_KEY)
16962
+ if text is None:
16963
+ return None
16964
+ try:
16965
+ raw = json.loads(str(text))
16966
+ except (TypeError, ValueError):
16967
+ return None
16968
+ if not (isinstance(raw, dict)
16969
+ and raw.get("v") == _CODEX_HEAL_RECORD_VERSION
16970
+ and raw.get("gen") == record.get("gen")):
16971
+ return None
16972
+ population = raw.get("population")
16973
+ if not (isinstance(population, list)
16974
+ and all(isinstance(p, str) for p in population)):
16975
+ return None
16976
+ known = None
16977
+ if record.get("mode") == "replay":
16978
+ known = raw.get("known")
16979
+ if not (isinstance(known, list)
16980
+ and all(isinstance(k, str) for k in known)):
16981
+ return None
16982
+ known = frozenset(known)
16983
+ return CodexHealPayload(str(raw["gen"]), frozenset(population), known)
16984
+
16985
+
16986
+ def _codex_heal_fence_active(conn: sqlite3.Connection) -> bool:
16987
+ """The fence: a heal record in phase ``units``."""
16988
+ record = load_codex_heal_record(conn)
16989
+ return record is not None and record.get("phase") == "units"
16990
+
16991
+
16992
+ def _codex_heal_fence_json(record: dict) -> str:
16993
+ fence = {key: value for key, value in record.items()
16994
+ if key not in ("malformed", "population", "known")}
16995
+ return json.dumps(fence, sort_keys=True, separators=(",", ":"))
16996
+
16997
+
16998
+ def _store_codex_heal_record(conn: sqlite3.Connection, record: dict, *,
16999
+ population, known=None) -> dict:
17000
+ """Write a NEW record, both keys under a fresh ``gen``, on the caller's
17001
+ transaction (the history start unit, the replay clear). Returns the
17002
+ fence key as written."""
17003
+ fence = {key: value for key, value in record.items()
17004
+ if key not in ("malformed", "population", "known", "reclear")}
17005
+ fence["v"] = _CODEX_HEAL_RECORD_VERSION
17006
+ fence["gen"] = uuid.uuid4().hex
17007
+ payload = {"v": _CODEX_HEAL_RECORD_VERSION, "gen": fence["gen"],
17008
+ "population": sorted(set(str(p) for p in population))}
17009
+ if fence.get("mode") == "replay":
17010
+ payload["known"] = sorted(set(known or ()))
17011
+ conn.execute(
17012
+ "INSERT OR REPLACE INTO cache_meta(key, value) VALUES (?, ?)",
17013
+ (CODEX_HEAL_PAYLOAD_KEY,
17014
+ json.dumps(payload, sort_keys=True, separators=(",", ":"))))
17015
+ conn.execute(
17016
+ "INSERT OR REPLACE INTO cache_meta(key, value) VALUES (?, ?)",
17017
+ (CODEX_HEAL_WALK_KEY, _codex_heal_fence_json(fence)))
17018
+ return fence
17019
+
17020
+
17021
+ def _store_codex_heal_fence(conn: sqlite3.Connection, record: dict) -> None:
17022
+ """Rewrite only the fence key of an existing record (its ``gen`` and
17023
+ payload unchanged): ``rehydrated``, phase ``certify``, ``reclear``."""
17024
+ conn.execute(
17025
+ "INSERT OR REPLACE INTO cache_meta(key, value) VALUES (?, ?)",
17026
+ (CODEX_HEAL_WALK_KEY, _codex_heal_fence_json(record)))
17027
+
17028
+
17029
+ def _delete_codex_heal_record(conn: sqlite3.Connection) -> None:
17030
+ conn.execute("DELETE FROM cache_meta WHERE key IN (?, ?)",
17031
+ (CODEX_HEAL_WALK_KEY, CODEX_HEAL_PAYLOAD_KEY))
17032
+
17033
+
17034
+ def arm_codex_replay_from_zero(conn: sqlite3.Connection) -> None:
17035
+ """Arm the guarded byte-zero Codex replay, on the caller's transaction.
17036
+
17037
+ Every arming site goes through here (spec 4.5a). Without a replay heal
17038
+ record this writes ``CODEX_REPLAY_FROM_ZERO_KEY`` and nothing else. With
17039
+ one (a refill in progress or a blocked replay's ``certify`` record) the
17040
+ clear already happened, so a bare marker would only be refilled and the
17041
+ repair it was armed for would never re-derive the rows already refilled.
17042
+ The record is marked ``reclear`` instead: the next writer clears again
17043
+ exactly as an armed marker does today, with the fresh D1 capture united
17044
+ with the record's persisted ``known`` (a rollout not yet refilled keeps
17045
+ its pre-mechanism identity, I7), and writes a new record. A malformed
17046
+ record is left as it is: it already fails closed.
17047
+ """
17048
+ _set_cache_meta(conn, CODEX_REPLAY_FROM_ZERO_KEY, "1")
17049
+ record = load_codex_heal_record(conn)
17050
+ if (record is not None and record.get("mode") == "replay"
17051
+ and not record.get("malformed") and not record.get("reclear")):
17052
+ record["reclear"] = True
17053
+ _store_codex_heal_fence(conn, record)
17054
+
17055
+
17056
+ def _codex_replay_reclear_due(record: "dict | None",
17057
+ replay_armed: bool) -> bool:
17058
+ """Whether a replay record's ``reclear`` mark asks for a fresh clear.
17059
+
17060
+ ``arm_codex_replay_from_zero`` writes the mark with the replay marker in
17061
+ one transaction, and the consumption block deletes the marker with the
17062
+ record, so the mark acts only while its marker is armed. A mark whose
17063
+ marker is gone (only a damaged ``cache_meta`` has one) asks for nothing:
17064
+ every writer treats the record as one that no longer needs a replay,
17065
+ refills with its persisted D1 set and retires it, and none clears."""
17066
+ return bool(replay_armed and record is not None
17067
+ and record.get("mode") == "replay" and record.get("reclear"))
17068
+
17069
+
17070
+ def _codex_heal_generation_stale(record: "dict | None", replay_armed: bool,
17071
+ generation: "str | None") -> bool:
17072
+ """#982 spec 4.5a (Generation): whether an executor unit of the pass
17073
+ that started with record generation ``generation`` must write nothing.
17074
+
17075
+ True when the store now asks for a fresh clear, which only the start
17076
+ unit of a new pass (or an unbudgeted ordinary sync outside any pass) may
17077
+ perform: the replay marker is armed without a replay record, or the
17078
+ replay record is marked ``reclear`` with that marker armed
17079
+ (``_codex_replay_reclear_due``). Also true when the record is not
17080
+ the pass's (another ``gen``). An unreadable record has no ``gen``, so it
17081
+ is the pass's only when the pass started from it (a malformed history
17082
+ record, which that pass's transition removes; a malformed replay record
17083
+ still declines every writer below). A missing record with no armed
17084
+ marker is not stale: the pass's transition (or another writer's
17085
+ complete walk) ended it, and the finalization units and the closing walk
17086
+ run after that."""
17087
+ if replay_armed and (record is None or record.get("mode") != "replay"):
17088
+ return True
17089
+ if record is None:
17090
+ return False
17091
+ return (_codex_replay_reclear_due(record, replay_armed)
17092
+ or record.get("gen") != generation)
17093
+
17094
+
17095
+ def _codex_heal_fenced_kernel(conn: sqlite3.Connection, strict: bool) -> bool:
17096
+ """The stamping kernels' own check (spec 4.3): True means "stamp nothing";
17097
+ strict callers get ``CodexHealFenced`` instead."""
17098
+ try:
17099
+ fenced = _codex_heal_fence_active(conn)
17100
+ except sqlite3.DatabaseError as exc:
17101
+ if (isinstance(exc, sqlite3.OperationalError)
17102
+ and _is_sqlite_interrupt(exc)):
17103
+ # A cancellation, never a swallowable read error (D4).
17104
+ raise CodexFinalizationCancelled("interrupt") from exc
17105
+ if strict:
17106
+ raise
17107
+ return True
17108
+ if fenced and strict:
17109
+ raise CodexHealFenced(CODEX_HEAL_FENCED_TOKEN)
17110
+ return fenced
17111
+
17112
+
15348
17113
  def _codex_malformed_debt() -> CodexFinalizationDebt:
15349
17114
  return CodexFinalizationDebt(
15350
17115
  full=True, model=True, attribution=True, malformed=True)
@@ -15993,6 +17758,13 @@ def _codex_settle_finalization(
15993
17758
  """
15994
17759
  outcome = FinalizationOutcome()
15995
17760
  budgeted = deadline is not None
17761
+ if _codex_heal_fence_active(conn):
17762
+ # #982 spec 4.3: while a heal holds its population, adoption and
17763
+ # reconciliation (one-way NULL -> account stamps) would decide from a
17764
+ # partly ingested store. Expansion and model resolution are safe:
17765
+ # model resolution reads only the observation's own rollout.
17766
+ stages = tuple(s for s in stages if s in _CODEX_HEAL_FENCED_STAGES)
17767
+ outcome.fenced = True
15996
17768
  if strike_state is None:
15997
17769
  # One call, one strike: a caller settling in several steps (the sync's
15998
17770
  # pre-walk settle and its finalization) shares one state instead.
@@ -16359,6 +18131,9 @@ def apply_codex_window_spend_adoption(
16359
18131
  high = max(span[1] for span in spans.values())
16360
18132
  reset_bounds = (low - _CODEX_ACCOUNT_WEEK, high + _CODEX_ACCOUNT_WEEK)
16361
18133
  candidate_bounds = (low - _CODEX_ACCOUNT_WEEK, high)
18134
+ # #982 spec 4.3: no one-way stamp while a heal holds its population.
18135
+ if _codex_heal_fenced_kernel(conn, strict):
18136
+ return 0
16362
18137
 
16363
18138
  from _cctally_quota import load_codex_quota_observations
16364
18139
 
@@ -16501,6 +18276,59 @@ def _codex_stored_identity_path(
16501
18276
  return _codex_identity_path(selected, _codex_root_for_logical(logical, roots))
16502
18277
 
16503
18278
 
18279
+ def _codex_legacy_null_root_owner(
18280
+ path: str,
18281
+ discovered_by_identity: "dict[str, CodexDiscoveredFile]",
18282
+ roots: "list[CodexProviderRoot]",
18283
+ ) -> "tuple[str | None, CodexDiscoveredFile | None]":
18284
+ """#983 spec 5.2: ``(root_key, discovered_file | None)`` for a stored path
18285
+ whose row carries no ``source_root_key`` (pre-#294 S1), or ``(None, None)``.
18286
+
18287
+ Ownership follows discovery's own answer, never literal containment of the
18288
+ stored spelling, because a symlinked spelling, overlapping roots and the
18289
+ #970 archive release all defeat that. In order:
18290
+
18291
+ 1. the stored identity names a discovered file: that file's root, and the
18292
+ file itself so the rebuild's D1 snapshot gains exactly the identity the
18293
+ walk computes for it;
18294
+ 2. otherwise the first root in configured order whose CANONICALIZED walk
18295
+ or archive scope contains the identity (``_codex_scope_root`` would
18296
+ compare the configured spelling, which misses a symlinked root);
18297
+ 3. otherwise a configured root ``_codex_provider_roots`` omitted (an
18298
+ unmounted volume, a missing directory), keyed as ``_codex_prune_scope``
18299
+ keys it, so the scope refuses it as configured but unrecognizable;
18300
+ 4. otherwise no owner: the path is outside every configured root and stays
18301
+ removable, as a root-keyed row whose root left the configured set is.
18302
+ """
18303
+ try:
18304
+ phys = _codex_stored_identity_path(path, roots)
18305
+ except (OSError, TypeError, ValueError):
18306
+ return None, None
18307
+ discovered = discovered_by_identity.get(str(phys))
18308
+ if discovered is not None:
18309
+ return discovered.source_root_key, discovered
18310
+ for root in roots:
18311
+ scopes = [root.walk_root]
18312
+ if root.archive_root is not None:
18313
+ scopes.append(root.archive_root)
18314
+ for scope in scopes:
18315
+ try:
18316
+ if phys.is_relative_to(_canonical_codex_path(scope)):
18317
+ return root.source_root_key, None
18318
+ except (OSError, TypeError, ValueError):
18319
+ continue
18320
+ walked = {root.source_root_key for root in roots}
18321
+ for raw_root in _cctally()._codex_home_roots():
18322
+ try:
18323
+ canonical = _canonical_codex_path(raw_root)
18324
+ key = source_root_key(str(canonical))
18325
+ if key not in walked and phys.is_relative_to(canonical):
18326
+ return key, None
18327
+ except (OSError, TypeError, ValueError):
18328
+ continue
18329
+ return None, None
18330
+
18331
+
16504
18332
 
16505
18333
  def codex_file_key_for_entry_path(
16506
18334
  source_root_key: str, source_path: str, *,
@@ -16620,9 +18448,9 @@ class _CodexHistoryContext:
16620
18448
  """One walk's provenance state, read once before the walk loop."""
16621
18449
 
16622
18450
  __slots__ = ("payload", "traced", "settle_on_finish", "ledger_paths",
16623
- "consulted", "archives_covered")
18451
+ "consulted", "archives_covered", "ledger_source")
16624
18452
 
16625
- def __init__(self, conn):
18453
+ def __init__(self, conn, *, ledger_source=None):
16626
18454
  import _cctally_journal as _jr
16627
18455
  self.payload = _codex_history_marker(conn)
16628
18456
  self.archives_covered = _codex_history_marker_covers_archives(
@@ -16635,6 +18463,9 @@ class _CodexHistoryContext:
16635
18463
  and _jr.journal_high_water() is None)
16636
18464
  self.ledger_paths = None
16637
18465
  self.consulted = False
18466
+ # #982 spec 4.2: a heal unit reads the ledger path set through its
18467
+ # share (one full read per pass, then only new rows).
18468
+ self.ledger_source = ledger_source
16638
18469
 
16639
18470
  def classify(self, conn, identity: str, path: str, *, known=(),
16640
18471
  archive: bool = False) -> str:
@@ -16657,7 +18488,9 @@ class _CodexHistoryContext:
16657
18488
  self.consulted = True
16658
18489
  return "history"
16659
18490
  if self.ledger_paths is None:
16660
- self.ledger_paths = _codex_ledger_paths(conn)
18491
+ self.ledger_paths = (_codex_ledger_paths(conn)
18492
+ if self.ledger_source is None
18493
+ else self.ledger_source(conn))
16661
18494
  if _codex_has_history_provenance(conn, identity, path,
16662
18495
  ledger_paths=self.ledger_paths):
16663
18496
  return "history"
@@ -16721,6 +18554,40 @@ def _codex_history_candidates(*, archive_only: bool = False) -> "list[str]":
16721
18554
  conn.close()
16722
18555
 
16723
18556
 
18557
+ def _codex_provenance_identity_write(
18558
+ share: _CodexWalkShare, path: str, apply, *, wait_for_requests: bool):
18559
+ """One provenance identity's admitted transaction over ``share``.
18560
+
18561
+ ``apply(conn, anchors)`` receives the identity's resolver, built from
18562
+ copies of the share's committed-evidence base (#975, spec 4.2), and
18563
+ returns whether it applied. After the commit, still under the writer
18564
+ locks, the base absorbs that identity's stored rows and the token is
18565
+ sealed. A refusal, an exception or a rollback adds nothing and drops the
18566
+ base, so the next identity seeds from the committed store.
18567
+ """
18568
+ applied: list = []
18569
+
18570
+ def run(conn):
18571
+ result = apply(conn, share.identity_resolver(conn))
18572
+ applied.append(bool(result))
18573
+ return result
18574
+
18575
+ def absorb(conn) -> None:
18576
+ if applied and applied[0]:
18577
+ share.absorb_identity(conn, path)
18578
+ share.seal(conn)
18579
+
18580
+ try:
18581
+ admitted, result = _codex_admitted_cache_write(
18582
+ run, wait_for_requests=wait_for_requests, after_commit=absorb)
18583
+ except BaseException:
18584
+ share.invalidate()
18585
+ raise
18586
+ if not admitted or not result:
18587
+ share.invalidate()
18588
+ return admitted, result
18589
+
18590
+
16724
18591
  def run_codex_history_provenance_drain(
16725
18592
  *, wait_for_requests: bool = True) -> ProvenanceDrainResult:
16726
18593
  """Settle provenance and retained quota before historical raw ingestion.
@@ -16753,6 +18620,9 @@ def run_codex_history_provenance_drain(
16753
18620
  for path, item in prepared.items()}
16754
18621
  traced, high_water, quotas, decisions = _jr._codex_journal_history_evidence(
16755
18622
  candidates, identities=identities.values(), capture=True)
18623
+ # #975 RC2: one committed-evidence seed for the whole pass, so each
18624
+ # anchor group is read once rather than once per identity.
18625
+ share = _CodexWalkShare()
16756
18626
  for path in candidates:
16757
18627
  item, identity = prepared[path], identities[path]
16758
18628
  # Codex legacy normalization is independent of Claude's cutover
@@ -16761,7 +18631,7 @@ def run_codex_history_provenance_drain(
16761
18631
  records = list(_jr._decoded_quota_stream(quotas.get(path, ()), None))
16762
18632
  file_ops = decisions.get(identity, ())
16763
18633
 
16764
- def apply(conn, path=path, item=item, identity=identity,
18634
+ def apply(conn, anchors, path=path, item=item, identity=identity,
16765
18635
  records=records, file_ops=file_ops):
16766
18636
  live = _qualify_codex_targets({path})
16767
18637
  if (len(live) != 1
@@ -16774,7 +18644,8 @@ def run_codex_history_provenance_drain(
16774
18644
  return False # another writer moved the captured population
16775
18645
  before = conn.total_changes
16776
18646
  _jr._apply_file_account_records(conn, file_ops)
16777
- _jr._apply_quota_records(conn, records, quiet=True)
18647
+ _jr._apply_quota_records(conn, records, quiet=True,
18648
+ anchors=anchors)
16778
18649
  if conn.total_changes != before:
16779
18650
  # Preserve triggers/change ledgers; no partial recovery
16780
18651
  # mints journal coverage or leaves old coverage standing.
@@ -16782,8 +18653,8 @@ def run_codex_history_provenance_drain(
16782
18653
  _bump_codex_physical_mutation_seq(conn)
16783
18654
  return True
16784
18655
 
16785
- admitted, applied = _codex_admitted_cache_write(
16786
- apply, wait_for_requests=wait_for_requests)
18656
+ admitted, applied = _codex_provenance_identity_write(
18657
+ share, path, apply, wait_for_requests=wait_for_requests)
16787
18658
  if not admitted or not applied:
16788
18659
  return ProvenanceDrainResult("yielded", len(candidates), len(traced))
16789
18660
  # A changed inventory is retried from one new captured census; never
@@ -16819,6 +18690,511 @@ def run_codex_history_provenance_drain(
16819
18690
  _release_worker_lock(worker)
16820
18691
 
16821
18692
 
18693
+ def _codex_replay_heal_due(conn: sqlite3.Connection) -> bool:
18694
+ """An armed byte-zero replay or a replay heal record (#982 spec 4.6)."""
18695
+ if conn.execute("SELECT 1 FROM cache_meta WHERE key=? LIMIT 1",
18696
+ (CODEX_REPLAY_FROM_ZERO_KEY,)).fetchone() is not None:
18697
+ return True
18698
+ record = load_codex_heal_record(conn)
18699
+ return record is not None and record.get("mode") == "replay"
18700
+
18701
+
18702
+ class HealResult(NamedTuple):
18703
+ """What one ``run_codex_admitted_heal`` pass did (#982 spec 4.1)."""
18704
+
18705
+ outcome: str # "done", "busy", "yielded", "contended", "degraded",
18706
+ # "refused", "replay_due", "restart" or "none"
18707
+ mode: "str | None"
18708
+ units: int # admitted targeted units: population and owed pass
18709
+ owed_units: int # of them, the owed pass's
18710
+ degraded: int # population files still owed after their attempts (U6)
18711
+ finalization_units: int
18712
+ certification_rounds: int # fenced and closing full rounds
18713
+ #: The last admitted sync's stats (the start unit's when it refused, the
18714
+ #: last walk piece's otherwise) with every work counter of the summary
18715
+ #: line (`_CODEX_HEAL_SUMMED_COUNTERS`) summed over every unit and piece
18716
+ #: of the pass, except the skipped count: the files the pass only found
18717
+ #: unchanged, each once. The failed and torn-deferred counts stay the
18718
+ #: last piece's. Each file is reported once, as the single walk does.
18719
+ #: ``None`` when no unit ran.
18720
+ stats: "CodexIngestStats | None" = None
18721
+
18722
+
18723
+ #: The work counters a heal pass reports summed over its units and pieces.
18724
+ _CODEX_HEAL_SUMMED_COUNTERS = (
18725
+ "files_processed", "files_skipped_unchanged", "files_reset_truncated",
18726
+ "rows_changed", "lines_malformed", "token_events_skipped")
18727
+
18728
+
18729
+ class _CodexHealYield(Exception):
18730
+ """A fresh hook request made a non-waiting executor give up its pass."""
18731
+
18732
+
18733
+ class _CodexHealContended(Exception):
18734
+ """Another writer held the store locks for longer than the replay worker
18735
+ has always waited for them (``CODEX_REPLAY_DRAIN_LOCK_TIMEOUT_SECONDS``)."""
18736
+
18737
+
18738
+ class _CodexHealRestart(Exception):
18739
+ """A unit found its pass generation stale (spec 4.5a): the record was
18740
+ rewritten, or a fresh clear only a new pass's start unit may perform is
18741
+ due. The unit wrote nothing; the pass ends ``restart``."""
18742
+
18743
+
18744
+ def _codex_heal_owed_files(
18745
+ conn: sqlite3.Connection,
18746
+ files: "list[CodexDiscoveredFile]") -> "list[CodexDiscoveredFile]":
18747
+ """The discovered files whose bytes ``_codex_backlog_after`` still owes:
18748
+ no cursor, an incomplete cursor, or a size beyond the cursor."""
18749
+ existing = _load_codex_session_files_rows(
18750
+ conn, [str(item.source_path) for item in files])
18751
+ return [item for item in files
18752
+ if _codex_backlog_after([item], existing)[0]]
18753
+
18754
+
18755
+ def run_codex_admitted_heal(
18756
+ *, wait_for_requests: bool = True,
18757
+ population: "list[str] | None" = None) -> HealResult:
18758
+ """Run a Codex history heal as admitted per-file units (#982 spec 4.1).
18759
+
18760
+ Single-flight through ``codex-heal-walk.lock`` (busy returns ``busy``).
18761
+ No writer lock is held between units, and every unit is admitted the way
18762
+ ``_apply_codex_verdict`` admits a verdict: the store is opened before
18763
+ admission, ``admit`` takes only the gate (the sync owns its locks, tried
18764
+ without waiting), and the sync rechecks for a fresh request once it holds
18765
+ them (``_after_locks``). A contended or rechecked unit changed nothing;
18766
+ with ``wait_for_requests`` the executor waits out every fresh request and
18767
+ retries the same unit, otherwise it returns ``yielded``. Store locks that
18768
+ another writer holds continuously for ``CODEX_REPLAY_DRAIN_LOCK_TIMEOUT_SECONDS``
18769
+ (the replay worker's lock wait) end the pass ``contended``.
18770
+
18771
+ History mode runs when ``population`` (the census a settled provenance
18772
+ pass just produced) is given or a history heal record exists, in order:
18773
+
18774
+ 1. the start unit writes the record (phase ``units``: the fence);
18775
+ 2. one targeted unit per population file, in discovery order;
18776
+ 3. the owed pass: every discovered file still owed, at most twice each;
18777
+ 4. the fenced walk: one full walk in admitted pieces (``_ingest_limit=1``,
18778
+ ``_heal_census``, ``_heal_visited``, settlement deferred), each
18779
+ ingesting or resetting at most one file, until a piece leaves no
18780
+ census file unwalked; that piece makes the transition, deleting the
18781
+ record, whether or not a file failed (it stays owed);
18782
+ 5. the #980 executor's unbudgeted finalization units until the debt is
18783
+ empty (a failed or idle unit hands the rest to ``_codex-prune-drain``);
18784
+ 6. the closing walk: a second walk in pieces, with its own census and
18785
+ visited set and with ordinary settlement and sealing; its last piece
18786
+ certifies the walk.
18787
+
18788
+ A piece that neither completes its walk nor adds a census file to the
18789
+ visited set (only an existing refusal such as ``quota_index_pending``
18790
+ does) ends the pass as ``degraded``: the record stays, and the worker is
18791
+ requested again. Every other piece but the last adds a visited file, so a
18792
+ walk takes at most one piece more than its census has files.
18793
+
18794
+ Replay mode (spec 4.5) runs when ``CODEX_REPLAY_FROM_ZERO_KEY`` or a
18795
+ replay record exists, whatever ``population`` says. Its start unit
18796
+ (``_replay_stage="start"``) captures the D1 snapshot, clears and writes
18797
+ the record (population: every discovered file), then rehydrates; a
18798
+ resumed pass whose record is not yet ``rehydrated`` reruns it, which
18799
+ rehydrates without clearing again. A refused start (``rebuild_refused``
18800
+ or a prune refusal) clears nothing and returns ``refused``. Steps 2-6
18801
+ follow exactly as in history mode, except that the fenced walk's last
18802
+ piece ends the record through the unchanged consumption block: deleted
18803
+ with the marker, or moved to phase ``certify`` when the replay is
18804
+ blocked (a failed or torn-deferred file). A ``certify`` record resumes at
18805
+ step 5: its fence is over, and the closing walk retries the blocked file
18806
+ and consumes the marker once it walks clean.
18807
+
18808
+ Generation (spec 4.5a): every unit after the start unit carries the
18809
+ ``gen`` of the record the pass started with (``_heal_generation``), and
18810
+ each finalization unit checks it under its store locks too. A unit that
18811
+ finds another ``gen``, a ``reclear`` mark or an armed marker without a
18812
+ replay record writes nothing; the pass then ends ``restart``, discarding
18813
+ its census, visited set and cached payload, and requests the worker under
18814
+ the short retry throttle. Only the next pass's start unit performs the
18815
+ fresh clear, so no piece skips a file it visited before that clear.
18816
+ """
18817
+ import _lib_codex_drain_admission as adm
18818
+ worker = _worker_lock(CODEX_HEAL_WALK_LOCK_NAME)
18819
+ if worker is None:
18820
+ return HealResult("busy", None, 0, 0, 0, 0, 0)
18821
+ counts = {"units": 0, "owed": 0, "degraded": 0, "final": 0, "rounds": 0}
18822
+ summed = collections.Counter()
18823
+ last: dict = {"stats": None, "mode": "history"}
18824
+ # The summary reports each file at most once per pass, as the single
18825
+ # walk does: a file any unit processed (or reset) counts as processed
18826
+ # only, and one the pass only found unchanged counts as skipped once,
18827
+ # however many walks re-find it. A piece's skips of visited and
18828
+ # out-of-census files stay in its own `files_skipped_unchanged` (the
18829
+ # barrier census sum needs them) and never reach the pass's count.
18830
+ pass_files: "dict[str, set[str]]" = {"processed": set(), "unchanged": set()}
18831
+ # A worker requested by this pass (`contended`, `degraded`) is spawned
18832
+ # after the heal lock is released: `_defer_codex_replay_drain` spawns
18833
+ # nothing while a pass holds it.
18834
+ worker_owed = {"replay": False}
18835
+
18836
+ def request_worker() -> None:
18837
+ worker_owed["replay"] = True
18838
+
18839
+ def count_pass_files(only_paths, stats: CodexIngestStats) -> None:
18840
+ if only_paths is not None:
18841
+ if stats.files_processed or stats.files_reset_truncated:
18842
+ pass_files["processed"].update(only_paths)
18843
+ elif stats.files_skipped_unchanged:
18844
+ pass_files["unchanged"].update(only_paths)
18845
+ return
18846
+ counted = set(stats.heal_counted_paths)
18847
+ pass_files["processed"] |= counted
18848
+ pass_files["unchanged"] |= set(stats.heal_examined) - counted
18849
+
18850
+ def result(outcome: str, mode: "str | None" = None) -> HealResult:
18851
+ stats = last["stats"]
18852
+ if stats is not None:
18853
+ stats = _dc_replace(stats, **{
18854
+ **{name: summed[name] for name in _CODEX_HEAL_SUMMED_COUNTERS},
18855
+ "files_skipped_unchanged": len(
18856
+ pass_files["unchanged"] - pass_files["processed"])})
18857
+ return HealResult(outcome, last["mode"] if mode is None else mode,
18858
+ counts["units"], counts["owed"],
18859
+ counts["degraded"], counts["final"],
18860
+ counts["rounds"], stats)
18861
+
18862
+ share = _CodexWalkShare()
18863
+ # The generation of the record this pass works on (spec 4.5a), set once
18864
+ # the start unit (or the probe of a resumed record) has it.
18865
+ pass_state: dict = {"gen": None}
18866
+
18867
+ def recheck() -> bool:
18868
+ return bool(adm.fresh_requests(_cctally_core.APP_DIR))
18869
+
18870
+ def admitted_sync(kind: str, **kwargs) -> CodexIngestStats:
18871
+ # The store locks held by another writer (not a hook request, which
18872
+ # expires) are waited for as long as the replay worker's single sync
18873
+ # always waited for them, then the pass gives up as `contended`.
18874
+ contended_since: "float | None" = None
18875
+ while True:
18876
+ stats = None
18877
+ conn = open_cache_db()
18878
+ try:
18879
+ admission = adm.admit(_cctally_core.APP_DIR,
18880
+ lambda: "sync-owns-locks",
18881
+ lambda token: None)
18882
+ if admission.admitted:
18883
+ try:
18884
+ with _perf.phase("codex_heal_unit") as _p_unit:
18885
+ _p_unit.set_meta(kind=kind)
18886
+ if kind != "start":
18887
+ kwargs["_heal_generation"] = pass_state["gen"]
18888
+ stats = sync_codex_cache(
18889
+ conn, maintenance="caller", lock_timeout=0,
18890
+ _walk_share=share, _after_locks=recheck,
18891
+ **kwargs)
18892
+ finally:
18893
+ adm.finish(admission, lambda token: None)
18894
+ finally:
18895
+ conn.close()
18896
+ if stats is not None and stats.heal_generation_stale:
18897
+ raise _CodexHealRestart()
18898
+ if stats is not None and not stats.lock_contended:
18899
+ last["stats"] = stats
18900
+ for name in _CODEX_HEAL_SUMMED_COUNTERS:
18901
+ summed[name] += int(getattr(stats, name) or 0)
18902
+ count_pass_files(kwargs.get("only_paths"), stats)
18903
+ return stats
18904
+ if not wait_for_requests:
18905
+ raise _CodexHealYield()
18906
+ if (stats is not None and not stats.drain_yielded
18907
+ and not adm.fresh_requests(_cctally_core.APP_DIR)):
18908
+ now = time.monotonic()
18909
+ if contended_since is None:
18910
+ contended_since = now
18911
+ if now - contended_since >= CODEX_REPLAY_DRAIN_LOCK_TIMEOUT_SECONDS:
18912
+ raise _CodexHealContended()
18913
+ else:
18914
+ # An admission refusal, a drain yield, or a raw refusal met
18915
+ # while a request (a Codex hook's, or the Claude waiter,
18916
+ # #1055 spec 4.3) is fresh: a deliberate wait, not continuous
18917
+ # contention.
18918
+ contended_since = None
18919
+ _drain_backoff(adm)
18920
+
18921
+ def run_finalization_units() -> None:
18922
+ """Step 5: unbudgeted finalization units until the debt is empty.
18923
+ Store locks another writer holds continuously are waited for as long
18924
+ as every other unit waits for them, then the pass is `contended`."""
18925
+ contended_since: "float | None" = None
18926
+ while True:
18927
+ conn = open_cache_db()
18928
+ try:
18929
+ if load_codex_finalization_debt(conn).empty:
18930
+ break
18931
+ conn.rollback()
18932
+ admission = adm.admit(_cctally_core.APP_DIR, _cache_store_locks,
18933
+ _release_cache_store_locks)
18934
+ if not admission.admitted:
18935
+ if not wait_for_requests:
18936
+ raise _CodexHealYield()
18937
+ if adm.fresh_requests(_cctally_core.APP_DIR):
18938
+ contended_since = None
18939
+ else:
18940
+ now = time.monotonic()
18941
+ if contended_since is None:
18942
+ contended_since = now
18943
+ if (now - contended_since
18944
+ >= CODEX_REPLAY_DRAIN_LOCK_TIMEOUT_SECONDS):
18945
+ raise _CodexHealContended()
18946
+ _drain_backoff(adm)
18947
+ continue
18948
+ contended_since = None
18949
+ try:
18950
+ # Spec 4.5a: under the store locks, the unit belongs to
18951
+ # this pass's record generation or writes nothing.
18952
+ if _codex_heal_generation_stale(
18953
+ load_codex_heal_record(conn),
18954
+ conn.execute(
18955
+ "SELECT 1 FROM cache_meta WHERE key=? LIMIT 1",
18956
+ (CODEX_REPLAY_FROM_ZERO_KEY,)).fetchone()
18957
+ is not None,
18958
+ pass_state["gen"]):
18959
+ raise _CodexHealRestart()
18960
+ with _perf.phase("codex_heal_unit") as _p_unit:
18961
+ _p_unit.set_meta(kind="finalization")
18962
+ ok, model_cleared = _run_codex_executor_unit(
18963
+ conn, "finalization")
18964
+ except BaseException:
18965
+ conn.rollback()
18966
+ raise
18967
+ finally:
18968
+ adm.finish(admission, _release_cache_store_locks)
18969
+ finally:
18970
+ conn.close()
18971
+ if model_cleared:
18972
+ try:
18973
+ from _cctally_quota import _defer_codex_quota_verification
18974
+ _defer_codex_quota_verification()
18975
+ except Exception:
18976
+ pass
18977
+ if not ok:
18978
+ # The prune drain's executor owns this debt with its
18979
+ # `blocked` rule; the closing round's settlement retries too.
18980
+ _request_codex_drain_worker(CODEX_PRUNE_DRAIN_COMMAND)
18981
+ break
18982
+ counts["final"] += 1
18983
+
18984
+ def probe_state() -> "tuple[dict | None, bool]":
18985
+ """The record's fence key and whether the replay marker is armed."""
18986
+ conn = open_cache_db()
18987
+ try:
18988
+ return (load_codex_heal_record(conn), conn.execute(
18989
+ "SELECT 1 FROM cache_meta WHERE key=? LIMIT 1",
18990
+ (CODEX_REPLAY_FROM_ZERO_KEY,)).fetchone() is not None)
18991
+ finally:
18992
+ conn.close()
18993
+
18994
+ try:
18995
+ record, replay = probe_state()
18996
+ replay_mode = replay or (record is not None
18997
+ and record.get("mode") == "replay")
18998
+ if replay_mode:
18999
+ last["mode"] = "replay"
19000
+ if (record is not None and record.get("mode") == "replay"
19001
+ and record.get("malformed")):
19002
+ # Spec 4.5a: a replay record whose D1 set cannot be read is
19003
+ # never cleared again (every writer declines); nothing here
19004
+ # can make progress without it.
19005
+ return result("degraded")
19006
+ # 1. The replay's start unit: the clear (which writes the record)
19007
+ # and the rehydration. Rerun on a resumed record that never
19008
+ # rehydrated; it clears nothing once the record exists, unless a
19009
+ # later arming site re-armed it (`reclear`, spec 4.5a).
19010
+ if (record is None or record.get("mode") != "replay"
19011
+ or not record.get("rehydrated")
19012
+ or _codex_replay_reclear_due(record, replay)):
19013
+ start_stats = admitted_sync("start", _replay_stage="start")
19014
+ record, replay = probe_state()
19015
+ if record is None or record.get("mode") != "replay":
19016
+ if not replay:
19017
+ return result("none") # consumed by another writer
19018
+ if start_stats.rebuild_refused or start_stats.prune_refused:
19019
+ return result("refused")
19020
+ request_worker()
19021
+ return result("degraded")
19022
+ if record.get("malformed"):
19023
+ return result("degraded")
19024
+ if _codex_replay_reclear_due(record, replay):
19025
+ # The fresh clear did not run (refused or deferred).
19026
+ if start_stats.rebuild_refused or start_stats.prune_refused:
19027
+ return result("refused")
19028
+ request_worker()
19029
+ return result("degraded")
19030
+ if not record.get("rehydrated"):
19031
+ # The rehydration failed and rolled back; a later
19032
+ # worker reruns it before any rollout is read.
19033
+ request_worker()
19034
+ return result("degraded")
19035
+ else:
19036
+ if record is None and not population:
19037
+ return result("none", None)
19038
+ if record is not None and record.get("phase") != "units":
19039
+ return result("none")
19040
+ if record is not None:
19041
+ pass_state["gen"] = record.get("gen")
19042
+
19043
+ # 1. The start unit.
19044
+ if record is None:
19045
+ fresh_population = sorted(set(str(p) for p in population))
19046
+
19047
+ def start(conn):
19048
+ current = load_codex_heal_record(conn)
19049
+ if current is not None:
19050
+ return current
19051
+ written = _store_codex_heal_record(
19052
+ conn, {"mode": "history", "phase": "units"},
19053
+ population=fresh_population)
19054
+ share.remember_heal_payload(CodexHealPayload(
19055
+ written["gen"], frozenset(fresh_population), None))
19056
+ return written
19057
+
19058
+ admitted, record = _codex_admitted_cache_write(
19059
+ start, wait_for_requests=wait_for_requests,
19060
+ contention_bound=CODEX_REPLAY_DRAIN_LOCK_TIMEOUT_SECONDS)
19061
+ if not admitted:
19062
+ return result("yielded")
19063
+ if record.get("mode") != "history" or record.get("phase") != "units":
19064
+ return result("none", record.get("mode"))
19065
+ pass_state["gen"] = record.get("gen")
19066
+
19067
+ def walk_in_pieces(kind: str, **kwargs) -> bool:
19068
+ """Steps 4 and 7: one full walk in admitted pieces. True when a
19069
+ piece left no census file unwalked; False when a piece neither
19070
+ completed the walk nor visited a file (no progress). Each
19071
+ visited file's committed row is kept for the last piece's
19072
+ revalidation (revision 7)."""
19073
+ census: "frozenset[str] | None" = None
19074
+ visited: "dict[str, tuple | None]" = {}
19075
+ while True:
19076
+ stats = admitted_sync(
19077
+ kind, _ingest_limit=1, _heal_census=census,
19078
+ _heal_visited=frozenset(visited),
19079
+ _heal_visited_rows=dict(visited), **kwargs)
19080
+ counts["rounds"] += 1
19081
+ if census is None and stats.heal_discovered is not None:
19082
+ census = stats.heal_discovered
19083
+ added = set(stats.heal_examined) - set(visited)
19084
+ for path in added:
19085
+ visited[path] = stats.heal_visit_rows.get(path)
19086
+ if stats.heal_walk_complete:
19087
+ return True
19088
+ if not added:
19089
+ return False
19090
+
19091
+ if record.get("phase") == "units":
19092
+ # The pass's one parse of the population (and, in replay mode,
19093
+ # the D1 set the units then take from the share).
19094
+ conn = open_cache_db()
19095
+ try:
19096
+ payload = share.heal_payload(conn, record)
19097
+ finally:
19098
+ conn.close()
19099
+ if payload is None and last["mode"] == "replay":
19100
+ return result("degraded") # unreadable D1 set: fail closed
19101
+ members = (frozenset() if payload is None
19102
+ else payload.population)
19103
+
19104
+ # 2. Population units, in the single walk's (discovery) order.
19105
+ attempts: "collections.Counter[str]" = collections.Counter()
19106
+ for item in _discover_codex_files_with_roots():
19107
+ path = str(item.source_path)
19108
+ if path not in members:
19109
+ continue
19110
+ attempts[path] += 1
19111
+ admitted_sync("population", only_paths={path})
19112
+ counts["units"] += 1
19113
+
19114
+ # 3. The owed pass: appearances after the census, tails another
19115
+ # writer left, and population files a targeted unit declined.
19116
+ while True:
19117
+ files = _discover_codex_files_with_roots()
19118
+ conn = open_cache_db()
19119
+ try:
19120
+ owed = _codex_heal_owed_files(conn, files)
19121
+ finally:
19122
+ conn.close()
19123
+ todo = [item for item in owed
19124
+ if attempts[str(item.source_path)] < 2]
19125
+ if not todo:
19126
+ # Only population files: a file that appeared after the
19127
+ # census is a later sync's, as for the single walk.
19128
+ counts["degraded"] = sum(
19129
+ 1 for item in owed if str(item.source_path) in members)
19130
+ break
19131
+ for item in todo:
19132
+ path = str(item.source_path)
19133
+ attempts[path] += 1
19134
+ admitted_sync("owed", only_paths={path})
19135
+ counts["units"] += 1
19136
+ counts["owed"] += 1
19137
+
19138
+ # Spec 4.5a: a replay armed meanwhile (or re-armed over this
19139
+ # replay) makes every piece below clear and refill one file;
19140
+ # end the pass instead, without a transition. The next pass
19141
+ # starts in replay mode, whose clear retires a history record.
19142
+ current, armed = probe_state()
19143
+ if ((last["mode"] == "history" and armed)
19144
+ or _codex_replay_reclear_due(current, armed)):
19145
+ return result("replay_due")
19146
+
19147
+ # 4. The fenced walk, whose last piece makes the transition
19148
+ # (replay: the unchanged consumption block deletes the record
19149
+ # with the marker, or moves it to `certify` when blocked).
19150
+ walk_in_pieces("certification", _defer_finalization=True,
19151
+ _heal_transition=True)
19152
+ conn = open_cache_db()
19153
+ try:
19154
+ current = load_codex_heal_record(conn)
19155
+ finally:
19156
+ conn.close()
19157
+ if current is not None and current.get("phase") == "units":
19158
+ # No progress (or a complete walk that could not end the
19159
+ # record): the fence stays, and every hook keeps requesting
19160
+ # the worker while the record exists. Without a record (or
19161
+ # with a `certify` one), the last piece made the transition
19162
+ # or another walk (a rebuild) retired it meanwhile.
19163
+ request_worker()
19164
+ return result("degraded")
19165
+
19166
+ # 5. Finalization units of the #980 executor, now unfenced.
19167
+ run_finalization_units()
19168
+
19169
+ # 6. The closing walk: ordinary settlement; it certifies the walk.
19170
+ # The fence is gone, so a refusal here is the ordinary syncs' to
19171
+ # retry (the refusing piece requested its own worker).
19172
+ if not walk_in_pieces("closing"):
19173
+ return result("degraded")
19174
+ return result("done")
19175
+ except _CodexHealYield:
19176
+ return result("yielded")
19177
+ except _CodexHealContended:
19178
+ # The record (and with it the fence) stays: request the worker, which
19179
+ # the record's short retry throttle lets through within minutes.
19180
+ request_worker()
19181
+ return result("contended")
19182
+ except _CodexHealRestart:
19183
+ # Spec 4.5a: the census and visited set died with the walk's frame;
19184
+ # the cached payload is another generation's. The next pass starts
19185
+ # over from its own start unit (the fresh clear, when one is due).
19186
+ share.forget_heal_payload()
19187
+ # Spawns nothing while this pass holds the heal lock (every caller
19188
+ # re-runs the executor inline on `restart`), which keeps a re-run and
19189
+ # a spawned worker from racing for the lock.
19190
+ _defer_codex_replay_drain(short=True)
19191
+ return result("restart")
19192
+ finally:
19193
+ _release_worker_lock(worker)
19194
+ if worker_owed["replay"]:
19195
+ _request_codex_drain_worker(CODEX_REPLAY_DRAIN_COMMAND)
19196
+
19197
+
16822
19198
  class _CodexFileBaselineResolver:
16823
19199
  """Per-row ``codex_file_accounts`` baselines, memoised per file.
16824
19200
 
@@ -16916,6 +19292,10 @@ def reconcile_codex_window_attribution_spend(
16916
19292
  """
16917
19293
  from _cctally_quota import resolve_codex_window_attributions_with_evidence
16918
19294
 
19295
+ # #982 spec 4.3: `account attribute` calls this directly, so the kernel
19296
+ # checks the heal fence itself (strict: `CodexHealFenced`).
19297
+ if _codex_heal_fenced_kernel(conn, strict):
19298
+ return (0, 0)
16919
19299
  try:
16920
19300
  with _codex_cancellable_read(conn, deadline, clock):
16921
19301
  active = load_active_window_attributions(conn)
@@ -21355,6 +23735,7 @@ def _claude_batch_committed(conn, released, sessions, stats, touched_sessions,
21355
23735
  _CONVERSATION_RESIDUAL_FIELDS = frozenset({
21356
23736
  "failed_paths", "barrier_deferred_keys", "barrier_deferred_targets",
21357
23737
  "barrier_published_paths", "barrier_retry_causes",
23738
+ "barrier_terminal_keys", # #1052
21358
23739
  })
21359
23740
 
21360
23741
 
@@ -22125,6 +24506,32 @@ def _cache_side_replay_pending(conn: sqlite3.Connection) -> bool:
22125
24506
  return False
22126
24507
 
22127
24508
 
24509
+ def _cache_side_history_heal_pending(conn: sqlite3.Connection) -> bool:
24510
+ """Whether cache.db holds a history heal fence (#982 spec 4.3, R10).
24511
+
24512
+ Per-file heal units commit one population file at a time, and a
24513
+ cursorless contributor's membership is unknown until its own unit reads
24514
+ it, so the barrier cannot hold a conversation whose other contributor is
24515
+ still to come. The pass therefore publishes nothing while the fence holds;
24516
+ the heal's transition ends it once every population file is accounted.
24517
+ Qualified through ``cache_db`` like ``_cache_side_replay_pending``, and a
24518
+ connection without the attachment reports False.
24519
+ """
24520
+ try:
24521
+ row = conn.execute(
24522
+ "SELECT value FROM cache_db.cache_meta WHERE key=?",
24523
+ (CODEX_HEAL_WALK_KEY,)).fetchone()
24524
+ except sqlite3.OperationalError:
24525
+ return False
24526
+ if row is None:
24527
+ return False
24528
+ try:
24529
+ record = json.loads(str(row[0]))
24530
+ except (TypeError, ValueError):
24531
+ return True # malformed fails closed, as the fence
24532
+ return not (isinstance(record, dict) and record.get("phase") != "units")
24533
+
24534
+
22128
24535
  def _clear_codex_conversation_store(conn: sqlite3.Connection) -> None:
22129
24536
  """Clear only the re-derivable Codex transcript families."""
22130
24537
  conn.execute("DELETE FROM codex_conversation_events")
@@ -22201,6 +24608,75 @@ def _codex_conversation_cursor_row(conn, path: str):
22201
24608
  return None if row is None else tuple(row[1:])
22202
24609
 
22203
24610
 
24611
+ def _codex_conversation_accepted(row) -> _AcceptedContainer:
24612
+ """The container evidence of a conversations cursor row (``row[1:]``)."""
24613
+ return _accepted_container(row, ext=14, mtime=1, device=12, inode=13, size=0)
24614
+
24615
+
24616
+ def _codex_cursor_unverified(row) -> bool:
24617
+ """A conversations cursor row whose state is explicitly ``unverified``."""
24618
+ return row is not None and _codex_conversation_accepted(row).state == "unverified"
24619
+
24620
+
24621
+ _CURSOR_UNREAD = object()
24622
+
24623
+
24624
+ def _classify_codex_unit_file(conn, discovered, *, executor_paths,
24625
+ protected_paths, requal_held, unit, targeted,
24626
+ cursor=_CURSOR_UNREAD) -> str:
24627
+ """``"blocked"``, ``"terminal"`` or ``"quiet"`` for one unit file (#1052
24628
+ spec 5.1), by the accessors and in the order of the conversation loop's
24629
+ own checks. Never decodes: a cursorless compressed container, whose
24630
+ decision would validate it, is blocked before the decision is reached,
24631
+ and an executor path, the only other decoding entrance, is blocked first.
24632
+
24633
+ Terminal is the conversations store's terminal verdict exactly: an
24634
+ explicitly ``unverified`` cursor whose decision is the quiet
24635
+ ``skip/verification-unverified``. The decision maps every non-accepted,
24636
+ non-pending state to that reason, so the reason alone is not enough.
24637
+
24638
+ Quiet is a ``read`` decision the loop would not refuse before reading a
24639
+ byte of it. The loop refuses two such files, and so does this (#1052
24640
+ review F2): in a targeted pass, a retained cursor the file no longer
24641
+ continues (truncated, replaced, or under another root), and, in ``unit``,
24642
+ a certificate that names another generation of the file. ``cursor`` is
24643
+ the row the caller already read, if any."""
24644
+ path_str = str(discovered.source_path)
24645
+ if path_str in protected_paths or path_str in executor_paths:
24646
+ return "blocked"
24647
+ selected = _select_codex_container(discovered.source_path)
24648
+ if selected is None:
24649
+ return "blocked"
24650
+ prev = (_codex_conversation_cursor_row(conn, path_str)
24651
+ if cursor is _CURSOR_UNREAD else cursor)
24652
+ if requal_held and _codex_held_by_requalification(
24653
+ requal_held, frozenset(), discovered, selected.location,
24654
+ first_sight=prev is None):
24655
+ return "blocked"
24656
+ if prev is None and selected.kind == _rc.ZSTD:
24657
+ return "blocked"
24658
+ decision = _codex_container_decision(
24659
+ selected, None if prev is None else _codex_conversation_accepted(prev),
24660
+ executor_path=False, should_stop=None)
24661
+ if decision.action == "read":
24662
+ fp = selected.fingerprint
24663
+ size = int(decision.size)
24664
+ if targeted and prev is not None and (
24665
+ size < prev[0] or prev[3] != discovered.source_root_key
24666
+ or _ingest_frontier.source_identity_replaced(
24667
+ prev[12], prev[13], fp.device, fp.inode)):
24668
+ return "blocked"
24669
+ if unit is not None and unit.cap_for(path_str, _rc.generation_token(
24670
+ fp, size if selected.kind == _rc.ZSTD else None)) is None:
24671
+ return "blocked"
24672
+ return "quiet"
24673
+ if (decision.action == "skip"
24674
+ and decision.reason == "verification-unverified"
24675
+ and _codex_cursor_unverified(prev)):
24676
+ return "terminal"
24677
+ return "blocked"
24678
+
24679
+
22204
24680
  def _codex_conversation_population_subset(conn, keys) -> "set[str]":
22205
24681
  found: set[str] = set()
22206
24682
  for chunk in _codex_barrier_chunks({k for k in keys if k}):
@@ -22291,15 +24767,24 @@ class _CodexPublicationWork:
22291
24767
  files as ``(discovered, unit)`` and ``(None, unit)`` (the loop commits the
22292
24768
  unit). Planning captures one read snapshot between the two phases and
22293
24769
  releases it before any publication transaction begins.
24770
+
24771
+ #1052 spec 5.1: each planned unit is classified before any of its files
24772
+ is yielded. A terminal deferral (a terminal contributor and no blocked
24773
+ file) yields nothing, so nothing of it is read, decoded or published.
22294
24774
  """
22295
24775
 
22296
- def __init__(self, conn, files, *, only_paths, stats):
24776
+ def __init__(self, conn, files, *, only_paths, stats,
24777
+ executor_paths=frozenset(), protected_paths=frozenset(),
24778
+ requal_held=None):
22297
24779
  self.conn = conn
22298
24780
  self.files = list(files)
22299
24781
  self.files_by_path = {str(d.source_path): d for d in self.files}
22300
24782
  self.only_paths = (None if only_paths is None
22301
24783
  else {str(path) for path in only_paths})
22302
24784
  self.stats = stats
24785
+ self.executor_paths = frozenset(executor_paths)
24786
+ self.protected_paths = frozenset(protected_paths)
24787
+ self.requal_held = requal_held or {}
22303
24788
  self.population_present = conn.execute(
22304
24789
  "SELECT 1 FROM cache_db.codex_barrier_population LIMIT 1"
22305
24790
  ).fetchone() is not None
@@ -22314,10 +24799,49 @@ class _CodexPublicationWork:
22314
24799
  yield None, None
22315
24800
  self.units = _plan_codex_publication_units(self.conn, self)
22316
24801
  for unit in self.units:
24802
+ terminal = self._terminal_paths(unit)
24803
+ if terminal:
24804
+ # #1052 spec 5.2: deferred, never retried. A retry flag raised
24805
+ # elsewhere is neither set nor cleared here.
24806
+ unit.failed = True
24807
+ self.stats.barrier_keys_deferred += len(unit.keys)
24808
+ self.stats.barrier_keys_terminal += len(unit.keys)
24809
+ named = self.stats.barrier_terminal_keys
24810
+ for path in terminal:
24811
+ named[path] = named.get(path, frozenset()) | frozenset(unit.keys)
24812
+ continue
22317
24813
  for discovered in unit.files:
22318
24814
  yield discovered, unit
22319
24815
  yield None, unit
22320
24816
 
24817
+ def _terminal_paths(self, unit) -> "tuple[str, ...]":
24818
+ """The terminal files of a terminal deferral; ``()`` for every other
24819
+ unit, which the loop then treats exactly as before (#1052 spec 5.1).
24820
+
24821
+ A unit none of whose files has an ``unverified`` conversations cursor
24822
+ is not classified further. Otherwise one blocked file makes the whole
24823
+ unit an ordinary one, so its own refusal (G10, pending verification,
24824
+ a protected or executor path, an unread new identity) still reaches
24825
+ the loop's retryable failure."""
24826
+ cursors = [
24827
+ (discovered, _codex_conversation_cursor_row(
24828
+ self.conn, str(discovered.source_path)))
24829
+ for discovered in unit.files]
24830
+ if not any(_codex_cursor_unverified(row) for _d, row in cursors):
24831
+ return ()
24832
+ terminal = []
24833
+ for discovered, row in cursors:
24834
+ verdict = _classify_codex_unit_file(
24835
+ self.conn, discovered, executor_paths=self.executor_paths,
24836
+ protected_paths=self.protected_paths,
24837
+ requal_held=self.requal_held, unit=unit,
24838
+ targeted=self.only_paths is not None, cursor=row)
24839
+ if verdict == "blocked":
24840
+ return ()
24841
+ if verdict == "terminal":
24842
+ terminal.append(str(discovered.source_path))
24843
+ return tuple(terminal)
24844
+
22321
24845
  def _defer(self, discovered, keys, *, generation, size) -> bool:
22322
24846
  path = str(discovered.source_path)
22323
24847
  prior = self.deferred.get(path)
@@ -22792,6 +25316,38 @@ def _rollback_codex_file_savepoint(conn: sqlite3.Connection) -> bool:
22792
25316
  return False
22793
25317
 
22794
25318
 
25319
+ class _CodexConversationCensus:
25320
+ """One conversation pass's container skips and held reads (#1052 5.3).
25321
+
25322
+ The census is phase 1's: its skips are ``container_skipped``. A
25323
+ unit-phase skip of a path phase 1 did not skip is counted once in
25324
+ ``barrier_container_skipped`` instead, so certification never counts one
25325
+ file twice. ``container_reads_held`` counts distinct logical paths across
25326
+ both phases."""
25327
+
25328
+ def __init__(self, stats):
25329
+ self.stats = stats
25330
+ self.skipped_phase1: "set[str]" = set()
25331
+ self.skipped_units: "set[str]" = set()
25332
+ self.held: "set[str]" = set()
25333
+
25334
+ def skip(self, path: str, reason: str, *, unit_phase: bool) -> None:
25335
+ stats = self.stats
25336
+ if not unit_phase:
25337
+ self.skipped_phase1.add(path)
25338
+ stats.container_skipped[reason] = (
25339
+ stats.container_skipped.get(reason, 0) + 1)
25340
+ elif path not in self.skipped_phase1 and path not in self.skipped_units:
25341
+ self.skipped_units.add(path)
25342
+ stats.barrier_container_skipped[reason] = (
25343
+ stats.barrier_container_skipped.get(reason, 0) + 1)
25344
+
25345
+ def hold(self, path: str) -> None:
25346
+ if path not in self.held:
25347
+ self.held.add(path)
25348
+ self.stats.container_reads_held += 1
25349
+
25350
+
22795
25351
  def _sync_codex_conversations_locked(
22796
25352
  conn: sqlite3.Connection,
22797
25353
  *,
@@ -22879,6 +25435,9 @@ def _sync_codex_conversations_locked(
22879
25435
  if _cache_side_replay_pending(conn):
22880
25436
  stats.deferred_reason = "cache_replay_pending"
22881
25437
  return stats
25438
+ if _cache_side_history_heal_pending(conn):
25439
+ stats.deferred_reason = "cache_heal_pending"
25440
+ return stats
22882
25441
  # #968 spec 5.11: the barrier extends this admission point. Its state
22883
25442
  # or attachment missing fails closed rather than reading as "no
22884
25443
  # population".
@@ -23078,7 +25637,10 @@ def _sync_codex_conversations_locked(
23078
25637
  location_marks: "list[tuple[str, _rc.SelectedContainer, _AcceptedContainer]]" = []
23079
25638
  batch = _CodexConversationBatch()
23080
25639
  work = _CodexPublicationWork(
23081
- conn, files, only_paths=only_paths, stats=stats)
25640
+ conn, files, only_paths=only_paths, stats=stats,
25641
+ executor_paths=executor_paths, protected_paths=protected_paths,
25642
+ requal_held=requal_held)
25643
+ census = _CodexConversationCensus(stats)
23082
25644
  for discovered, unit in work:
23083
25645
  if discovered is None:
23084
25646
  if unit is None: # the phase boundary
@@ -23118,7 +25680,7 @@ def _sync_codex_conversations_locked(
23118
25680
  requal_held, frozenset(), discovered, selected.location,
23119
25681
  first_sight=prev is None):
23120
25682
  # G10: held until both stores retire the direct-era spelling.
23121
- stats.container_reads_held += 1
25683
+ census.hold(path_str)
23122
25684
  stats.drain_requested = tuple(sorted(
23123
25685
  set(stats.drain_requested) | {CODEX_PRUNE_DRAIN_COMMAND}))
23124
25686
  if unit is not None:
@@ -23127,8 +25689,8 @@ def _sync_codex_conversations_locked(
23127
25689
  continue
23128
25690
  _fp = selected.fingerprint
23129
25691
  st = _CodexContainerStat(_fp.size, _fp.mtime_ns, _fp.device, _fp.inode)
23130
- accepted = (None if prev is None else _accepted_container(
23131
- prev, ext=14, mtime=1, device=12, inode=13, size=0))
25692
+ accepted = (None if prev is None
25693
+ else _codex_conversation_accepted(prev))
23132
25694
  container_decision = _codex_container_decision(
23133
25695
  selected, accepted, executor_path=path_str in executor_paths, should_stop=None)
23134
25696
  if container_decision.action == "adopt-location":
@@ -23145,12 +25707,15 @@ def _sync_codex_conversations_locked(
23145
25707
  continue
23146
25708
  if container_decision.action == "skip":
23147
25709
  reason = container_decision.reason
23148
- stats.container_skipped[reason] = stats.container_skipped.get(reason, 0) + 1
25710
+ census.skip(path_str, reason, unit_phase=unit is not None)
23149
25711
  if selected.kind == _rc.ZSTD and prev is None:
23150
- stats.container_reads_held += 1
25712
+ census.hold(path_str)
23151
25713
  if unit is not None:
23152
25714
  # A failed publication is still due, even when its failure
23153
25715
  # was an unreadable skip rather than pending verification.
25716
+ # #1052: a unit reaching a terminal contributor here also
25717
+ # carries a blocked file (or its cursor changed after
25718
+ # classification), so it stays retryable.
23154
25719
  batch = _fail_codex_publication_unit(
23155
25720
  conn, batch, stats, retry=True)
23156
25721
  _report_conversation_progress(progress, "ingest", stats)
@@ -23357,9 +25922,9 @@ def _sync_codex_conversations_locked(
23357
25922
  raise _rc.ContainerError("unstable-stat")
23358
25923
  except _rc.ContainerError as exc:
23359
25924
  reason = _container_skip_reason(exc.category)
23360
- stats.container_skipped[reason] = stats.container_skipped.get(reason, 0) + 1
25925
+ census.skip(path_str, reason, unit_phase=unit is not None)
23361
25926
  if selected.kind == _rc.ZSTD and prev is None:
23362
- stats.container_reads_held += 1
25927
+ census.hold(path_str)
23363
25928
  if unit is not None:
23364
25929
  batch = _fail_codex_publication_unit(conn, batch, stats, retry=True)
23365
25930
  _report_conversation_progress(progress, "ingest", stats)
@@ -23564,6 +26129,11 @@ def _sync_codex_conversations_locked(
23564
26129
  _commit_codex_conversation_batch(conn, batch, stats, progress)
23565
26130
  batch = _CodexConversationBatch()
23566
26131
  _commit_codex_conversation_batch(conn, batch, stats, progress)
26132
+ if only_paths is None:
26133
+ # #972 §5.4: the rollups the walk did not rewrite still follow the
26134
+ # current thread metadata and filesystem. Failed files do not stop
26135
+ # it; a targeted pass never runs it.
26136
+ _reconcile_codex_rollup_attribution(conn, stats)
23567
26137
  if (work.seal_seq is not None and work.backlog_complete
23568
26138
  and not stats.barrier_retry):
23569
26139
  # Every certificate sealed up to this token was considered. Units
@@ -23581,7 +26151,8 @@ def _sync_codex_conversations_locked(
23581
26151
  conn, "codex_conversation_source_files", path, adopted, snap)
23582
26152
  conn.commit()
23583
26153
  if (stats.container_pending
23584
- or stats.container_skipped.get("verification-pending", 0)):
26154
+ or stats.container_skipped.get("verification-pending", 0)
26155
+ or stats.barrier_container_skipped.get("verification-pending", 0)):
23585
26156
  stats.drain_requested = ("_codex-replay-drain",)
23586
26157
  run_codex_find_projection_backfill(conn)
23587
26158
  _report_conversation_progress(progress, "finalize", stats)
@@ -23626,7 +26197,7 @@ def _sync_codex_conversations_locked(
23626
26197
  and not stats.prune_refused
23627
26198
  and not stats.files_deferred_torn
23628
26199
  and not stats.container_pending
23629
- and not any(stats.container_skipped.values())
26200
+ and not any(stats.all_container_skipped().values())
23630
26201
  and stats.files_processed + stats.files_skipped_unchanged
23631
26202
  == stats.files_total),
23632
26203
  cursor_table="codex_conversation_source_files")
@@ -23805,7 +26376,9 @@ def _run_transcript_rebuild_worker(
23805
26376
  emit({
23806
26377
  "event": "result",
23807
26378
  "stats": {
23808
- name: value for name, value in asdict(stats).items()
26379
+ name: (sorted(value) if isinstance(value, (set, frozenset))
26380
+ else value)
26381
+ for name, value in asdict(stats).items()
23809
26382
  if name not in _CONVERSATION_RESIDUAL_FIELDS
23810
26383
  },
23811
26384
  "statsType": type(stats).__name__,
@@ -24136,6 +26709,21 @@ def cmd_cache_sync(args: argparse.Namespace) -> int:
24136
26709
  def _sync_codex_leg(
24137
26710
  active_conn: sqlite3.Connection,
24138
26711
  ) -> CodexIngestStats:
26712
+ if not args.rebuild and _codex_replay_heal_due(active_conn):
26713
+ # #982 spec 4.6: an armed byte-zero replay (or one a dead
26714
+ # executor left) clears and refills through the admitted heal
26715
+ # executor, before provenance runs, as the first sync did.
26716
+ with _perf.phase("codex_heal") as _p_heal:
26717
+ heal = run_codex_admitted_heal(wait_for_requests=True)
26718
+ if heal.outcome in ("replay_due", "restart"):
26719
+ # Spec 4.5a: re-armed during the pass (or a unit
26720
+ # found its generation stale); replay again.
26721
+ heal = run_codex_admitted_heal(wait_for_requests=True)
26722
+ _p_heal.set_meta(outcome=heal.outcome, units=heal.units,
26723
+ mode=heal.mode)
26724
+ if heal.outcome in ("busy", "contended"):
26725
+ return CodexIngestStats(lock_contended=True)
26726
+ return heal.stats or CodexIngestStats()
24139
26727
  with _perf.phase("sync_codex_cache"):
24140
26728
  return sync_codex_cache(
24141
26729
  active_conn,
@@ -24233,14 +26821,7 @@ def cmd_cache_sync(args: argparse.Namespace) -> int:
24233
26821
  )
24234
26822
  contended = True
24235
26823
  elif not stats.lock_contended:
24236
- eprint(
24237
- f"[cache-sync] codex done: {stats.files_processed} processed, "
24238
- f"{stats.files_skipped_unchanged} skipped, "
24239
- f"{stats.files_reset_truncated} reset, "
24240
- f"{stats.rows_changed} rows changed, "
24241
- f"{stats.lines_malformed} malformed, "
24242
- f"{stats.token_events_skipped} drift-skipped"
24243
- )
26824
+ _print_codex_done_line(stats)
24244
26825
  # #416 review B4: emitted on EVERY branch (including a contended or
24245
26826
  # incomplete rebuild) — a deferral means Codex spend and quota stopped
24246
26827
  # updating, which the "done" line's zeroes look identical to. Exit code
@@ -24256,14 +26837,41 @@ def cmd_cache_sync(args: argparse.Namespace) -> int:
24256
26837
  codex_stats = stats
24257
26838
  if not stats.lock_contended:
24258
26839
  history = run_codex_history_provenance_drain(wait_for_requests=True)
24259
- if history.outcome == "settled" and history.candidates:
24260
- # Same accounting path and lock policy, without repeating the
24261
- # rebuild clear: the now-settled historical inputs can be read.
24262
- codex_stats = sync_codex_cache(
24263
- conn, progress=_progress_codex_stderr, lock_timeout=lt,
24264
- maintenance="caller")
24265
- if codex_stats.lock_contended:
26840
+ heal_record = load_codex_heal_record(conn)
26841
+ # A replay the first leg left blocked (or unfinished) keeps the
26842
+ # executor in replay mode; the history heal waits for it.
26843
+ if not _codex_replay_heal_due(conn) and (
26844
+ (history.outcome == "settled" and history.candidates)
26845
+ or (heal_record is not None
26846
+ and heal_record.get("mode") == "history")):
26847
+ # #982 spec 4.6: the now-settled historical inputs are read by
26848
+ # the admitted heal executor, one file per unit, in place of a
26849
+ # second whole-tree sync under one writer-lock hold.
26850
+ with _perf.phase("codex_heal") as _p_heal:
26851
+ heal = run_codex_admitted_heal(
26852
+ wait_for_requests=True,
26853
+ population=(None if heal_record is not None
26854
+ else _codex_history_candidates()))
26855
+ if heal.outcome in ("replay_due", "restart"):
26856
+ # Spec 4.5a: a replay armed during this heal ended
26857
+ # its pass before the fenced walk; replay now, as the
26858
+ # first leg would have.
26859
+ heal = run_codex_admitted_heal(wait_for_requests=True)
26860
+ _p_heal.set_meta(outcome=heal.outcome, units=heal.units)
26861
+ if heal.stats is not None and heal.stats.rebuild_refused:
26862
+ # The inline re-run's replay was refused by the #485/#968
26863
+ # preflight: reported exactly as the first leg reports a
26864
+ # refused replay (every marker was kept).
26865
+ conn.close()
26866
+ eprint(CODEX_REBUILD_REFUSED_MESSAGE)
26867
+ _p_root.__exit__(None, None, None)
26868
+ return 3
26869
+ if heal.outcome in ("busy", "contended"):
26870
+ codex_stats = CodexIngestStats(lock_contended=True)
24266
26871
  contended = True
26872
+ elif heal.stats is not None:
26873
+ # The heal's summary, once, in the first leg's format.
26874
+ _print_codex_done_line(heal.stats)
24267
26875
  if not codex_stats.lock_contended:
24268
26876
  run_codex_barrier_membership_scan(wait_for_requests=True)
24269
26877
  run_codex_barrier_seal_drain(wait_for_requests=True)