cctally 1.117.0 → 1.118.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +46 -0
- package/README.md +7 -3
- package/bin/_cctally_cache.py +2783 -175
- package/bin/_cctally_completion.py +106 -33
- package/bin/_cctally_core.py +69 -4
- package/bin/_cctally_dashboard.py +227 -23
- package/bin/_cctally_dashboard_conversation.py +5 -1
- package/bin/_cctally_dashboard_envelope.py +60 -19
- package/bin/_cctally_dashboard_sources.py +23 -1
- package/bin/_cctally_db.py +193 -153
- package/bin/_cctally_diagnosis_sources.py +63 -7
- package/bin/_cctally_doctor.py +865 -93
- package/bin/_cctally_five_hour.py +124 -54
- package/bin/_cctally_forecast.py +301 -7
- package/bin/_cctally_journal.py +186 -18
- package/bin/_cctally_milestone_history.py +1193 -108
- package/bin/_cctally_oauth_samples.py +286 -15
- package/bin/_cctally_parser.py +2 -2
- package/bin/_cctally_quota.py +104 -12
- package/bin/_cctally_record.py +153 -29
- package/bin/_cctally_rederive.py +608 -68
- package/bin/_cctally_statusline.py +108 -81
- package/bin/_cctally_store.py +179 -38
- package/bin/_cctally_tui.py +1210 -170
- package/bin/_cctally_update.py +32 -13
- package/bin/_lib_codex_conversation_query.py +52 -14
- package/bin/_lib_codex_drain_admission.py +224 -6
- package/bin/_lib_codex_metadata.py +39 -0
- package/bin/_lib_conversation_anon.py +15 -8
- package/bin/_lib_conversation_dispatch.py +7 -0
- package/bin/_lib_conversation_query.py +289 -70
- package/bin/_lib_dashboard_sources.py +131 -9
- package/bin/_lib_detector_access_registry.py +497 -26
- package/bin/_lib_doctor.py +85 -20
- package/bin/_lib_doctor_scan_index.py +321 -0
- package/bin/_lib_ingest_frontier.py +20 -1
- package/bin/_lib_oauth_samples.py +23 -7
- package/bin/_lib_pricing.py +146 -16
- package/bin/_lib_quota.py +19 -0
- package/bin/_lib_quota_model.py +29 -11
- package/bin/_lib_rederive.py +11 -6
- package/bin/_lib_retained_records.py +516 -0
- package/bin/_lib_snapshot_cache.py +18 -0
- package/bin/_lib_stale_proof.py +272 -43
- package/bin/_lib_stats_content_revision.py +831 -0
- package/bin/_lib_tick_stats.py +10 -5
- package/bin/_lib_write_io.py +12 -5
- package/bin/cctally +8 -1
- package/dashboard/static/assets/ConversationsView-D4Iv4ku0.js +72 -0
- package/dashboard/static/assets/{DoctorModal-BqmXbnhn.js → DoctorModal-D4QSq6uz.js} +1 -1
- package/dashboard/static/assets/ModalRoot-C9F5dhZi.js +1 -0
- package/dashboard/static/assets/{ProjectsDrillPanel-B-HZkDnT.js → ProjectsDrillPanel-WxAdRDMt.js} +1 -1
- package/dashboard/static/assets/{SourceDetailModal-QhJkHB7i.js → SourceDetailModal-NyyEfXmc.js} +1 -1
- package/dashboard/static/assets/{UpdateModal-CkXEIQcK.js → UpdateModal-CFLq9pT1.js} +1 -1
- package/dashboard/static/assets/{index-HKS1s56o.js → index-BKTWH9JE.js} +2 -2
- package/dashboard/static/assets/{outlineNavigation-BNGaNPCp.js → outlineNavigation-BnGUwCS3.js} +3 -3
- package/dashboard/static/dashboard.html +1 -1
- package/package.json +4 -1
- package/dashboard/static/assets/ConversationsView-Dcfn8T6f.js +0 -72
- package/dashboard/static/assets/ModalRoot-Ck3WJK2V.js +0 -1
package/bin/_cctally_cache.py
CHANGED
|
@@ -105,6 +105,7 @@ import contextvars
|
|
|
105
105
|
import datetime as dt
|
|
106
106
|
import errno
|
|
107
107
|
import fcntl
|
|
108
|
+
import hashlib
|
|
108
109
|
import json
|
|
109
110
|
import os
|
|
110
111
|
import pathlib
|
|
@@ -117,6 +118,7 @@ import subprocess
|
|
|
117
118
|
import sys
|
|
118
119
|
import tempfile
|
|
119
120
|
import time
|
|
121
|
+
import uuid
|
|
120
122
|
from dataclasses import asdict, dataclass, field, replace as _dc_replace
|
|
121
123
|
from typing import Any, Callable, Iterable, Iterator, NamedTuple
|
|
122
124
|
|
|
@@ -708,6 +710,14 @@ CODEX_REPLAY_DRAIN_MARKER_NAME = "codex-replay-drain.last-attempt"
|
|
|
708
710
|
#: far more often than the `doctor` WARN it produces is actionable.
|
|
709
711
|
CODEX_REPLAY_DRAIN_SPAWN_THROTTLE_SECONDS = 3600
|
|
710
712
|
|
|
713
|
+
#: #982 spec 4.5a: the same spawn's spacing while a well-formed heal record
|
|
714
|
+
#: exists. A pass that ended `contended` or `degraded` leaves the record (in
|
|
715
|
+
#: phase `units`, a global fence over the population and every one-way stamp),
|
|
716
|
+
#: and the executor resumes it per file, so the retry costs one unit's work,
|
|
717
|
+
#: not a whole-tree re-read; waiting the hourly slot would keep the fence up
|
|
718
|
+
#: for an hour after any stall.
|
|
719
|
+
CODEX_HEAL_RETRY_THROTTLE_SECONDS = 300
|
|
720
|
+
|
|
711
721
|
#: How long the drain worker waits for the cache writer flocks. The default
|
|
712
722
|
#: ``None`` is a single non-blocking attempt, which spends the whole hourly slot
|
|
713
723
|
#: on whichever ordinary sync happened to hold the lock at that instant. Nobody
|
|
@@ -871,6 +881,22 @@ def _set_cache_meta_if_changed(
|
|
|
871
881
|
)
|
|
872
882
|
|
|
873
883
|
|
|
884
|
+
def _clear_claude_walk_verdicts(conn: sqlite3.Connection) -> None:
|
|
885
|
+
"""Delete both Claude walk verdicts inside a destructive wipe's transaction.
|
|
886
|
+
|
|
887
|
+
The walk-complete marker and #916's residual token each vouch for the
|
|
888
|
+
cache's contents, so a rebuild or truncation wipe drops them in the same
|
|
889
|
+
transaction as the rows: a crash between the wipe's commit and the end of
|
|
890
|
+
its walk must not leave either one vouching for an emptied store. The
|
|
891
|
+
caller commits.
|
|
892
|
+
"""
|
|
893
|
+
conn.execute(
|
|
894
|
+
"DELETE FROM cache_meta WHERE key IN (?, ?)",
|
|
895
|
+
("claude_ingest_walk_complete",
|
|
896
|
+
_ingest_frontier.CLAUDE_WALK_ORPHAN_RESIDUAL_KEY),
|
|
897
|
+
)
|
|
898
|
+
|
|
899
|
+
|
|
874
900
|
def _withdraw_cache_meta_marker(conn: sqlite3.Connection, key: str) -> None:
|
|
875
901
|
"""Remove a walk-completion marker, writing only when it is present.
|
|
876
902
|
|
|
@@ -1076,6 +1102,29 @@ class ProjectKey:
|
|
|
1076
1102
|
is_no_git: bool = field(default=False, compare=False)
|
|
1077
1103
|
|
|
1078
1104
|
|
|
1105
|
+
def _is_repository_marker(marker: str) -> bool:
|
|
1106
|
+
"""Whether a `.git` entry marks a repository for the project walk (#976).
|
|
1107
|
+
|
|
1108
|
+
A bounded structural filter, not a Git validity check: an absent entry (or
|
|
1109
|
+
a dangling symlink) is no marker; a `.git` DIRECTORY (a symlink to one
|
|
1110
|
+
included) counts only when it holds ``HEAD`` and the ``objects/`` and
|
|
1111
|
+
``refs/`` directories, so a stray non-repository `.git/` in a worktree
|
|
1112
|
+
container is skipped; anything else (a worktree or submodule `gitdir:`
|
|
1113
|
+
file) counts by presence. Metadata calls only: nothing is opened and no
|
|
1114
|
+
`gitdir:` or `commondir` is followed, which keeps the walk inside the
|
|
1115
|
+
frozen-roots stat contract (`bench/write-attribution/frozen_roots.py`).
|
|
1116
|
+
Only the `git-root` walk uses this; `_is_dev_checkout()` and the other
|
|
1117
|
+
`.git` presence predicates keep their own installation-safety contract.
|
|
1118
|
+
"""
|
|
1119
|
+
if not os.path.exists(marker):
|
|
1120
|
+
return False
|
|
1121
|
+
if os.path.isdir(marker):
|
|
1122
|
+
return (os.path.exists(os.path.join(marker, "HEAD"))
|
|
1123
|
+
and os.path.isdir(os.path.join(marker, "objects"))
|
|
1124
|
+
and os.path.isdir(os.path.join(marker, "refs")))
|
|
1125
|
+
return True
|
|
1126
|
+
|
|
1127
|
+
|
|
1079
1128
|
def _resolve_project_key(
|
|
1080
1129
|
project_path: str | None,
|
|
1081
1130
|
mode: str, # "git-root" | "full-path"
|
|
@@ -1083,8 +1132,9 @@ def _resolve_project_key(
|
|
|
1083
1132
|
) -> ProjectKey:
|
|
1084
1133
|
"""Resolve a raw project_path to its ProjectKey.
|
|
1085
1134
|
|
|
1086
|
-
Walks parents looking for
|
|
1087
|
-
git
|
|
1135
|
+
Walks parents looking for a repository marker (`_is_repository_marker`: a
|
|
1136
|
+
`.git` file, or a `.git` directory holding HEAD, objects/ and refs/) to
|
|
1137
|
+
find the canonical git-root. Non-git paths fall back to the normalized path. NULL input
|
|
1088
1138
|
becomes a literal `(unknown)` bucket.
|
|
1089
1139
|
"""
|
|
1090
1140
|
if project_path is None:
|
|
@@ -1136,7 +1186,7 @@ def _resolve_project_key(
|
|
|
1136
1186
|
while True:
|
|
1137
1187
|
if cur == home or cur == "/" or os.path.dirname(cur) == cur:
|
|
1138
1188
|
break
|
|
1139
|
-
if
|
|
1189
|
+
if _is_repository_marker(os.path.join(cur, ".git")):
|
|
1140
1190
|
key = ProjectKey(
|
|
1141
1191
|
bucket_path=cur,
|
|
1142
1192
|
display_key=os.path.basename(cur) or cur,
|
|
@@ -1708,12 +1758,18 @@ class CodexResetAnchorResolver:
|
|
|
1708
1758
|
"""
|
|
1709
1759
|
|
|
1710
1760
|
def __init__(self, conn: sqlite3.Connection, *,
|
|
1711
|
-
deadline: "float | None" = None, clock=None
|
|
1761
|
+
deadline: "float | None" = None, clock=None,
|
|
1762
|
+
seed_source: "Callable[[tuple[str, str, str, object], str], _lib_quota.ResetAnchorComponents | None] | None" = None):
|
|
1712
1763
|
self._conn = conn
|
|
1713
1764
|
# #980 F3: a budgeted walk passes its walk deadline and clock, which
|
|
1714
1765
|
# bound the group seed read; unbudgeted callers pass neither.
|
|
1715
1766
|
self._deadline = deadline
|
|
1716
1767
|
self._clock = clock if clock is not None else time.monotonic
|
|
1768
|
+
# #975 (spec 4.2): `seed_source(group, logical_limit_key)` supplies a
|
|
1769
|
+
# group's committed evidence in place of the seed SELECT whenever it
|
|
1770
|
+
# returns components (the provenance drain hands out a private copy of
|
|
1771
|
+
# its committed-evidence base); `None` falls back to the SELECT.
|
|
1772
|
+
self._seed_source = seed_source
|
|
1717
1773
|
self._groups: "dict[tuple[str, str, str, object], _lib_quota.ResetAnchorComponents]" = {}
|
|
1718
1774
|
self._pending_merges: list[
|
|
1719
1775
|
tuple[tuple[str, str, str, object], str, dt.datetime,
|
|
@@ -1744,6 +1800,11 @@ class CodexResetAnchorResolver:
|
|
|
1744
1800
|
components = self._groups.get(group)
|
|
1745
1801
|
if components is not None:
|
|
1746
1802
|
return components
|
|
1803
|
+
if self._seed_source is not None:
|
|
1804
|
+
components = self._seed_source(group, logical_limit_key)
|
|
1805
|
+
if components is not None:
|
|
1806
|
+
self._groups[group] = components
|
|
1807
|
+
return components
|
|
1747
1808
|
components = _lib_quota.ResetAnchorComponents()
|
|
1748
1809
|
if not self._seed_failed:
|
|
1749
1810
|
candidates = _lib_jsonl.codex_snap_equivalent_limit_keys(
|
|
@@ -1922,6 +1983,219 @@ def _codex_anchor_iso(value: dt.datetime) -> str:
|
|
|
1922
1983
|
return value.astimezone(dt.timezone.utc).isoformat().replace("+00:00", "Z")
|
|
1923
1984
|
|
|
1924
1985
|
|
|
1986
|
+
class _CodexWalkShare:
|
|
1987
|
+
"""History-sized state shared across the admitted units of one pass.
|
|
1988
|
+
|
|
1989
|
+
#982 / #975 spec 4.2. One in-process object per heal-executor or
|
|
1990
|
+
provenance pass, never persisted. Every reuse is guarded by a token read
|
|
1991
|
+
under the writer locks: ``(st_dev, st_ino)`` of cache.db, ``PRAGMA
|
|
1992
|
+
schema_version``, ``quota_window_change_log``'s ``sqlite_sequence`` value
|
|
1993
|
+
and ``codex_physical_mutation_seq``. The ledger's three triggers record
|
|
1994
|
+
every insert, delete and seeded-column update of a Codex quota row, so any
|
|
1995
|
+
other writer's quota mutation between two units changes the token; while
|
|
1996
|
+
the ledger table or a trigger is missing the token is ``None`` and nothing
|
|
1997
|
+
is reused. A bind consumes the seal, so a unit that never sealed (an
|
|
1998
|
+
exception, a rollback, a refusal) leaves the next unit seeding afresh.
|
|
1999
|
+
|
|
2000
|
+
- ``bind_walk`` / ``seal``: the walk memo, one ``CodexResetAnchorResolver``
|
|
2001
|
+
shared across population units, i.e. the single walk's own resolver.
|
|
2002
|
+
- ``identity_resolver`` / ``absorb_identity`` / ``seal``: the provenance
|
|
2003
|
+
committed-evidence seed. Each identity resolves against copies of a
|
|
2004
|
+
per-group base equal to a fresh seed of the committed rows; after the
|
|
2005
|
+
identity commits, the base gains exactly that identity's stored rows.
|
|
2006
|
+
Components and anchors do not depend on insertion order, so base plus
|
|
2007
|
+
absorbed rows equals a fresh seed, and evidence an identity resolved but
|
|
2008
|
+
did not store dies with its copy, exactly as a new resolver per call.
|
|
2009
|
+
- ``ledger_paths``: the accounting ledger's path set, read in full once
|
|
2010
|
+
and then only past the last ``seq`` seen. A positive signal only (I7),
|
|
2011
|
+
so a path whose rows a prune later removed may stay in the set.
|
|
2012
|
+
- ``heal_payload``: the heal record's population and D1 set, parsed once
|
|
2013
|
+
per pass and reused while the fence key keeps the same ``gen`` (a
|
|
2014
|
+
payload is only ever written with a new ``gen``).
|
|
2015
|
+
"""
|
|
2016
|
+
|
|
2017
|
+
_LEDGER_TRIGGERS = (
|
|
2018
|
+
"trg_qws_ledger_ins", "trg_qws_ledger_del", "trg_qws_ledger_upd")
|
|
2019
|
+
|
|
2020
|
+
def __init__(self) -> None:
|
|
2021
|
+
self._sealed: "tuple | None" = None
|
|
2022
|
+
self._walk: "CodexResetAnchorResolver | None" = None
|
|
2023
|
+
self._base: "CodexResetAnchorResolver | None" = None
|
|
2024
|
+
self._ledger: "set[str] | None" = None
|
|
2025
|
+
self._ledger_seq = 0
|
|
2026
|
+
self._ledger_identity: "tuple | None" = None
|
|
2027
|
+
self._heal_payload: "CodexHealPayload | None" = None
|
|
2028
|
+
|
|
2029
|
+
def heal_payload(self, conn: sqlite3.Connection,
|
|
2030
|
+
record: "dict | None") -> "CodexHealPayload | None":
|
|
2031
|
+
"""The record's parsed payload: the one this pass already parsed
|
|
2032
|
+
while ``record`` keeps its ``gen``, otherwise parsed once now."""
|
|
2033
|
+
if record is None or record.get("malformed"):
|
|
2034
|
+
return None
|
|
2035
|
+
cached = self._heal_payload
|
|
2036
|
+
if cached is not None and cached.gen == record.get("gen"):
|
|
2037
|
+
return cached
|
|
2038
|
+
payload = load_codex_heal_payload(conn, record)
|
|
2039
|
+
if payload is not None:
|
|
2040
|
+
self._heal_payload = payload
|
|
2041
|
+
return payload
|
|
2042
|
+
|
|
2043
|
+
def remember_heal_payload(self, payload: "CodexHealPayload") -> None:
|
|
2044
|
+
"""A payload this pass wrote itself (the history start unit)."""
|
|
2045
|
+
self._heal_payload = payload
|
|
2046
|
+
|
|
2047
|
+
def forget_heal_payload(self) -> None:
|
|
2048
|
+
"""A pass that ended ``restart`` (spec 4.5a) keeps no payload."""
|
|
2049
|
+
self._heal_payload = None
|
|
2050
|
+
|
|
2051
|
+
@classmethod
|
|
2052
|
+
def token(cls, conn: sqlite3.Connection) -> "tuple | None":
|
|
2053
|
+
"""The reuse token, or ``None`` when no reuse may be trusted."""
|
|
2054
|
+
from _cctally_quota import codex_physical_mutation_seq
|
|
2055
|
+
try:
|
|
2056
|
+
main = next((row[2] for row in conn.execute("PRAGMA database_list")
|
|
2057
|
+
if row[1] == "main"), None)
|
|
2058
|
+
if not main:
|
|
2059
|
+
return None
|
|
2060
|
+
st = os.stat(main)
|
|
2061
|
+
present = {row[0] for row in conn.execute(
|
|
2062
|
+
"SELECT name FROM sqlite_master "
|
|
2063
|
+
"WHERE (type='table' AND name='quota_window_change_log') "
|
|
2064
|
+
" OR (type='trigger' AND name IN (?,?,?))",
|
|
2065
|
+
cls._LEDGER_TRIGGERS)}
|
|
2066
|
+
if len(present) != 1 + len(cls._LEDGER_TRIGGERS):
|
|
2067
|
+
return None
|
|
2068
|
+
schema_version = int(
|
|
2069
|
+
conn.execute("PRAGMA schema_version").fetchone()[0])
|
|
2070
|
+
row = conn.execute(
|
|
2071
|
+
"SELECT seq FROM sqlite_sequence "
|
|
2072
|
+
"WHERE name='quota_window_change_log'").fetchone()
|
|
2073
|
+
ledger_seq = 0 if row is None else int(row[0])
|
|
2074
|
+
except (sqlite3.Error, OSError, TypeError, ValueError):
|
|
2075
|
+
return None
|
|
2076
|
+
return (st.st_dev, st.st_ino, schema_version, ledger_seq,
|
|
2077
|
+
codex_physical_mutation_seq(conn))
|
|
2078
|
+
|
|
2079
|
+
def _validate(self, conn: sqlite3.Connection) -> None:
|
|
2080
|
+
token = self.token(conn)
|
|
2081
|
+
if token is None or self._sealed is None or token != self._sealed:
|
|
2082
|
+
self._walk = None
|
|
2083
|
+
self._base = None
|
|
2084
|
+
self._sealed = None
|
|
2085
|
+
|
|
2086
|
+
def bind_walk(self, conn: sqlite3.Connection) -> CodexResetAnchorResolver:
|
|
2087
|
+
"""The shared walk resolver rebound to ``conn`` (new on a token change)."""
|
|
2088
|
+
self._validate(conn)
|
|
2089
|
+
if self._walk is None:
|
|
2090
|
+
self._walk = CodexResetAnchorResolver(conn)
|
|
2091
|
+
else:
|
|
2092
|
+
self._walk._conn = conn
|
|
2093
|
+
self._walk._pending_merges.clear()
|
|
2094
|
+
return self._walk
|
|
2095
|
+
|
|
2096
|
+
def seal(self, conn: sqlite3.Connection) -> None:
|
|
2097
|
+
"""Record the token after this unit's last commit, under its locks."""
|
|
2098
|
+
self._sealed = self.token(conn)
|
|
2099
|
+
|
|
2100
|
+
def invalidate(self) -> None:
|
|
2101
|
+
"""Drop everything: the next unit seeds from the committed store."""
|
|
2102
|
+
self._sealed = None
|
|
2103
|
+
self._walk = None
|
|
2104
|
+
self._base = None
|
|
2105
|
+
self._ledger = None
|
|
2106
|
+
self._ledger_seq = 0
|
|
2107
|
+
self._ledger_identity = None
|
|
2108
|
+
|
|
2109
|
+
def identity_resolver(
|
|
2110
|
+
self, conn: sqlite3.Connection) -> CodexResetAnchorResolver:
|
|
2111
|
+
"""A resolver for one provenance identity over copies of the base."""
|
|
2112
|
+
self._validate(conn)
|
|
2113
|
+
if self._base is None:
|
|
2114
|
+
self._base = CodexResetAnchorResolver(conn)
|
|
2115
|
+
else:
|
|
2116
|
+
self._base._conn = conn
|
|
2117
|
+
return CodexResetAnchorResolver(conn, seed_source=self._seed_copy)
|
|
2118
|
+
|
|
2119
|
+
def _seed_copy(self, group, logical_limit_key: str):
|
|
2120
|
+
base = self._base
|
|
2121
|
+
if base is None:
|
|
2122
|
+
return None
|
|
2123
|
+
components = base._components_for(group, logical_limit_key)
|
|
2124
|
+
if base._seed_failed:
|
|
2125
|
+
# Never remember a degraded seed; the identity's own resolver
|
|
2126
|
+
# then degrades exactly as a new resolver per call does.
|
|
2127
|
+
self._base = None
|
|
2128
|
+
return None
|
|
2129
|
+
return components.copy()
|
|
2130
|
+
|
|
2131
|
+
def absorb_identity(
|
|
2132
|
+
self, conn: sqlite3.Connection, source_path: str) -> None:
|
|
2133
|
+
"""Add one committed identity's stored quota rows to the base.
|
|
2134
|
+
|
|
2135
|
+
Mirrors the seed SELECT's membership exactly: a row belongs to every
|
|
2136
|
+
seeded group with its root and slot whose snap-equivalent limit keys
|
|
2137
|
+
include the row's key (the seed does not filter on the window), with
|
|
2138
|
+
its real order key ``(source_path, line_offset, id)``. A group not yet
|
|
2139
|
+
seeded is skipped: its first seed reads these rows anyway.
|
|
2140
|
+
"""
|
|
2141
|
+
base = self._base
|
|
2142
|
+
if base is None or not base._groups:
|
|
2143
|
+
return
|
|
2144
|
+
by_scope: "dict[tuple[str, str], list]" = {}
|
|
2145
|
+
for group in base._groups:
|
|
2146
|
+
by_scope.setdefault((group[0], group[1]), []).append(group)
|
|
2147
|
+
limit_keys: "dict[tuple, frozenset[str]]" = {}
|
|
2148
|
+
for (slot, limit_key, _window, root, path, offset, row_id,
|
|
2149
|
+
raw_text) in conn.execute(
|
|
2150
|
+
"SELECT observed_slot, logical_limit_key, window_minutes, "
|
|
2151
|
+
"source_root_key, source_path, line_offset, id, resets_at_utc "
|
|
2152
|
+
"FROM quota_window_snapshots "
|
|
2153
|
+
"WHERE source='codex' AND source_path=?", (str(source_path),)):
|
|
2154
|
+
groups = by_scope.get((root, slot))
|
|
2155
|
+
if not groups:
|
|
2156
|
+
continue
|
|
2157
|
+
raw = _parse_anchor_iso(raw_text)
|
|
2158
|
+
if raw is None:
|
|
2159
|
+
continue
|
|
2160
|
+
for group in groups:
|
|
2161
|
+
keys = limit_keys.get(group)
|
|
2162
|
+
if keys is None:
|
|
2163
|
+
keys = limit_keys[group] = frozenset(
|
|
2164
|
+
_lib_jsonl.codex_snap_equivalent_limit_keys(
|
|
2165
|
+
str(group[2])))
|
|
2166
|
+
if limit_key in keys:
|
|
2167
|
+
base._groups[group].add(
|
|
2168
|
+
raw, order_key=(str(path), int(offset), int(row_id)))
|
|
2169
|
+
|
|
2170
|
+
def ledger_paths(self, conn: sqlite3.Connection) -> "frozenset[str]":
|
|
2171
|
+
"""``_codex_ledger_paths`` across units: one full read per pass, then
|
|
2172
|
+
only the ledger rows past the last ``seq`` seen. A cache.db file or
|
|
2173
|
+
``schema_version`` change (or an unusable token) forces a full read."""
|
|
2174
|
+
token = self.token(conn)
|
|
2175
|
+
identity = None if token is None else token[:3]
|
|
2176
|
+
if (identity is None or self._ledger is None
|
|
2177
|
+
or identity != self._ledger_identity):
|
|
2178
|
+
rows = conn.execute(
|
|
2179
|
+
"SELECT seq, source_path FROM codex_accounting_change_log "
|
|
2180
|
+
"WHERE change_kind='path'").fetchall()
|
|
2181
|
+
self._ledger, self._ledger_seq = set(), 0
|
|
2182
|
+
self._ledger_identity = identity
|
|
2183
|
+
else:
|
|
2184
|
+
rows = conn.execute(
|
|
2185
|
+
"SELECT seq, source_path FROM codex_accounting_change_log "
|
|
2186
|
+
"WHERE change_kind='path' AND seq > ?",
|
|
2187
|
+
(self._ledger_seq,)).fetchall()
|
|
2188
|
+
for seq, path in rows:
|
|
2189
|
+
if path:
|
|
2190
|
+
self._ledger.add(path)
|
|
2191
|
+
if seq > self._ledger_seq:
|
|
2192
|
+
self._ledger_seq = seq
|
|
2193
|
+
paths = frozenset(self._ledger)
|
|
2194
|
+
if identity is None:
|
|
2195
|
+
self._ledger = None
|
|
2196
|
+
return paths
|
|
2197
|
+
|
|
2198
|
+
|
|
1925
2199
|
def _codex_provider_roots() -> list[CodexProviderRoot]:
|
|
1926
2200
|
"""Return configured provider roots with their sessions/direct walk roots.
|
|
1927
2201
|
|
|
@@ -4258,14 +4532,26 @@ def _codex_barrier_seal_pending(conn, *, priority=(), max_keys=None,
|
|
|
4258
4532
|
|
|
4259
4533
|
The caller holds the cache writer locks and has committed its own work.
|
|
4260
4534
|
Reads only cache.db (no stat, no decode) and commits once.
|
|
4261
|
-
"""
|
|
4262
|
-
if allow_init:
|
|
4263
|
-
_codex_barrier_init_targets(conn)
|
|
4264
4535
|
|
|
4536
|
+
#982 spec 4.3 (seal fence): while a heal fence holds, nothing is sealed
|
|
4537
|
+
(census ``heal-fenced``, no scan needed, so no caller requests the worker
|
|
4538
|
+
for it): a per-file unit could otherwise seal a conversation whose
|
|
4539
|
+
cursorless population contributor is still unread, under a census set
|
|
4540
|
+
before the fence. Sealing resumes after the transition, whose piece sets
|
|
4541
|
+
the census from the whole walk's counters.
|
|
4542
|
+
"""
|
|
4265
4543
|
def remaining() -> int:
|
|
4266
4544
|
return int(conn.execute(
|
|
4267
4545
|
"SELECT COUNT(*) FROM codex_barrier_repairs").fetchone()[0])
|
|
4268
4546
|
|
|
4547
|
+
if _codex_heal_fence_active(conn):
|
|
4548
|
+
if conn.in_transaction:
|
|
4549
|
+
conn.commit()
|
|
4550
|
+
return BarrierSealResult(0, 0, remaining(), "heal-fenced", False,
|
|
4551
|
+
start_after)
|
|
4552
|
+
if allow_init:
|
|
4553
|
+
_codex_barrier_init_targets(conn)
|
|
4554
|
+
|
|
4269
4555
|
if not _codex_barrier_census_proven(conn):
|
|
4270
4556
|
if conn.in_transaction:
|
|
4271
4557
|
conn.commit()
|
|
@@ -4334,7 +4620,10 @@ def _codex_barrier_after_walk(conn, stats, *, targeted: bool, deadline,
|
|
|
4334
4620
|
set(stats.drain_requested) | {CODEX_REPLAY_DRAIN_COMMAND}))
|
|
4335
4621
|
|
|
4336
4622
|
|
|
4337
|
-
def _codex_admitted_cache_write(
|
|
4623
|
+
def _codex_admitted_cache_write(
|
|
4624
|
+
fn, *, wait_for_requests: bool,
|
|
4625
|
+
after_commit: "Callable[[sqlite3.Connection], None] | None" = None,
|
|
4626
|
+
contention_bound: "float | None" = None):
|
|
4338
4627
|
"""Run ``fn(conn)`` through the drain gate under the cache lock domain.
|
|
4339
4628
|
|
|
4340
4629
|
The store is opened BEFORE admission takes the cache flocks (review N5;
|
|
@@ -4347,10 +4636,20 @@ def _codex_admitted_cache_write(fn, *, wait_for_requests: bool):
|
|
|
4347
4636
|
is busy". A connection opened for a refused admission is closed before
|
|
4348
4637
|
the backoff, and the next attempt opens a fresh one.
|
|
4349
4638
|
|
|
4639
|
+
``after_commit(conn)`` runs after ``fn``'s transaction committed and
|
|
4640
|
+
before ``adm.finish``, so it still holds both store locks: the provenance
|
|
4641
|
+
drain (#975, spec 4.2) refreshes its committed-evidence seed there, where
|
|
4642
|
+
no other writer can have interleaved. It does not run when ``fn`` raises.
|
|
4643
|
+
|
|
4644
|
+
``contention_bound`` (a heal unit's, #982): store locks another writer
|
|
4645
|
+
holds continuously (no fresh hook request in between) for that many
|
|
4646
|
+
seconds raise ``_CodexHealContended``, as every other heal unit gives up.
|
|
4647
|
+
|
|
4350
4648
|
Returns ``(True, result)``, or ``(False, None)`` when a fresh hook request
|
|
4351
4649
|
made the caller yield and it may not wait.
|
|
4352
4650
|
"""
|
|
4353
4651
|
import _lib_codex_drain_admission as adm
|
|
4652
|
+
contended_since: "float | None" = None
|
|
4354
4653
|
while True:
|
|
4355
4654
|
conn = open_cache_db()
|
|
4356
4655
|
try:
|
|
@@ -4364,12 +4663,23 @@ def _codex_admitted_cache_write(fn, *, wait_for_requests: bool):
|
|
|
4364
4663
|
conn.close()
|
|
4365
4664
|
if not wait_for_requests:
|
|
4366
4665
|
return False, None
|
|
4666
|
+
if contention_bound is not None:
|
|
4667
|
+
if adm.fresh_requests(_cctally_core.APP_DIR):
|
|
4668
|
+
contended_since = None
|
|
4669
|
+
else:
|
|
4670
|
+
now = time.monotonic()
|
|
4671
|
+
if contended_since is None:
|
|
4672
|
+
contended_since = now
|
|
4673
|
+
if now - contended_since >= contention_bound:
|
|
4674
|
+
raise _CodexHealContended()
|
|
4367
4675
|
_drain_backoff(adm)
|
|
4368
4676
|
try:
|
|
4369
4677
|
try:
|
|
4370
4678
|
result = fn(conn)
|
|
4371
4679
|
if conn.in_transaction:
|
|
4372
4680
|
conn.commit()
|
|
4681
|
+
if after_commit is not None:
|
|
4682
|
+
after_commit(conn)
|
|
4373
4683
|
return True, result
|
|
4374
4684
|
except BaseException:
|
|
4375
4685
|
conn.rollback()
|
|
@@ -4608,9 +4918,14 @@ def _delete_codex_file_derived_rows(
|
|
|
4608
4918
|
"DELETE FROM codex_session_entries WHERE source_path = ?" + root_clause,
|
|
4609
4919
|
params,
|
|
4610
4920
|
)
|
|
4921
|
+
# #971 spec D3: ``+`` keeps the root term out of index selection, so the
|
|
4922
|
+
# statement seeks the UNIQUE (source, source_path, ...) autoindex instead
|
|
4923
|
+
# of walking every quota row of the root through idx_qws_codex_load_order.
|
|
4924
|
+
quota_root_clause = (
|
|
4925
|
+
" AND +source_root_key IS ?" if match_source_root else "")
|
|
4611
4926
|
conn.execute(
|
|
4612
4927
|
"DELETE FROM quota_window_snapshots WHERE source = 'codex' "
|
|
4613
|
-
"AND source_path = ?" +
|
|
4928
|
+
"AND source_path = ?" + quota_root_clause,
|
|
4614
4929
|
params,
|
|
4615
4930
|
)
|
|
4616
4931
|
conn.execute(
|
|
@@ -4697,6 +5012,10 @@ def _prune_codex_accounting_change_log(
|
|
|
4697
5012
|
if retain_sequences < 1:
|
|
4698
5013
|
raise ValueError("retain_sequences must be positive")
|
|
4699
5014
|
try:
|
|
5015
|
+
if _codex_heal_fence_active(conn):
|
|
5016
|
+
# #982 spec 4.3: the heal's shared ledger path set reads these
|
|
5017
|
+
# rows; a due prune stays owed for the executor after the fence.
|
|
5018
|
+
return 0
|
|
4700
5019
|
cutoff = _codex_change_log_prune_cutoff(conn, retain_sequences)
|
|
4701
5020
|
if cutoff is None:
|
|
4702
5021
|
return 0
|
|
@@ -4970,16 +5289,14 @@ COVERAGE_WRITER_ACTIONS: "dict[str, str]" = {
|
|
|
4970
5289
|
"_cctally_journal.rehydrate_codex_journal_families": "preserve",
|
|
4971
5290
|
# Spec §4.3's migrations row, enumerated rather than named — and enumerated
|
|
4972
5291
|
# by the action each one TAKES, not by the action the spec's one-line row
|
|
4973
|
-
# assumed.
|
|
4974
|
-
# only
|
|
4975
|
-
#
|
|
4976
|
-
#
|
|
4977
|
-
#
|
|
5292
|
+
# assumed. No migration deletes rows the journal still retains: since #983
|
|
5293
|
+
# 024 only arms the guarded byte-zero replay, whose clear is the
|
|
5294
|
+
# `_clear_codex_derived_rows` entry above, so it writes no covered family
|
|
5295
|
+
# and is absent. `_038_codex_session_files_ingest_complete` is deliberately
|
|
5296
|
+
# ABSENT too: the review that asked for this row listed it, but it adds a
|
|
5297
|
+
# column to `codex_session_files`, which is not a covered family — which is
|
|
5298
|
+
# why the guard scans rather than trusts a hand list.
|
|
4978
5299
|
#
|
|
4979
|
-
# `DELETE FROM quota_window_snapshots WHERE source = 'codex'` with NO
|
|
4980
|
-
# sequence bump. Left standing, the certificate would be stale-VALID over a
|
|
4981
|
-
# cache holding zero Codex quota rows.
|
|
4982
|
-
"_cctally_db._024_codex_fused_ingest_rebuild": "invalidate",
|
|
4983
5300
|
# `UPDATE quota_window_snapshots SET observed_model = ?`, and it bumps
|
|
4984
5301
|
# `codex_physical_mutation_seq` in the same transaction whenever it changes
|
|
4985
5302
|
# a row. The certificate is left standing and goes invalid on the SEQUENCE
|
|
@@ -5222,6 +5539,11 @@ def _codex_conversation_project_attribution(
|
|
|
5222
5539
|
label, else git identity, else unassigned) so the browse live-recompute
|
|
5223
5540
|
fallback matches the stored rollup. Degrades to (None, None) when the S3
|
|
5224
5541
|
kernel or a valid source root is unavailable — never guesses.
|
|
5542
|
+
|
|
5543
|
+
Each call resolves with a fresh `_resolve_project_key` memo, so the answer
|
|
5544
|
+
never depends on which conversation resolved first: the no-git fallback
|
|
5545
|
+
memoizes the first raw spelling's label under the normalized path, which
|
|
5546
|
+
would hand one symlink alias's basename to another (#972 §5.8).
|
|
5225
5547
|
"""
|
|
5226
5548
|
if not source_root_key:
|
|
5227
5549
|
return None, None
|
|
@@ -5323,12 +5645,115 @@ def _next_conversation_render_revision(conn: sqlite3.Connection) -> int:
|
|
|
5323
5645
|
return revision
|
|
5324
5646
|
|
|
5325
5647
|
|
|
5648
|
+
def _codex_first_row_root(
|
|
5649
|
+
conn: sqlite3.Connection, conversation_key: str,
|
|
5650
|
+
) -> "str | None":
|
|
5651
|
+
"""The source root of a conversation's first normalized row, in the
|
|
5652
|
+
writer's order (#972 §5.3), without loading any message body.
|
|
5653
|
+
|
|
5654
|
+
Reads ``(timestamp_utc, id)`` index order and stops at the end of the
|
|
5655
|
+
first timestamp tie run, which `in_physical_order` then orders exactly as
|
|
5656
|
+
`_load_codex_normalized_rows` orders it. None when there is no row."""
|
|
5657
|
+
run: list = []
|
|
5658
|
+
cursor = conn.execute(
|
|
5659
|
+
"SELECT source_root_key, timestamp_utc, source_path, line_offset "
|
|
5660
|
+
"FROM codex_conversation_messages WHERE conversation_key=? "
|
|
5661
|
+
"ORDER BY timestamp_utc, id",
|
|
5662
|
+
(conversation_key,),
|
|
5663
|
+
)
|
|
5664
|
+
try:
|
|
5665
|
+
for row in cursor:
|
|
5666
|
+
if run and row[1] != run[0][1]:
|
|
5667
|
+
break
|
|
5668
|
+
run.append(row)
|
|
5669
|
+
finally:
|
|
5670
|
+
cursor.close()
|
|
5671
|
+
if not run:
|
|
5672
|
+
return None
|
|
5673
|
+
return _lib_codex_conversation.in_physical_order(
|
|
5674
|
+
run, timestamp=lambda row: row[1],
|
|
5675
|
+
position=lambda row: (row[2], row[3]))[0][0]
|
|
5676
|
+
|
|
5677
|
+
|
|
5678
|
+
def _codex_rollup_thread_fields(
|
|
5679
|
+
conn: sqlite3.Connection,
|
|
5680
|
+
conversation_key: str,
|
|
5681
|
+
*,
|
|
5682
|
+
first_row_root: "Callable[[], str | None]",
|
|
5683
|
+
attribution_cache: "dict | None" = None,
|
|
5684
|
+
require_thread: bool = False,
|
|
5685
|
+
) -> "tuple[str | None, str | None, str | None, str | None] | None":
|
|
5686
|
+
"""``(source_root_key, parent_thread_id, project_key, project_label)`` of
|
|
5687
|
+
one conversation's rollup, from its thread row (#972 §5.3).
|
|
5688
|
+
|
|
5689
|
+
The one derivation the rollup writer and the per-pass reconcile share, so
|
|
5690
|
+
the reconcile detects exactly what the writer would write. The thread
|
|
5691
|
+
root wins when non-empty; otherwise ``first_row_root()`` supplies the
|
|
5692
|
+
first normalized row's root (called lazily, only then).
|
|
5693
|
+
|
|
5694
|
+
``attribution_cache`` (#972 §5.8 rule 3) memoizes the project attribution
|
|
5695
|
+
per exact ``(source_root_key, cwd, git_json)`` input; each entry is still
|
|
5696
|
+
computed with a fresh resolver memo, so sharing it across conversations
|
|
5697
|
+
never makes one answer depend on another. With ``require_thread`` a
|
|
5698
|
+
conversation with no thread row answers None instead (rule 2: a missing
|
|
5699
|
+
thread row is no evidence for the reconcile)."""
|
|
5700
|
+
# #845 §4.7. The two columns are read as BLOBs and decoded field by
|
|
5701
|
+
# field, so one undecodable byte no longer raises here and no longer
|
|
5702
|
+
# fails the whole recompute. The table name stays UNQUALIFIED, because
|
|
5703
|
+
# this writer must honor account scoping: under
|
|
5704
|
+
# `scope_conversations_db_to_account` it sees the empty TEMP view on
|
|
5705
|
+
# purpose, and under #968's barrier the last-good view.
|
|
5706
|
+
thread = conn.execute(
|
|
5707
|
+
"SELECT CAST(cwd AS BLOB) AS cwd_blob, "
|
|
5708
|
+
"CAST(git_json AS BLOB) AS git_json_blob, parent_thread_id, "
|
|
5709
|
+
"source_root_key "
|
|
5710
|
+
"FROM codex_conversation_threads WHERE conversation_key = ?",
|
|
5711
|
+
(conversation_key,),
|
|
5712
|
+
).fetchone()
|
|
5713
|
+
if thread is None and require_thread:
|
|
5714
|
+
return None
|
|
5715
|
+
cwd = git_json = parent_thread_id = thread_root = None
|
|
5716
|
+
thread_malformed = False
|
|
5717
|
+
if thread is not None:
|
|
5718
|
+
cwd_blob, git_json_blob, parent_thread_id, thread_root = thread
|
|
5719
|
+
decoded = _lib_codex_metadata.decode_codex_project_metadata(
|
|
5720
|
+
cwd_blob, git_json_blob)
|
|
5721
|
+
cwd, git_json = decoded.cwd, decoded.git_json
|
|
5722
|
+
# The DIRECT-ONLY predicate: this writer resolves the thread's own
|
|
5723
|
+
# fields through `_codex_conversation_project_attribution`, which
|
|
5724
|
+
# stops at a usable `cwd`, so a valid `cwd` beside an undecodable
|
|
5725
|
+
# `git_json` is still attributed.
|
|
5726
|
+
thread_malformed = _lib_codex_metadata.codex_metadata_is_malformed(
|
|
5727
|
+
decoded, None)
|
|
5728
|
+
source_root_key = thread_root if thread_root else first_row_root()
|
|
5729
|
+
if thread_malformed:
|
|
5730
|
+
# `(unassigned)` is a real answer for a thread with no metadata.
|
|
5731
|
+
# Persisting it for a thread whose metadata could not be READ would
|
|
5732
|
+
# stamp a wrong answer that every later read prefers, which is the
|
|
5733
|
+
# failure class `docs/codex-gotchas.md` records for the materialized
|
|
5734
|
+
# `"(unassigned)"`. NULL is the absence every reader already
|
|
5735
|
+
# handles.
|
|
5736
|
+
return source_root_key, parent_thread_id, None, None
|
|
5737
|
+
if attribution_cache is None:
|
|
5738
|
+
project_key, project_label = _codex_conversation_project_attribution(
|
|
5739
|
+
source_root_key, cwd, git_json)
|
|
5740
|
+
else:
|
|
5741
|
+
inputs = (source_root_key, cwd, git_json)
|
|
5742
|
+
attribution = attribution_cache.get(inputs)
|
|
5743
|
+
if attribution is None:
|
|
5744
|
+
attribution = _codex_conversation_project_attribution(*inputs)
|
|
5745
|
+
attribution_cache[inputs] = attribution
|
|
5746
|
+
project_key, project_label = attribution
|
|
5747
|
+
return source_root_key, parent_thread_id, project_key, project_label
|
|
5748
|
+
|
|
5749
|
+
|
|
5326
5750
|
def _recompute_codex_rollups(
|
|
5327
5751
|
conn: sqlite3.Connection,
|
|
5328
5752
|
conversation_keys,
|
|
5329
5753
|
*,
|
|
5330
5754
|
advance_render_revision: bool = True,
|
|
5331
5755
|
inputs_changed: "bool | set[str] | frozenset[str]" = True,
|
|
5756
|
+
attribution_cache: "dict | None" = None,
|
|
5332
5757
|
) -> None:
|
|
5333
5758
|
"""Recompute-affected-or-delete the rollup for each conversation (§3.2).
|
|
5334
5759
|
|
|
@@ -5346,9 +5771,16 @@ def _recompute_codex_rollups(
|
|
|
5346
5771
|
outline memo keyed on it is never stale. Any other conversation's row is
|
|
5347
5772
|
written only when one of its stored values would change, so a true no-op
|
|
5348
5773
|
writes nothing (and reserves no revision).
|
|
5774
|
+
|
|
5775
|
+
#972: the thread-derived fields come from `_codex_rollup_thread_fields`,
|
|
5776
|
+
the derivation the per-pass reconcile shares; ``attribution_cache`` is the
|
|
5777
|
+
caller's per-pass attribution memo (none by default: every conversation
|
|
5778
|
+
resolves afresh). §5.8 rule 5: every key's outcome is derived before the
|
|
5779
|
+
first write, because the first write takes the conversations write lock
|
|
5780
|
+
and a key's derivation reads only its own rows and stored rollup.
|
|
5349
5781
|
"""
|
|
5350
5782
|
kern = _lib_codex_conversation
|
|
5351
|
-
keys = {key for key in conversation_keys if key}
|
|
5783
|
+
keys = sorted({key for key in conversation_keys if key})
|
|
5352
5784
|
if not keys:
|
|
5353
5785
|
return
|
|
5354
5786
|
reserved_revision: "int | None" = None
|
|
@@ -5361,13 +5793,12 @@ def _recompute_codex_rollups(
|
|
|
5361
5793
|
reserved_revision = _next_conversation_render_revision(conn)
|
|
5362
5794
|
return reserved_revision
|
|
5363
5795
|
|
|
5796
|
+
# (conversation_key, None) deletes; (conversation_key, values) upserts.
|
|
5797
|
+
outcomes: list = []
|
|
5364
5798
|
for conversation_key in keys:
|
|
5365
5799
|
rows = _load_codex_normalized_rows(conn, conversation_key)
|
|
5366
5800
|
if not rows:
|
|
5367
|
-
|
|
5368
|
-
"DELETE FROM codex_conversation_rollups WHERE conversation_key = ?",
|
|
5369
|
-
(conversation_key,),
|
|
5370
|
-
)
|
|
5801
|
+
outcomes.append((conversation_key, None))
|
|
5371
5802
|
continue
|
|
5372
5803
|
item_count = kern.rollup_item_count(rows)
|
|
5373
5804
|
title = kern.derive_title(rows)
|
|
@@ -5376,46 +5807,11 @@ def _recompute_codex_rollups(
|
|
|
5376
5807
|
last_activity = max(timestamps) if timestamps else None
|
|
5377
5808
|
models = sorted({r.model for r in rows if r.model})
|
|
5378
5809
|
models_json = json.dumps(models) if models else None
|
|
5379
|
-
|
|
5380
|
-
|
|
5381
|
-
|
|
5382
|
-
|
|
5383
|
-
|
|
5384
|
-
# `scope_conversations_db_to_account` it sees the empty TEMP view on
|
|
5385
|
-
# purpose.
|
|
5386
|
-
thread = conn.execute(
|
|
5387
|
-
"SELECT CAST(cwd AS BLOB) AS cwd_blob, "
|
|
5388
|
-
"CAST(git_json AS BLOB) AS git_json_blob, parent_thread_id, "
|
|
5389
|
-
"source_root_key "
|
|
5390
|
-
"FROM codex_conversation_threads WHERE conversation_key = ?",
|
|
5391
|
-
(conversation_key,),
|
|
5392
|
-
).fetchone()
|
|
5393
|
-
cwd = git_json = parent_thread_id = None
|
|
5394
|
-
thread_malformed = False
|
|
5395
|
-
if thread is not None:
|
|
5396
|
-
cwd_blob, git_json_blob, parent_thread_id, thread_root = thread
|
|
5397
|
-
decoded = _lib_codex_metadata.decode_codex_project_metadata(
|
|
5398
|
-
cwd_blob, git_json_blob)
|
|
5399
|
-
cwd, git_json = decoded.cwd, decoded.git_json
|
|
5400
|
-
# The DIRECT-ONLY predicate: this writer resolves the thread's own
|
|
5401
|
-
# fields through `_codex_conversation_project_attribution`, which
|
|
5402
|
-
# stops at a usable `cwd`, so a valid `cwd` beside an undecodable
|
|
5403
|
-
# `git_json` is still attributed.
|
|
5404
|
-
thread_malformed = _lib_codex_metadata.codex_metadata_is_malformed(
|
|
5405
|
-
decoded, None)
|
|
5406
|
-
if thread_root:
|
|
5407
|
-
source_root_key = thread_root
|
|
5408
|
-
if thread_malformed:
|
|
5409
|
-
# `(unassigned)` is a real answer for a thread with no metadata.
|
|
5410
|
-
# Persisting it for a thread whose metadata could not be READ would
|
|
5411
|
-
# stamp a wrong answer that every later read prefers, which is the
|
|
5412
|
-
# failure class `docs/codex-gotchas.md` records for the materialized
|
|
5413
|
-
# `"(unassigned)"`. NULL is the absence every reader already
|
|
5414
|
-
# handles.
|
|
5415
|
-
project_key = project_label = None
|
|
5416
|
-
else:
|
|
5417
|
-
project_key, project_label = _codex_conversation_project_attribution(
|
|
5418
|
-
source_root_key, cwd, git_json)
|
|
5810
|
+
first_root = rows[0].source_root_key
|
|
5811
|
+
source_root_key, parent_thread_id, project_key, project_label = (
|
|
5812
|
+
_codex_rollup_thread_fields(
|
|
5813
|
+
conn, conversation_key, first_row_root=lambda: first_root,
|
|
5814
|
+
attribution_cache=attribution_cache))
|
|
5419
5815
|
values = (source_root_key, parent_thread_id, item_count, started,
|
|
5420
5816
|
last_activity, project_key, project_label, models_json, title)
|
|
5421
5817
|
if inputs_changed is not True and conversation_key not in (
|
|
@@ -5429,6 +5825,14 @@ def _recompute_codex_rollups(
|
|
|
5429
5825
|
).fetchone()
|
|
5430
5826
|
if stored is not None and tuple(stored) == values:
|
|
5431
5827
|
continue
|
|
5828
|
+
outcomes.append((conversation_key, values))
|
|
5829
|
+
for conversation_key, values in outcomes:
|
|
5830
|
+
if values is None:
|
|
5831
|
+
conn.execute(
|
|
5832
|
+
"DELETE FROM codex_conversation_rollups WHERE conversation_key = ?",
|
|
5833
|
+
(conversation_key,),
|
|
5834
|
+
)
|
|
5835
|
+
continue
|
|
5432
5836
|
conn.execute(
|
|
5433
5837
|
"INSERT INTO codex_conversation_rollups "
|
|
5434
5838
|
"(conversation_key, source_root_key, parent_thread_id, item_count, "
|
|
@@ -5446,6 +5850,254 @@ def _recompute_codex_rollups(
|
|
|
5446
5850
|
)
|
|
5447
5851
|
|
|
5448
5852
|
|
|
5853
|
+
#: #972 §5.4: the most conversations one reconcile transaction rewrites.
|
|
5854
|
+
CODEX_ROLLUP_RECONCILE_BATCH = 64
|
|
5855
|
+
|
|
5856
|
+
|
|
5857
|
+
def _reconcile_codex_rollup_attribution(
|
|
5858
|
+
conn: sqlite3.Connection, stats: "CodexIngestStats",
|
|
5859
|
+
) -> int:
|
|
5860
|
+
"""Rewrite every rollup whose thread-derived fields drifted (#972 §5.4).
|
|
5861
|
+
|
|
5862
|
+
A rollup is materialized when the conversation pass writes it, from the
|
|
5863
|
+
thread metadata and filesystem of that moment. Neither a metadata-only
|
|
5864
|
+
contributor that accounting accepts later (the R4 interleave) nor a
|
|
5865
|
+
worktree removed later rewrites it, so every full pass compares each
|
|
5866
|
+
persisted rollup with what `_codex_rollup_thread_fields` derives now and
|
|
5867
|
+
rewrites the ones that differ through the writer itself.
|
|
5868
|
+
|
|
5869
|
+
Detection runs with no transaction open over a materialized list of
|
|
5870
|
+
`main.codex_conversation_rollups`; repair runs the sorted drifted keys in
|
|
5871
|
+
batches of at most `CODEX_ROLLUP_RECONCILE_BATCH` through
|
|
5872
|
+
`_reconcile_codex_rollup_batch`, each in its own explicit transaction
|
|
5873
|
+
(one snapshot of both stores per batch, one render revision per written
|
|
5874
|
+
batch). A database error rolls back its batch and propagates; earlier
|
|
5875
|
+
batches stay committed and the next pass detects the rest. Never on an
|
|
5876
|
+
account-scoped connection, whose TEMP rollups are a view of the
|
|
5877
|
+
persisted ones. Returns the conversations written, which it also adds to
|
|
5878
|
+
``stats.rollups_reattributed``.
|
|
5879
|
+
|
|
5880
|
+
Admission (§5.8, revisions 5 and 6) is per conversation and shared by
|
|
5881
|
+
detection and every batch (`_admit_codex_rollup_reconcile`): a
|
|
5882
|
+
conversation is compared only once accounting covers, from the same file
|
|
5883
|
+
generation, every `session_meta` line the conversation pass consumed for
|
|
5884
|
+
it (rule 1; a stale cursor of a replaced or truncated file is not
|
|
5885
|
+
coverage, `_codex_rollup_reconcile_uncovered`), never when it has no thread
|
|
5886
|
+
row (rule 2), and a non-database error deriving one rollup skips it
|
|
5887
|
+
(``stats.rollups_reconcile_skipped``, rule 4). No global condition, such
|
|
5888
|
+
as unrelated prune debt or a pending verification, gates the pass. One
|
|
5889
|
+
attribution cache keyed by the raw thread inputs serves the whole pass,
|
|
5890
|
+
and detection reads the rollups in key order (rule 3).
|
|
5891
|
+
"""
|
|
5892
|
+
if conn.execute(
|
|
5893
|
+
"SELECT 1 FROM sqlite_temp_master "
|
|
5894
|
+
"WHERE type='table' AND name='_conversation_account_scope'"
|
|
5895
|
+
).fetchone() is not None:
|
|
5896
|
+
raise RuntimeError(
|
|
5897
|
+
"the rollup reconcile never runs on an account-scoped connection")
|
|
5898
|
+
if conn.in_transaction:
|
|
5899
|
+
raise RuntimeError("the rollup reconcile needs no open transaction")
|
|
5900
|
+
cursor = conn.execute(
|
|
5901
|
+
"SELECT conversation_key, source_root_key, parent_thread_id, "
|
|
5902
|
+
"project_key, project_label FROM main.codex_conversation_rollups "
|
|
5903
|
+
"ORDER BY conversation_key")
|
|
5904
|
+
try:
|
|
5905
|
+
stored = cursor.fetchall()
|
|
5906
|
+
finally:
|
|
5907
|
+
cursor.close()
|
|
5908
|
+
attributions: dict = {}
|
|
5909
|
+
drifted = _admit_codex_rollup_reconcile(conn, stored, stats, attributions)
|
|
5910
|
+
drifted.sort()
|
|
5911
|
+
written = 0
|
|
5912
|
+
for start in range(0, len(drifted), CODEX_ROLLUP_RECONCILE_BATCH):
|
|
5913
|
+
changed = _reconcile_codex_rollup_batch(
|
|
5914
|
+
conn, drifted[start:start + CODEX_ROLLUP_RECONCILE_BATCH],
|
|
5915
|
+
stats, attributions)
|
|
5916
|
+
written += changed
|
|
5917
|
+
stats.rollups_reattributed += changed
|
|
5918
|
+
return written
|
|
5919
|
+
|
|
5920
|
+
|
|
5921
|
+
def _codex_rollup_reconcile_uncovered(
|
|
5922
|
+
conn: sqlite3.Connection, keys: "list[str] | None" = None,
|
|
5923
|
+
) -> "set[str]":
|
|
5924
|
+
"""Conversations whose consumed `session_meta` accounting has not yet
|
|
5925
|
+
processed from the same file generation (#972 §5.8 rule 1, revision 7).
|
|
5926
|
+
|
|
5927
|
+
A conversation is covered when, for every `(source_path, line_offset)`
|
|
5928
|
+
of its `session_meta` events, (a) `cache_db.codex_session_files` holds
|
|
5929
|
+
that path with `last_byte_offset` beyond that line, and (b) that
|
|
5930
|
+
accounting cursor and the conversation cursor for the same path
|
|
5931
|
+
(`main.codex_conversation_source_files`) agree on `size_bytes` and, when
|
|
5932
|
+
both describe the same physical container (equal `container_kind` and
|
|
5933
|
+
`container_path`), show no inode disagreement under
|
|
5934
|
+
`_lib_ingest_frontier.source_identity_replaced` (the inode decides; a
|
|
5935
|
+
missing or non-integer inode on either side is no evidence).
|
|
5936
|
+
|
|
5937
|
+
A path and an offset are coordinates, not generation evidence (VC-1):
|
|
5938
|
+
the conversation walk resets its cursor and the file's retained events
|
|
5939
|
+
when a different file now occupies the pathname, or when the file
|
|
5940
|
+
shrank, while an accounting walk that has not reached the file yet still
|
|
5941
|
+
holds the earlier incarnation's larger offset; comparing coordinates
|
|
5942
|
+
alone let the reconcile write the other contributor's project from a
|
|
5943
|
+
thread row that never saw the new metadata.
|
|
5944
|
+
|
|
5945
|
+
Only legs both walks converge on for every quiet file are compared, so
|
|
5946
|
+
the rule delays a repair while one walk lags and never withholds a
|
|
5947
|
+
settled conversation forever. Each walk adopts a verified container into
|
|
5948
|
+
its own cursor on its own schedule, so one cursor can describe the
|
|
5949
|
+
adopted `.zst` (its inode) for days while the other still describes the
|
|
5950
|
+
plain file it replaced (26 rows of 9 compressed conversations on a real
|
|
5951
|
+
store, 2026-11-05): inodes of different containers are not compared.
|
|
5952
|
+
Content anchors are not compared at all: after a partial read followed
|
|
5953
|
+
by an in-place shrink the two walks settle at one size on one inode with
|
|
5954
|
+
different anchors. The residual (spec §8, R9): a same-size in-place
|
|
5955
|
+
rewrite, or a same-size replacement under a cursor with no inode, while
|
|
5956
|
+
accounting has not re-read the file can write a project from the earlier
|
|
5957
|
+
generation, which the next full pass after accounting reads the file
|
|
5958
|
+
corrects. Decoded sizes, the logical identity (`_codex_identity_path`)
|
|
5959
|
+
and decoded offsets are shared by both walks for plain, compressed and
|
|
5960
|
+
archived rollouts alike. A conversation with no `session_meta` event is
|
|
5961
|
+
covered: there is nothing for accounting to lag behind.
|
|
5962
|
+
|
|
5963
|
+
One set-based read over the `session_meta` partial index
|
|
5964
|
+
(`idx_codex_events_session_meta_provenance`) joined to both cursors by
|
|
5965
|
+
path: never one query per conversation through the conversation-key
|
|
5966
|
+
index, which loads every event of each conversation (89.6 s for 1,251
|
|
5967
|
+
conversations on a real store, against 0.014 s for this read). For the
|
|
5968
|
+
same reason the statement constrains `conversation_key` only through a
|
|
5969
|
+
unary ``+`` and leaves de-duplication and NULL keys to Python: a
|
|
5970
|
+
``DISTINCT`` or an ``IS NOT NULL`` there makes SQLite prefer a scan of
|
|
5971
|
+
every event through that index. The statement returns only the rows
|
|
5972
|
+
that fail a leg SQL decides exactly (``hard``) or whose same-container
|
|
5973
|
+
inodes SQL sees as different; the inode verdict itself is the shared
|
|
5974
|
+
Python rule, so this read cannot drift from the walks' replacement test.
|
|
5975
|
+
"""
|
|
5976
|
+
hard = (
|
|
5977
|
+
"f.path IS NULL OR c.path IS NULL "
|
|
5978
|
+
"OR f.last_byte_offset <= e.line_offset "
|
|
5979
|
+
"OR f.size_bytes IS NOT c.size_bytes")
|
|
5980
|
+
same_container = (
|
|
5981
|
+
"f.container_kind IS c.container_kind "
|
|
5982
|
+
"AND f.container_path IS c.container_path")
|
|
5983
|
+
sql = (
|
|
5984
|
+
f"SELECT e.conversation_key, CASE WHEN {hard} THEN 1 ELSE 0 END, "
|
|
5985
|
+
"f.device_id, f.inode, c.device_id, c.inode "
|
|
5986
|
+
"FROM main.codex_conversation_events AS e "
|
|
5987
|
+
"LEFT JOIN cache_db.codex_session_files AS f ON f.path = e.source_path "
|
|
5988
|
+
# A conversation cursor exists beside its retained events: the walk
|
|
5989
|
+
# writes a file's events and its cursor in one savepoint, and the
|
|
5990
|
+
# prune drain deletes them together. Should one be missing anyway,
|
|
5991
|
+
# there is no generation to compare, so the line is uncovered.
|
|
5992
|
+
"LEFT JOIN main.codex_conversation_source_files AS c "
|
|
5993
|
+
"ON c.path = e.source_path "
|
|
5994
|
+
f"WHERE e.record_type = 'session_meta' AND ({hard} "
|
|
5995
|
+
f"OR ({same_container} AND f.inode IS NOT NULL "
|
|
5996
|
+
"AND c.inode IS NOT NULL AND f.inode IS NOT c.inode))")
|
|
5997
|
+
params: "tuple | list" = ()
|
|
5998
|
+
if keys is not None:
|
|
5999
|
+
if not keys:
|
|
6000
|
+
return set()
|
|
6001
|
+
sql += (" AND +e.conversation_key IN ("
|
|
6002
|
+
+ ",".join("?" * len(keys)) + ")")
|
|
6003
|
+
params = keys
|
|
6004
|
+
uncovered: "set[str]" = set()
|
|
6005
|
+
for key, is_hard, f_device, f_inode, c_device, c_inode in conn.execute(
|
|
6006
|
+
sql, params):
|
|
6007
|
+
if key is None or key in uncovered:
|
|
6008
|
+
continue
|
|
6009
|
+
if is_hard or _ingest_frontier.source_identity_replaced(
|
|
6010
|
+
f_device, f_inode, c_device, c_inode):
|
|
6011
|
+
uncovered.add(key)
|
|
6012
|
+
return uncovered
|
|
6013
|
+
|
|
6014
|
+
|
|
6015
|
+
def _admit_codex_rollup_reconcile(
|
|
6016
|
+
conn: sqlite3.Connection, stored: list,
|
|
6017
|
+
stats: "CodexIngestStats", attributions: dict,
|
|
6018
|
+
*, keys: "list[str] | None" = None,
|
|
6019
|
+
) -> "list[str]":
|
|
6020
|
+
"""The drifted, admitted keys among ``stored`` rows (#972 §5.8 rules 1,
|
|
6021
|
+
2, 4 and the drift comparison), read on the caller's snapshot.
|
|
6022
|
+
|
|
6023
|
+
``stored`` holds ``(conversation_key, source_root_key,
|
|
6024
|
+
parent_thread_id, project_key, project_label)`` rows; ``keys`` limits
|
|
6025
|
+
the coverage read to a batch's keys (None: every conversation), which
|
|
6026
|
+
compares both stores' cursors for the generation inside the caller's
|
|
6027
|
+
snapshot. The one admission detection and every repair batch share, so
|
|
6028
|
+
they cannot diverge. Fills ``attributions`` for each admitted
|
|
6029
|
+
derivation."""
|
|
6030
|
+
uncovered = _codex_rollup_reconcile_uncovered(conn, keys)
|
|
6031
|
+
drifted = []
|
|
6032
|
+
for conversation_key, *values in stored:
|
|
6033
|
+
if conversation_key in uncovered:
|
|
6034
|
+
continue
|
|
6035
|
+
try:
|
|
6036
|
+
expected = _codex_rollup_thread_fields(
|
|
6037
|
+
conn, conversation_key,
|
|
6038
|
+
first_row_root=lambda key=conversation_key: (
|
|
6039
|
+
_codex_first_row_root(conn, key)),
|
|
6040
|
+
attribution_cache=attributions, require_thread=True)
|
|
6041
|
+
except (ValueError, OSError, UnicodeError):
|
|
6042
|
+
# e.g. a cwd with an embedded NUL, which `realpath` refuses: this
|
|
6043
|
+
# rollup keeps its stored fields for the pass.
|
|
6044
|
+
stats.rollups_reconcile_skipped += 1
|
|
6045
|
+
continue
|
|
6046
|
+
if expected is not None and expected != tuple(values):
|
|
6047
|
+
drifted.append(conversation_key)
|
|
6048
|
+
return drifted
|
|
6049
|
+
|
|
6050
|
+
|
|
6051
|
+
def _reconcile_codex_rollup_batch(
|
|
6052
|
+
conn: sqlite3.Connection, batch: "list[str]",
|
|
6053
|
+
stats: "CodexIngestStats", attributions: dict,
|
|
6054
|
+
) -> int:
|
|
6055
|
+
"""Repair one batch of detected keys in one transaction (#972 §5.4,
|
|
6056
|
+
§5.8 rule 8); returns the conversations written.
|
|
6057
|
+
|
|
6058
|
+
Accounting writes under its own locks, so a thread row or cursor can
|
|
6059
|
+
change between detection and this batch. After ``BEGIN`` the batch
|
|
6060
|
+
re-admits its keys on the transaction's snapshot of both stores, which
|
|
6061
|
+
fills ``attributions``, and hands only the admitted, still-drifted keys to
|
|
6062
|
+
the writer; the writer reads the same snapshot and takes each attribution
|
|
6063
|
+
from the cache, so it writes nothing the admission did not see and makes
|
|
6064
|
+
no new filesystem call for those keys. A batch with no surviving key
|
|
6065
|
+
ends its read-only transaction and reserves no render revision."""
|
|
6066
|
+
if conn.in_transaction:
|
|
6067
|
+
raise RuntimeError("the rollup reconcile needs no open transaction")
|
|
6068
|
+
marks = ",".join("?" * len(batch))
|
|
6069
|
+
conn.execute("BEGIN")
|
|
6070
|
+
try:
|
|
6071
|
+
rows = conn.execute(
|
|
6072
|
+
"SELECT conversation_key, source_root_key, parent_thread_id, "
|
|
6073
|
+
"project_key, project_label, render_revision "
|
|
6074
|
+
"FROM main.codex_conversation_rollups "
|
|
6075
|
+
f"WHERE conversation_key IN ({marks}) ORDER BY conversation_key",
|
|
6076
|
+
batch).fetchall()
|
|
6077
|
+
before = {row[0]: row[5] for row in rows}
|
|
6078
|
+
admitted = _admit_codex_rollup_reconcile(
|
|
6079
|
+
conn, [row[:5] for row in rows], stats, attributions,
|
|
6080
|
+
keys=list(batch))
|
|
6081
|
+
if not admitted:
|
|
6082
|
+
conn.rollback()
|
|
6083
|
+
return 0
|
|
6084
|
+
_recompute_codex_rollups(
|
|
6085
|
+
conn, admitted, inputs_changed=set(),
|
|
6086
|
+
attribution_cache=attributions)
|
|
6087
|
+
after = dict(conn.execute(
|
|
6088
|
+
"SELECT conversation_key, render_revision "
|
|
6089
|
+
"FROM main.codex_conversation_rollups "
|
|
6090
|
+
f"WHERE conversation_key IN ({','.join('?' * len(admitted))})",
|
|
6091
|
+
admitted).fetchall())
|
|
6092
|
+
conn.commit()
|
|
6093
|
+
except BaseException:
|
|
6094
|
+
if conn.in_transaction:
|
|
6095
|
+
conn.rollback()
|
|
6096
|
+
raise
|
|
6097
|
+
_wal_checkpoint.after_commit(conn)
|
|
6098
|
+
return sum(1 for key in admitted if before.get(key) != after.get(key))
|
|
6099
|
+
|
|
6100
|
+
|
|
5449
6101
|
def _replay_codex_normalization(
|
|
5450
6102
|
conn: sqlite3.Connection,
|
|
5451
6103
|
*,
|
|
@@ -5715,8 +6367,11 @@ def _codex_identity_has_rows(
|
|
|
5715
6367
|
"WHERE path = ? AND source_root_key IS ? LIMIT 1",
|
|
5716
6368
|
"SELECT 1 FROM codex_session_entries "
|
|
5717
6369
|
"WHERE source_path = ? AND source_root_key IS ? LIMIT 1",
|
|
6370
|
+
# #971: ``+`` pins the quota probe to the UNIQUE (source, source_path,
|
|
6371
|
+
# ...) autoindex, as in _delete_codex_file_derived_rows; on the root
|
|
6372
|
+
# index an absent identity walked every quota row of the root.
|
|
5718
6373
|
"SELECT 1 FROM quota_window_snapshots WHERE source = 'codex' "
|
|
5719
|
-
"AND source_path = ? AND source_root_key IS ? LIMIT 1",
|
|
6374
|
+
"AND source_path = ? AND +source_root_key IS ? LIMIT 1",
|
|
5720
6375
|
"SELECT 1 FROM codex_conversation_threads "
|
|
5721
6376
|
"WHERE source_path = ? AND source_root_key IS ? LIMIT 1",
|
|
5722
6377
|
)
|
|
@@ -6350,9 +7005,14 @@ def _release_cache_store_locks(token) -> None:
|
|
|
6350
7005
|
def _drain_backoff(adm) -> None:
|
|
6351
7006
|
"""After a refused admission: wait out every fresh request; when none was
|
|
6352
7007
|
fresh (the gate or a store lock was busy), sleep one poll step so a
|
|
6353
|
-
waiting worker never spins on the gate.
|
|
7008
|
+
waiting worker never spins on the gate. The step is short when this
|
|
7009
|
+
process observed a fresh Claude waiter within the last
|
|
7010
|
+
``CLAUDE_WAITER_RECENT_SECONDS`` (#1055 spec 4.3: the waiter can vanish
|
|
7011
|
+
between the refusal and this check); the hint selects the interval only."""
|
|
6354
7012
|
if adm.fresh_requests(_cctally_core.APP_DIR):
|
|
6355
7013
|
adm.wait_until_no_fresh_requests(_cctally_core.APP_DIR)
|
|
7014
|
+
elif adm.claude_waiter_recently_observed(_cctally_core.APP_DIR):
|
|
7015
|
+
adm._sleep(adm.SHORT_POLL_SECONDS)
|
|
6356
7016
|
else:
|
|
6357
7017
|
adm._sleep(adm.POLL_SECONDS)
|
|
6358
7018
|
|
|
@@ -7479,6 +8139,10 @@ _CODEX_W11_EAGER_PREPARE_FOR_TESTS = False
|
|
|
7479
8139
|
_CODEX_W11_TRACE = None
|
|
7480
8140
|
#: Test seam: called immediately before each W11 commit (crash injection).
|
|
7481
8141
|
_CODEX_BATCH_PRECOMMIT_HOOK = None
|
|
8142
|
+
#: Test seam (#982 R1): called inside the locked Codex sync immediately before
|
|
8143
|
+
#: its own store-lock acquisition, i.e. after a heal unit's ``admit()`` has
|
|
8144
|
+
#: returned and before the sync's ``_after_locks`` recheck.
|
|
8145
|
+
_CODEX_PRE_STORE_LOCK_HOOK = None
|
|
7482
8146
|
_CODEX_W11_FILE_SAVEPOINT = "SAVEPOINT codex_w11_file"
|
|
7483
8147
|
_CODEX_W11_FILE_RELEASE = "RELEASE SAVEPOINT codex_w11_file"
|
|
7484
8148
|
_CODEX_W11_FILE_ROLLBACK_TO = "ROLLBACK TO SAVEPOINT codex_w11_file"
|
|
@@ -7646,9 +8310,14 @@ def _write_codex_file_batch(
|
|
|
7646
8310
|
container: "CodexContainerAcceptance | None" = None,
|
|
7647
8311
|
barrier: "CodexBarrierRead | None" = None,
|
|
7648
8312
|
commit: bool = True,
|
|
8313
|
+
thread_guard: "frozenset[str]" = frozenset(),
|
|
7649
8314
|
) -> int:
|
|
7650
8315
|
"""Write one fully-buffered Codex file atomically and return entry changes.
|
|
7651
8316
|
|
|
8317
|
+
``thread_guard`` (#982 spec 4.3) is the fenced heal population, passed by
|
|
8318
|
+
a sync without the heal executor's share: a thread row whose current
|
|
8319
|
+
``source_path`` is a population file is left for the executor's walk.
|
|
8320
|
+
|
|
7652
8321
|
``commit=False`` (#901 W11) leaves the caller's transaction open: an
|
|
7653
8322
|
unbudgeted sync applies each file inside its own savepoint and commits the
|
|
7654
8323
|
batch once.
|
|
@@ -7732,6 +8401,19 @@ def _write_codex_file_batch(
|
|
|
7732
8401
|
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)""",
|
|
7733
8402
|
quota_rows,
|
|
7734
8403
|
)
|
|
8404
|
+
if thread_rows and thread_guard:
|
|
8405
|
+
# The upsert overwrites every metadata column, so a conversation's row
|
|
8406
|
+
# is its last ingested contributor's. While the heal holds its
|
|
8407
|
+
# population, a row a population file owns is the single walk's to
|
|
8408
|
+
# replace (the timeline in which this write came before the walk).
|
|
8409
|
+
kept = []
|
|
8410
|
+
for row in thread_rows:
|
|
8411
|
+
current = conn.execute(
|
|
8412
|
+
"SELECT source_path FROM codex_conversation_threads "
|
|
8413
|
+
"WHERE conversation_key=?", (row[0],)).fetchone()
|
|
8414
|
+
if current is None or current[0] not in thread_guard:
|
|
8415
|
+
kept.append(row)
|
|
8416
|
+
thread_rows = kept
|
|
7735
8417
|
if thread_rows:
|
|
7736
8418
|
conn.executemany(
|
|
7737
8419
|
"""INSERT INTO codex_conversation_threads
|
|
@@ -7929,6 +8611,13 @@ class IngestStats:
|
|
|
7929
8611
|
assistant_lines_skipped: int = 0
|
|
7930
8612
|
skip_reasons: dict = field(default_factory=dict)
|
|
7931
8613
|
full_walk_complete: bool = False
|
|
8614
|
+
# #916: an exhaustive walk that visited every on-disk source cleanly and
|
|
8615
|
+
# whose only defect was orphan rows (tracked transcripts no longer on
|
|
8616
|
+
# disk). It wrote the residual token instead of the walk-complete marker,
|
|
8617
|
+
# so ``full_walk_complete`` stays False; the accounting frontier accepts
|
|
8618
|
+
# either (`_lib_ingest_frontier.provider_sync_certifiable`).
|
|
8619
|
+
full_walk_residual_only: bool = False
|
|
8620
|
+
orphan_residual_count: int = 0
|
|
7932
8621
|
maintenance_failed: bool = False
|
|
7933
8622
|
# #944 spec 5.5 step 5: files whose descriptor no longer matched the
|
|
7934
8623
|
# frozen stat (replaced between stat and open); retried next sync.
|
|
@@ -8172,18 +8861,38 @@ def _targeted_has_pending_global_work(conn) -> bool:
|
|
|
8172
8861
|
return row is not None
|
|
8173
8862
|
|
|
8174
8863
|
|
|
8175
|
-
def _acquire_cache_flock(lock_fh, *, timeout
|
|
8864
|
+
def _acquire_cache_flock(lock_fh, *, timeout, poll: float = 0.2,
|
|
8865
|
+
deadline: "float | None" = None):
|
|
8176
8866
|
"""Acquire the exclusive cache flock on ``lock_fh``.
|
|
8177
8867
|
|
|
8178
|
-
``timeout is None`` -> a single non-blocking attempt (today's
|
|
8179
|
-
for routine auto-syncs): returns False immediately on
|
|
8180
|
-
``timeout > 0`` -> retry ``LOCK_NB`` every
|
|
8181
|
-
|
|
8868
|
+
``timeout is None`` (or ``0``) -> a single non-blocking attempt (today's
|
|
8869
|
+
behavior for routine auto-syncs): returns False immediately on
|
|
8870
|
+
contention. ``timeout > 0`` -> retry ``LOCK_NB`` every ``poll`` seconds
|
|
8871
|
+
until acquired or the deadline elapses; the first attempt always happens.
|
|
8872
|
+
Returns True iff the lock is held on return.
|
|
8873
|
+
|
|
8874
|
+
An explicit absolute ``deadline`` (``time.monotonic()`` seconds; #1055
|
|
8875
|
+
spec 4.5) replaces ``timeout`` and has its own rule: an attempt is made
|
|
8876
|
+
only while ``monotonic() < deadline``, and each sleep is clamped to the
|
|
8877
|
+
time remaining, so an expired deadline makes no attempt at all.
|
|
8182
8878
|
|
|
8183
8879
|
A retry-with-sleep loop, NOT a SIGALRM-based blocking LOCK_EX: the
|
|
8184
8880
|
dashboard runs its sync on a background thread, where Python signals
|
|
8185
8881
|
never fire, so an alarm timeout would silently never trip.
|
|
8186
8882
|
"""
|
|
8883
|
+
if deadline is not None:
|
|
8884
|
+
while True:
|
|
8885
|
+
if deadline - time.monotonic() <= 0:
|
|
8886
|
+
return False
|
|
8887
|
+
try:
|
|
8888
|
+
fcntl.flock(lock_fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
8889
|
+
return True
|
|
8890
|
+
except BlockingIOError:
|
|
8891
|
+
pass
|
|
8892
|
+
remaining = deadline - time.monotonic()
|
|
8893
|
+
if remaining <= 0:
|
|
8894
|
+
return False
|
|
8895
|
+
time.sleep(min(poll, remaining))
|
|
8187
8896
|
deadline = None if timeout is None else (time.monotonic() + timeout)
|
|
8188
8897
|
while True:
|
|
8189
8898
|
try:
|
|
@@ -8192,7 +8901,38 @@ def _acquire_cache_flock(lock_fh, *, timeout):
|
|
|
8192
8901
|
except BlockingIOError:
|
|
8193
8902
|
if deadline is None or time.monotonic() >= deadline:
|
|
8194
8903
|
return False
|
|
8195
|
-
time.sleep(
|
|
8904
|
+
time.sleep(poll)
|
|
8905
|
+
|
|
8906
|
+
|
|
8907
|
+
def _wait_behind_drain_unit(lock_fh, wait_seconds: float, owner: dict) -> str:
|
|
8908
|
+
"""The Claude hook's waiting path after its first refusal (#1055 4.4).
|
|
8909
|
+
|
|
8910
|
+
Waits only when both the Codex provider lock and the drain gate are busy
|
|
8911
|
+
(best-effort evidence of an admitted drain unit), as the one registered
|
|
8912
|
+
Claude waiter, until one absolute deadline. Puts the waiter in
|
|
8913
|
+
``owner["waiter"]`` as soon as it exists, so the caller's cleanup
|
|
8914
|
+
releases it on every path. Returns the ``drain_wait`` outcome; the lock
|
|
8915
|
+
is held iff it is ``acquired``.
|
|
8916
|
+
"""
|
|
8917
|
+
import _lib_codex_drain_admission as adm
|
|
8918
|
+
deadline = time.monotonic() + wait_seconds
|
|
8919
|
+
if not adm.lock_busy(_cctally_core.CACHE_LOCK_CODEX_PATH):
|
|
8920
|
+
return "not_drain" # Claude against Claude, or another writer
|
|
8921
|
+
if not adm.lock_busy(_cctally_core.APP_DIR / adm.GATE_FILE_NAME):
|
|
8922
|
+
return "not_drain" # a Codex hook or an unadmitted Codex sync
|
|
8923
|
+
waiter = adm.ClaudeWaiter.register(
|
|
8924
|
+
_cctally_core.APP_DIR,
|
|
8925
|
+
deadline_ns=adm._monotonic_ns()
|
|
8926
|
+
+ round((deadline - time.monotonic()) * 1e9))
|
|
8927
|
+
if waiter is None:
|
|
8928
|
+
return "busy_waiter"
|
|
8929
|
+
owner["waiter"] = waiter
|
|
8930
|
+
if not _acquire_cache_flock(
|
|
8931
|
+
lock_fh, timeout=None, poll=adm.CLAUDE_HOOK_DRAIN_POLL_SECONDS,
|
|
8932
|
+
deadline=deadline):
|
|
8933
|
+
return "timeout"
|
|
8934
|
+
waiter.mark_holding()
|
|
8935
|
+
return "acquired"
|
|
8196
8936
|
|
|
8197
8937
|
|
|
8198
8938
|
def _prune_orphaned_cache_entries(conn, *, lock_timeout=None):
|
|
@@ -8679,11 +9419,21 @@ def sync_cache(
|
|
|
8679
9419
|
rebuild: bool = False,
|
|
8680
9420
|
only_paths: "set[str] | None" = None,
|
|
8681
9421
|
lock_timeout: "float | None" = None,
|
|
9422
|
+
drain_wait_seconds: "float | None" = None,
|
|
8682
9423
|
) -> IngestStats:
|
|
8683
9424
|
"""Read-through delta ingest. Acquires an exclusive fcntl.flock; if
|
|
8684
9425
|
another process holds it, returns immediately with lock_contended=True
|
|
8685
9426
|
and the caller should proceed with whatever data is already cached.
|
|
8686
9427
|
|
|
9428
|
+
``drain_wait_seconds`` (#1055; only the hook-tick Claude leg passes it,
|
|
9429
|
+
and only on its first invocation) applies when ``lock_timeout`` is None:
|
|
9430
|
+
after the first refusal, a hook that sees an admitted Codex drain unit
|
|
9431
|
+
may wait that long as the one Claude waiter the drain workers yield to.
|
|
9432
|
+
It must not exceed ``max(T, CLAUDE_WAITER_HOLD_ALLOWANCE_SECONDS)``: the
|
|
9433
|
+
drains treat a record ending further out as not fresh, so a larger
|
|
9434
|
+
allowance would poll while the drains never yield. Only the hook leg
|
|
9435
|
+
passes it, with ``CLAUDE_HOOK_DRAIN_WAIT_SECONDS``.
|
|
9436
|
+
|
|
8687
9437
|
When `rebuild=True`, clears the cached rows AFTER acquiring the lock
|
|
8688
9438
|
so a lost race does not wipe a cache another process is actively
|
|
8689
9439
|
populating. If the lock is contended on a rebuild, the cache is left
|
|
@@ -8699,10 +9449,25 @@ def sync_cache(
|
|
|
8699
9449
|
# verdict. Settled by the clean end-of-walk write; every other exit
|
|
8700
9450
|
# withdraws a present marker in ``finally``.
|
|
8701
9451
|
walk_marker_owed = False
|
|
9452
|
+
# #1055: the Claude waiter, once registered; released after the writer
|
|
9453
|
+
# unlock on every exit path (the outermost ``finally``).
|
|
9454
|
+
drain_waiter: dict = {}
|
|
8702
9455
|
try:
|
|
8703
9456
|
with _perf.phase("flock") as _p_flock:
|
|
8704
9457
|
_acquired = _acquire_cache_flock(lock_fh, timeout=lock_timeout)
|
|
8705
|
-
|
|
9458
|
+
_drain_wait, _waited = "none", 0.0
|
|
9459
|
+
if (not _acquired and lock_timeout is None
|
|
9460
|
+
and drain_wait_seconds is not None
|
|
9461
|
+
and drain_wait_seconds > 0):
|
|
9462
|
+
_refused_at = time.monotonic()
|
|
9463
|
+
try:
|
|
9464
|
+
_drain_wait = _wait_behind_drain_unit(
|
|
9465
|
+
lock_fh, drain_wait_seconds, drain_waiter)
|
|
9466
|
+
finally:
|
|
9467
|
+
_waited = time.monotonic() - _refused_at
|
|
9468
|
+
_acquired = _drain_wait == "acquired"
|
|
9469
|
+
_p_flock.set_meta(contended=not _acquired, drain_wait=_drain_wait,
|
|
9470
|
+
drain_wait_seconds=round(_waited, 3))
|
|
8706
9471
|
if not _acquired:
|
|
8707
9472
|
eprint("[cache] sync already in progress; using existing cache")
|
|
8708
9473
|
stats.lock_contended = True
|
|
@@ -8895,8 +9660,9 @@ def sync_cache(
|
|
|
8895
9660
|
# Clear the walk-complete sentinel atomically with the wipe
|
|
8896
9661
|
# (cctally-dev#93, D5/D2): a stale "complete" marker must never
|
|
8897
9662
|
# survive a destructive rebuild. The end-of-loop write below
|
|
8898
|
-
# re-establishes it only after this rebuild's clean walk.
|
|
8899
|
-
|
|
9663
|
+
# re-establishes it only after this rebuild's clean walk. #916:
|
|
9664
|
+
# the residual token goes in the same statement.
|
|
9665
|
+
_clear_claude_walk_verdicts(conn)
|
|
8900
9666
|
# Issue #139: a rebuild walks every file from offset 0, so the
|
|
8901
9667
|
# per-file fused walk below repopulates the whole message
|
|
8902
9668
|
# index — that satisfies any deferred existing-install backfill.
|
|
@@ -9219,12 +9985,24 @@ def sync_cache(
|
|
|
9219
9985
|
# one exists, so a repeated orphaned sync doesn't churn a no-op write
|
|
9220
9986
|
# txn every tick); merely withholding THIS run's end-of-loop rewrite
|
|
9221
9987
|
# is not enough, since a stale marker from a previous sync would
|
|
9222
|
-
# otherwise survive and keep vouching for completeness.
|
|
9223
|
-
#
|
|
9224
|
-
# the marker
|
|
9225
|
-
# upgrade gate DEFERs the 008/009/010 recomputes
|
|
9226
|
-
# certifying aggregates that still include data from
|
|
9227
|
-
# on disk).
|
|
9988
|
+
# otherwise survive and keep vouching for completeness. The orphan
|
|
9989
|
+
# set is kept in ``residual_orphans`` and the end-of-loop block
|
|
9990
|
+
# withholds the marker for this run whenever it is non-empty. With the
|
|
9991
|
+
# marker gone the upgrade gate DEFERs the 008/009/010 recomputes
|
|
9992
|
+
# (rather than certifying aggregates that still include data from
|
|
9993
|
+
# files no longer on disk).
|
|
9994
|
+
#
|
|
9995
|
+
# #916: orphans no longer set walk_clean=False. walk_clean describes
|
|
9996
|
+
# the on-disk sources only, so a walk whose ONLY defect is orphan rows
|
|
9997
|
+
# ends "residual-only" and writes the
|
|
9998
|
+
# `claude_ingest_walk_orphan_residual` token instead of the marker.
|
|
9999
|
+
# The token is the accounting frontier's completeness signal and
|
|
10000
|
+
# nobody else's: the walk never reads or writes the orphan rows, so
|
|
10001
|
+
# every on-disk source's bytes are still reflected in the cache, which
|
|
10002
|
+
# is all the frontier certifies. Without it a residual the prune gates
|
|
10003
|
+
# refuse kept the dashboard re-walking every transcript on every tick
|
|
10004
|
+
# until a rebuild. The marker keeps its D5a meaning unchanged.
|
|
10005
|
+
# The safe CLEANUP lives OUT of sync_cache, in
|
|
9228
10006
|
# _prune_orphaned_cache_entries — it re-derives the safe orphan set
|
|
9229
10007
|
# independently and removes the full derived surface under three
|
|
9230
10008
|
# gates: A (session-id not shared by a survivor), B (coverage — every
|
|
@@ -9251,6 +10029,7 @@ def sync_cache(
|
|
|
9251
10029
|
# file for an orphan and nuke the walk-complete marker. Skip it entirely
|
|
9252
10030
|
# for targeted: the live-tail fast path never prunes orphans (the full
|
|
9253
10031
|
# background sync owns that).
|
|
10032
|
+
residual_orphans: "tuple[str, ...]" = ()
|
|
9254
10033
|
if not targeted:
|
|
9255
10034
|
global _LAST_WARNED_ORPHAN_SET
|
|
9256
10035
|
on_disk_paths = {str(jp) for jp in paths}
|
|
@@ -9283,7 +10062,10 @@ def sync_cache(
|
|
|
9283
10062
|
"DELETE FROM cache_meta WHERE key='claude_ingest_walk_complete'"
|
|
9284
10063
|
)
|
|
9285
10064
|
conn.commit()
|
|
9286
|
-
|
|
10065
|
+
# Orphaned rows -> the cache doesn't mirror disk (D5a): the
|
|
10066
|
+
# end-of-loop block withholds the marker (#916: and writes the
|
|
10067
|
+
# residual token when the on-disk walk was clean).
|
|
10068
|
+
residual_orphans = tuple(sorted(orphaned_tracked_paths))
|
|
9287
10069
|
else:
|
|
9288
10070
|
# No orphans this walk: clear the throttle memory so a LATER,
|
|
9289
10071
|
# distinct orphan episode (even one recreated at the same paths)
|
|
@@ -9363,7 +10145,8 @@ def sync_cache(
|
|
|
9363
10145
|
# full-reset (cctally-dev#93, D5/D2): the cache is being wiped, so
|
|
9364
10146
|
# any "complete" marker is now stale. The end-of-loop write below
|
|
9365
10147
|
# re-establishes it only after this run's clean re-ingest walk.
|
|
9366
|
-
|
|
10148
|
+
# #916: the residual token goes in the same statement.
|
|
10149
|
+
_clear_claude_walk_verdicts(conn)
|
|
9367
10150
|
# Crash-safety: also clear session_files's size/offset tracking
|
|
9368
10151
|
# so a partial-state recovery on the NEXT sync forces every
|
|
9369
10152
|
# file's per-file branch to take the fresh-ingest path. Without
|
|
@@ -9413,15 +10196,19 @@ def sync_cache(
|
|
|
9413
10196
|
# "walk" phase (never per-row — Section 2 rule: volume is a count, not
|
|
9414
10197
|
# N timed phases). Opened via the context-manager protocol so the hot
|
|
9415
10198
|
# loop body below is not reindented; counts recorded after the loop.
|
|
9416
|
-
# #195: is the cache-write-split re-walk armed?
|
|
9417
|
-
# sync_cache call, before the file loop
|
|
9418
|
-
#
|
|
9419
|
-
#
|
|
9420
|
-
#
|
|
9421
|
-
#
|
|
9422
|
-
# IntegrityError and rolling back its whole
|
|
9423
|
-
#
|
|
9424
|
-
#
|
|
10199
|
+
# #195: is the cache-write-split re-walk armed? Read ONCE per
|
|
10200
|
+
# sync_cache call, before the file loop. Cache migration 030 sets this
|
|
10201
|
+
# flag and resets the per-file cursors to -1; the end-of-walk block
|
|
10202
|
+
# below clears it after a non-targeted full walk that is clean on disk,
|
|
10203
|
+
# either fully clean or residual-only (#916). While armed, every file
|
|
10204
|
+
# uses the chained-conflict variant so a replayed NULL-key row updates
|
|
10205
|
+
# in place instead of raising IntegrityError and rolling back its whole
|
|
10206
|
+
# file. #916: the statement is chosen PER FILE (``use_rewalk`` in the
|
|
10207
|
+
# loop): a file whose own cursor still reads -1 takes the chained
|
|
10208
|
+
# variant even after the flag retired. Scoped to the flag and to -1
|
|
10209
|
+
# cursors (not marker-absence) so migration 020's loud
|
|
10210
|
+
# duplicate-physical-key backstop stays intact on every other ingest
|
|
10211
|
+
# path.
|
|
9425
10212
|
rewalk_armed = conn.execute(
|
|
9426
10213
|
"SELECT 1 FROM cache_meta WHERE key=?",
|
|
9427
10214
|
(_cctally_db_sib.CACHE_CREATION_SPLIT_REWALK_KEY,),
|
|
@@ -9663,13 +10450,24 @@ def sync_cache(
|
|
|
9663
10450
|
# collides on idx_entries_physical and would roll back the
|
|
9664
10451
|
# whole per-file transaction. Steady state keeps the
|
|
9665
10452
|
# single-target SQL and its LOUD physical-key backstop.
|
|
10453
|
+
#
|
|
10454
|
+
# #916: the obligation is also carried per file. Migration
|
|
10455
|
+
# 030 is the only producer of a negative ``size_bytes``,
|
|
10456
|
+
# so a ``-1`` cursor is a file not yet re-walked since the
|
|
10457
|
+
# reset, still owed the chained clause even after a
|
|
10458
|
+
# residual-only walk retired the global flag (an orphan
|
|
10459
|
+
# whose transcript reappears). Read from THIS file's own
|
|
10460
|
+
# ``prev`` row: the ``prev_size`` local is unbound for a
|
|
10461
|
+
# new file and stale for a file after a ``-1`` one.
|
|
10462
|
+
use_rewalk = rewalk_armed or (
|
|
10463
|
+
prev is not None and prev[0] < 0)
|
|
9666
10464
|
if attribution_binds is not None:
|
|
9667
10465
|
_sql = (SESSION_ENTRY_UPSERT_SQL_REWALK_LINKED
|
|
9668
|
-
if
|
|
10466
|
+
if use_rewalk
|
|
9669
10467
|
else SESSION_ENTRY_UPSERT_SQL_LINKED)
|
|
9670
10468
|
else:
|
|
9671
10469
|
_sql = (SESSION_ENTRY_UPSERT_SQL_REWALK
|
|
9672
|
-
if
|
|
10470
|
+
if use_rewalk else SESSION_ENTRY_UPSERT_SQL)
|
|
9673
10471
|
conn.executemany(_sql, stamped_rows)
|
|
9674
10472
|
stats.rows_changed += conn.total_changes - before
|
|
9675
10473
|
# Conversation message ingest (Plan 1). Lands in the SAME
|
|
@@ -9851,7 +10649,12 @@ def sync_cache(
|
|
|
9851
10649
|
# writes nothing after the first; the value is therefore the instant
|
|
9852
10650
|
# the current run of complete walks began. Any exit that does not reach
|
|
9853
10651
|
# here withdraws the marker (``walk_marker_owed`` in ``finally``).
|
|
9854
|
-
|
|
10652
|
+
#
|
|
10653
|
+
# #916: a walk that is clean on disk but still tracks orphans
|
|
10654
|
+
# (``residual_orphans``) writes the residual token instead. The two
|
|
10655
|
+
# are mutually exclusive: the marker branch deletes the token, and the
|
|
10656
|
+
# orphan leg already deleted the marker before a token is written.
|
|
10657
|
+
if walk_clean and applied_at_start and not targeted and not residual_orphans:
|
|
9855
10658
|
conn.execute(
|
|
9856
10659
|
"INSERT INTO cache_meta(key, value) "
|
|
9857
10660
|
"VALUES('claude_ingest_walk_complete', ?) "
|
|
@@ -9866,9 +10669,53 @@ def sync_cache(
|
|
|
9866
10669
|
"DELETE FROM cache_meta WHERE key=?",
|
|
9867
10670
|
(_cctally_db_sib.CACHE_CREATION_SPLIT_REWALK_KEY,),
|
|
9868
10671
|
)
|
|
10672
|
+
# #916: the residual cleared (pruned, rebuilt or restored); the
|
|
10673
|
+
# marker now vouches, so the token goes in the same transaction.
|
|
10674
|
+
conn.execute(
|
|
10675
|
+
"DELETE FROM cache_meta WHERE key=?",
|
|
10676
|
+
(_ingest_frontier.CLAUDE_WALK_ORPHAN_RESIDUAL_KEY,),
|
|
10677
|
+
)
|
|
9869
10678
|
conn.commit()
|
|
9870
10679
|
stats.full_walk_complete = True
|
|
9871
10680
|
walk_marker_owed = False
|
|
10681
|
+
elif walk_clean and applied_at_start and not targeted:
|
|
10682
|
+
# #916: residual-only. Every on-disk source was visited cleanly,
|
|
10683
|
+
# so the split re-walk flag retires here too: the only files still
|
|
10684
|
+
# owed their re-walk are absent orphans, and each one's own ``-1``
|
|
10685
|
+
# cursor re-selects the chained clause if it reappears. The marker
|
|
10686
|
+
# stays absent (D5a), and ``full_walk_complete`` stays False.
|
|
10687
|
+
conn.execute(
|
|
10688
|
+
"DELETE FROM cache_meta WHERE key=?",
|
|
10689
|
+
(_cctally_db_sib.CACHE_CREATION_SPLIT_REWALK_KEY,),
|
|
10690
|
+
)
|
|
10691
|
+
# #901 W8: an unchanged orphan set writes nothing. ``at`` is
|
|
10692
|
+
# excluded from the comparison, so it records when the current
|
|
10693
|
+
# set was first seen; the value is debug-only and no reader acts
|
|
10694
|
+
# on it.
|
|
10695
|
+
token = {
|
|
10696
|
+
"orphans": len(residual_orphans),
|
|
10697
|
+
"fingerprint": hashlib.sha256(
|
|
10698
|
+
"\n".join(residual_orphans).encode("utf-8")).hexdigest(),
|
|
10699
|
+
}
|
|
10700
|
+
row = conn.execute(
|
|
10701
|
+
"SELECT value FROM cache_meta WHERE key=?",
|
|
10702
|
+
(_ingest_frontier.CLAUDE_WALK_ORPHAN_RESIDUAL_KEY,),
|
|
10703
|
+
).fetchone()
|
|
10704
|
+
try:
|
|
10705
|
+
held = json.loads(row[0]) if row is not None else None
|
|
10706
|
+
except (TypeError, ValueError):
|
|
10707
|
+
held = None
|
|
10708
|
+
if not (isinstance(held, dict)
|
|
10709
|
+
and held.get("orphans") == token["orphans"]
|
|
10710
|
+
and held.get("fingerprint") == token["fingerprint"]):
|
|
10711
|
+
token["at"] = dt.datetime.now(dt.timezone.utc).isoformat()
|
|
10712
|
+
_set_cache_meta(
|
|
10713
|
+
conn, _ingest_frontier.CLAUDE_WALK_ORPHAN_RESIDUAL_KEY,
|
|
10714
|
+
json.dumps(token))
|
|
10715
|
+
conn.commit()
|
|
10716
|
+
stats.full_walk_residual_only = True
|
|
10717
|
+
stats.orphan_residual_count = len(residual_orphans)
|
|
10718
|
+
walk_marker_owed = False
|
|
9872
10719
|
# #279 S2 F1: rolling parse-health record. Anomaly-delta-gated so
|
|
9873
10720
|
# steady-state (incl. targeted live-tail) syncs stay zero-write;
|
|
9874
10721
|
# targeted syncs still accumulate — they ingest real new bytes.
|
|
@@ -9892,15 +10739,29 @@ def sync_cache(
|
|
|
9892
10739
|
# stalling the lock under heavy-reader contention.
|
|
9893
10740
|
_maybe_truncate_wal(conn, _cctally_core.CACHE_DB_PATH)
|
|
9894
10741
|
finally:
|
|
9895
|
-
|
|
9896
|
-
|
|
9897
|
-
# #901 §5.3c (W9): the end of a cache sync is a checkpoint boundary.
|
|
9898
|
-
_wal_checkpoint.at_boundary(conn)
|
|
10742
|
+
# #1055 spec 4.5: nested so a cleanup failure still unlocks the
|
|
10743
|
+
# writer, and the Claude waiter is released only after that unlock.
|
|
9899
10744
|
try:
|
|
9900
|
-
|
|
9901
|
-
|
|
9902
|
-
|
|
9903
|
-
|
|
10745
|
+
try:
|
|
10746
|
+
if walk_marker_owed:
|
|
10747
|
+
_withdraw_cache_meta_marker(
|
|
10748
|
+
conn, "claude_ingest_walk_complete")
|
|
10749
|
+
# #916: an unsettled walk cannot vouch for residual-only
|
|
10750
|
+
# either.
|
|
10751
|
+
_withdraw_cache_meta_marker(
|
|
10752
|
+
conn, _ingest_frontier.CLAUDE_WALK_ORPHAN_RESIDUAL_KEY)
|
|
10753
|
+
# #901 §5.3c (W9): the end of a cache sync is a checkpoint
|
|
10754
|
+
# boundary.
|
|
10755
|
+
_wal_checkpoint.at_boundary(conn)
|
|
10756
|
+
finally:
|
|
10757
|
+
try:
|
|
10758
|
+
fcntl.flock(lock_fh, fcntl.LOCK_UN)
|
|
10759
|
+
except OSError:
|
|
10760
|
+
pass
|
|
10761
|
+
lock_fh.close()
|
|
10762
|
+
finally:
|
|
10763
|
+
if drain_waiter.get("waiter") is not None:
|
|
10764
|
+
drain_waiter["waiter"].release()
|
|
9904
10765
|
return stats
|
|
9905
10766
|
|
|
9906
10767
|
|
|
@@ -11301,7 +12162,9 @@ def _record_codex_container_status(conn, stats, *, census_complete: bool,
|
|
|
11301
12162
|
prior = decoded if isinstance(decoded, dict) else {}
|
|
11302
12163
|
except (TypeError, ValueError):
|
|
11303
12164
|
prior = {}
|
|
11304
|
-
|
|
12165
|
+
# #1052 spec 5.3: every skip the pass saw, its unit phase's included.
|
|
12166
|
+
seen_skipped = stats.all_container_skipped()
|
|
12167
|
+
observed = {k: int(seen_skipped.get(k, 0))
|
|
11305
12168
|
for k in _CONTAINER_SKIP_KEYS}
|
|
11306
12169
|
# D3: verification debt is pending work only. A terminal `unverified`
|
|
11307
12170
|
# container is reported (and warned about) on its own and never feeds V
|
|
@@ -11356,7 +12219,7 @@ def _record_codex_container_status(conn, stats, *, census_complete: bool,
|
|
|
11356
12219
|
# ingestion observation does not pretend that an idle walk probed Python.
|
|
11357
12220
|
new_capability_observation = bool(
|
|
11358
12221
|
stats.files_processed or stats.container_pending
|
|
11359
|
-
or any(
|
|
12222
|
+
or any(seen_skipped.values()) or stats.rebuild_refused)
|
|
11360
12223
|
decoder = prior.get("decoder")
|
|
11361
12224
|
decoder_observed_at = prior.get("decoderObservedAt", prior.get("observedAt"))
|
|
11362
12225
|
if new_capability_observation or decoder not in (
|
|
@@ -11892,9 +12755,57 @@ class CodexIngestStats:
|
|
|
11892
12755
|
barrier_deferred_targets: dict = field(default_factory=dict)
|
|
11893
12756
|
barrier_published_paths: set = field(default_factory=set)
|
|
11894
12757
|
barrier_retry_causes: set = field(default_factory=set)
|
|
12758
|
+
# #1052 spec 5.4: the conversation pass's terminal deferrals. A unit whose
|
|
12759
|
+
# contributor carries the conversations store's terminal `unverified`
|
|
12760
|
+
# verdict, and none of whose files is blocked, publishes nothing and is
|
|
12761
|
+
# not retried. ``barrier_keys_terminal`` counts its keys (they stay in
|
|
12762
|
+
# ``barrier_keys_deferred`` too) and ``barrier_terminal_keys`` maps each
|
|
12763
|
+
# terminal file to the keys it holds back (a residual field the
|
|
12764
|
+
# transcript worker does not serialize). ``barrier_container_skipped``
|
|
12765
|
+
# counts, by reason, the unit-phase skips of paths phase 1 did not skip:
|
|
12766
|
+
# ``container_skipped`` is phase 1's census only.
|
|
12767
|
+
barrier_keys_terminal: int = 0
|
|
12768
|
+
barrier_terminal_keys: dict = field(default_factory=dict)
|
|
12769
|
+
barrier_container_skipped: dict = field(default_factory=dict)
|
|
11895
12770
|
# The status writer already observes capability for unresolved work.
|
|
11896
12771
|
# Explicit adapters reuse it; unchanged accepted generations never probe.
|
|
11897
12772
|
container_decoder: "str | None" = None
|
|
12773
|
+
# #982 spec 4.1 steps 4 and 7: a heal walk piece's own report. The files
|
|
12774
|
+
# it ingested (or reset), whether its ``_ingest_limit`` stopped it before
|
|
12775
|
+
# the next such file, the census files it examined successfully
|
|
12776
|
+
# (ingested, reset or found unchanged; never a failed, torn-deferred,
|
|
12777
|
+
# held or U6-skipped file), its discovery set (the census when it is a
|
|
12778
|
+
# walk's first piece), whether it left no census file unwalked, and
|
|
12779
|
+
# whether it made the transition (deleted the heal record) in its
|
|
12780
|
+
# end-of-walk transaction.
|
|
12781
|
+
heal_counted_paths: set = field(default_factory=set)
|
|
12782
|
+
heal_limit_stopped: bool = False
|
|
12783
|
+
heal_examined: "frozenset[str]" = frozenset()
|
|
12784
|
+
heal_discovered: "frozenset[str] | None" = None
|
|
12785
|
+
heal_walk_complete: bool = False
|
|
12786
|
+
heal_transitioned: bool = False
|
|
12787
|
+
# Revision 7: the committed cursor row of every census file this piece
|
|
12788
|
+
# visited (read after its last commit, under its locks), and the visited
|
|
12789
|
+
# files of earlier pieces whose rows changed before the walk's last piece.
|
|
12790
|
+
heal_visit_rows: dict = field(default_factory=dict)
|
|
12791
|
+
heal_visit_changed: "frozenset[str]" = frozenset()
|
|
12792
|
+
# #982 spec 4.4: a budgeted call declined `replay_pending` while a replay
|
|
12793
|
+
# heal record exists (the store is cleared and partly refilled), so the
|
|
12794
|
+
# hook must not run its projection or budget leg over it. A budgeted call
|
|
12795
|
+
# that met contention sets it from an unlocked read of the same record
|
|
12796
|
+
# (`_codex_replay_refill_unlocked`), and its hook publishes no request.
|
|
12797
|
+
replay_refill: bool = False
|
|
12798
|
+
# #982 spec 4.5a (Generation): an executor unit whose pass generation is
|
|
12799
|
+
# not the heal record's (or that found a fresh clear only a new pass's
|
|
12800
|
+
# start unit may perform) wrote nothing; the executor ends its pass
|
|
12801
|
+
# `restart`.
|
|
12802
|
+
heal_generation_stale: bool = False
|
|
12803
|
+
# #972 §5.4: conversations whose rollup the full pass's reconcile
|
|
12804
|
+
# rewrote (thread-derived fields drifted from the current answer).
|
|
12805
|
+
rollups_reattributed: int = 0
|
|
12806
|
+
# #972 §5.8 rule 4: rollups the reconcile skipped because deriving their
|
|
12807
|
+
# expected fields raised a non-database error (an unresolvable cwd).
|
|
12808
|
+
rollups_reconcile_skipped: int = 0
|
|
11898
12809
|
|
|
11899
12810
|
@property
|
|
11900
12811
|
def targeted_clean(self) -> bool:
|
|
@@ -11906,7 +12817,29 @@ class CodexIngestStats:
|
|
|
11906
12817
|
and self.files_failed == 0
|
|
11907
12818
|
and not self.container_pending
|
|
11908
12819
|
and not self.barrier_retry
|
|
11909
|
-
and not any(self.
|
|
12820
|
+
and not any(self.all_container_skipped().values()))
|
|
12821
|
+
|
|
12822
|
+
def all_container_skipped(self) -> "dict[str, int]":
|
|
12823
|
+
"""Every container skip this pass saw, by reason (#1052 spec 5.3):
|
|
12824
|
+
phase 1's census plus the unit phase's skips of other paths. The two
|
|
12825
|
+
are path-disjoint by construction, so the sum counts each path once.
|
|
12826
|
+
"""
|
|
12827
|
+
merged = dict(self.container_skipped)
|
|
12828
|
+
for reason, count in self.barrier_container_skipped.items():
|
|
12829
|
+
merged[reason] = merged.get(reason, 0) + count
|
|
12830
|
+
return merged
|
|
12831
|
+
|
|
12832
|
+
|
|
12833
|
+
def _print_codex_done_line(stats: CodexIngestStats) -> None:
|
|
12834
|
+
"""`cache-sync`'s Codex summary line (the first leg's and the heal's)."""
|
|
12835
|
+
eprint(
|
|
12836
|
+
f"[cache-sync] codex done: {stats.files_processed} processed, "
|
|
12837
|
+
f"{stats.files_skipped_unchanged} skipped, "
|
|
12838
|
+
f"{stats.files_reset_truncated} reset, "
|
|
12839
|
+
f"{stats.rows_changed} rows changed, "
|
|
12840
|
+
f"{stats.lines_malformed} malformed, "
|
|
12841
|
+
f"{stats.token_events_skipped} drift-skipped"
|
|
12842
|
+
)
|
|
11910
12843
|
|
|
11911
12844
|
|
|
11912
12845
|
def _progress_codex_stderr(stats: CodexIngestStats, *, force: bool = False) -> None:
|
|
@@ -12113,7 +13046,7 @@ def _record_codex_replay_deferral(conn: sqlite3.Connection) -> None:
|
|
|
12113
13046
|
raise
|
|
12114
13047
|
|
|
12115
13048
|
|
|
12116
|
-
def _defer_codex_replay_drain() -> str:
|
|
13049
|
+
def _defer_codex_replay_drain(*, short: bool = False) -> str:
|
|
12117
13050
|
"""Hand the declined byte-zero replay to a detached worker (public #5).
|
|
12118
13051
|
|
|
12119
13052
|
Returns ``"spawned"``, ``"throttled"`` or ``"failed"`` — for the lifecycle
|
|
@@ -12128,6 +13061,10 @@ def _defer_codex_replay_drain() -> str:
|
|
|
12128
13061
|
re-read everything", and slicing that corrupts account attribution — so
|
|
12129
13062
|
there is nothing to fall back to. The record written at the decline site is
|
|
12130
13063
|
what makes a drain that never succeeds diagnosable.
|
|
13064
|
+
|
|
13065
|
+
``short`` (#982 spec 4.5a): a heal pass that ended ``restart`` asks for
|
|
13066
|
+
the short retry throttle even when no heal record exists (a replay
|
|
13067
|
+
re-armed after the pass's transition leaves only the marker).
|
|
12131
13068
|
"""
|
|
12132
13069
|
marker = _cctally_core.APP_DIR / CODEX_REPLAY_DRAIN_MARKER_NAME
|
|
12133
13070
|
try:
|
|
@@ -12135,6 +13072,20 @@ def _defer_codex_replay_drain() -> str:
|
|
|
12135
13072
|
except OSError:
|
|
12136
13073
|
age = None
|
|
12137
13074
|
if age is not None and 0 <= age < CODEX_REPLAY_DRAIN_SPAWN_THROTTLE_SECONDS:
|
|
13075
|
+
# #982 spec 4.5a: every path that requests this worker comes through
|
|
13076
|
+
# here, so the heal record's short retry throttle is applied once,
|
|
13077
|
+
# for all of them. Probed only inside the window it can change.
|
|
13078
|
+
if (age < CODEX_HEAL_RETRY_THROTTLE_SECONDS
|
|
13079
|
+
or not (short or _codex_heal_record_present())):
|
|
13080
|
+
return "throttled"
|
|
13081
|
+
if _codex_heal_walk_running():
|
|
13082
|
+
# A heal pass is running: hooks request the worker on every tick
|
|
13083
|
+
# while a record exists, and a worker spawned now would only run the
|
|
13084
|
+
# provenance, verification, reset, membership and seal drains beside
|
|
13085
|
+
# the executor. The pass requests the worker itself, after releasing
|
|
13086
|
+
# this lock, when it ends `contended` or `degraded`; on `restart` its
|
|
13087
|
+
# callers re-run it inline. The marker is not stamped, so the first
|
|
13088
|
+
# request after the pass is not throttled by this one.
|
|
12138
13089
|
return "throttled"
|
|
12139
13090
|
try:
|
|
12140
13091
|
_cctally_core.APP_DIR.mkdir(parents=True, exist_ok=True)
|
|
@@ -12145,6 +13096,51 @@ def _defer_codex_replay_drain() -> str:
|
|
|
12145
13096
|
else "failed")
|
|
12146
13097
|
|
|
12147
13098
|
|
|
13099
|
+
def _codex_heal_walk_running() -> bool:
|
|
13100
|
+
"""Whether a heal pass holds ``codex-heal-walk.lock``: a non-blocking
|
|
13101
|
+
exclusive probe on a separate open file description, released at once.
|
|
13102
|
+
Never creates the file; any other error answers False (spawn as before).
|
|
13103
|
+
"""
|
|
13104
|
+
path = _cctally_core.APP_DIR / CODEX_HEAL_WALK_LOCK_NAME
|
|
13105
|
+
try:
|
|
13106
|
+
fd = os.open(path, os.O_RDWR)
|
|
13107
|
+
except OSError:
|
|
13108
|
+
return False
|
|
13109
|
+
try:
|
|
13110
|
+
try:
|
|
13111
|
+
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
13112
|
+
except BlockingIOError:
|
|
13113
|
+
return True
|
|
13114
|
+
except OSError:
|
|
13115
|
+
return False
|
|
13116
|
+
fcntl.flock(fd, fcntl.LOCK_UN)
|
|
13117
|
+
return False
|
|
13118
|
+
finally:
|
|
13119
|
+
os.close(fd)
|
|
13120
|
+
|
|
13121
|
+
|
|
13122
|
+
def _codex_heal_record_present() -> bool:
|
|
13123
|
+
"""Whether cache.db holds a well-formed heal record, read through a raw
|
|
13124
|
+
``mode=rw`` connection (no schema apply, migration or store lock; never
|
|
13125
|
+
creates the file). Anything unreadable answers False, which keeps the
|
|
13126
|
+
hourly throttle."""
|
|
13127
|
+
path = pathlib.Path(_cctally_core.CACHE_DB_PATH)
|
|
13128
|
+
try:
|
|
13129
|
+
if not path.is_file():
|
|
13130
|
+
return False
|
|
13131
|
+
conn = sqlite3.connect(f"{path.resolve().as_uri()}?mode=rw",
|
|
13132
|
+
uri=True, timeout=1.0)
|
|
13133
|
+
except (OSError, sqlite3.Error):
|
|
13134
|
+
return False
|
|
13135
|
+
try:
|
|
13136
|
+
record = load_codex_heal_record(conn)
|
|
13137
|
+
except sqlite3.Error:
|
|
13138
|
+
return False
|
|
13139
|
+
finally:
|
|
13140
|
+
conn.close()
|
|
13141
|
+
return record is not None and not record.get("malformed")
|
|
13142
|
+
|
|
13143
|
+
|
|
12148
13144
|
class ResetDrainResult(NamedTuple):
|
|
12149
13145
|
outcome: str
|
|
12150
13146
|
reset: int
|
|
@@ -12338,14 +13334,33 @@ def _cmd_codex_replay_drain_body(args) -> int:
|
|
|
12338
13334
|
conn = open_cache_db()
|
|
12339
13335
|
try:
|
|
12340
13336
|
stats = CodexIngestStats()
|
|
12341
|
-
|
|
12342
|
-
|
|
12343
|
-
|
|
12344
|
-
|
|
12345
|
-
conn, maintenance="caller",
|
|
12346
|
-
lock_timeout=CODEX_REPLAY_DRAIN_LOCK_TIMEOUT_SECONDS)
|
|
13337
|
+
replay_armed = conn.execute(
|
|
13338
|
+
"SELECT 1 FROM cache_meta WHERE key=? LIMIT 1",
|
|
13339
|
+
(CODEX_REPLAY_FROM_ZERO_KEY,)).fetchone() is not None
|
|
13340
|
+
heal_record = load_codex_heal_record(conn)
|
|
12347
13341
|
finally:
|
|
12348
13342
|
conn.close()
|
|
13343
|
+
# #982 spec 4.6: the byte-zero replay and the history heal that
|
|
13344
|
+
# follows a settled census both run as admitted per-file units, never
|
|
13345
|
+
# as one locked walk; a record left by a dead executor resumes from
|
|
13346
|
+
# its phase. The executor picks replay mode itself when the marker or
|
|
13347
|
+
# a replay record exists.
|
|
13348
|
+
heal = HealResult("none", None, 0, 0, 0, 0, 0)
|
|
13349
|
+
if (replay_armed or heal_record is not None
|
|
13350
|
+
or (history.outcome == "settled" and history.candidates)):
|
|
13351
|
+
heal = run_codex_admitted_heal(
|
|
13352
|
+
wait_for_requests=True,
|
|
13353
|
+
population=(None if heal_record is not None or replay_armed
|
|
13354
|
+
else _codex_history_candidates()))
|
|
13355
|
+
if heal.outcome in ("replay_due", "restart"):
|
|
13356
|
+
# Spec 4.5a: a replay armed during the history heal ended
|
|
13357
|
+
# that pass before its fenced walk; this pass replays.
|
|
13358
|
+
heal = run_codex_admitted_heal(wait_for_requests=True)
|
|
13359
|
+
if heal.outcome == "contended":
|
|
13360
|
+
stats = CodexIngestStats(lock_contended=True)
|
|
13361
|
+
elif heal.stats is not None:
|
|
13362
|
+
# Either mode: the heal's own counts, summed over its units.
|
|
13363
|
+
stats = heal.stats
|
|
12349
13364
|
if stats.rebuild_refused:
|
|
12350
13365
|
_log("refused", f"rebuild_refused={stats.rebuild_refused}")
|
|
12351
13366
|
return 0
|
|
@@ -12379,7 +13394,9 @@ def _cmd_codex_replay_drain_body(args) -> int:
|
|
|
12379
13394
|
f"reset_done={reset_result.reset} reset_unchanged={reset_result.unchanged} "
|
|
12380
13395
|
f"barrier_proven={scan.proven} barrier_sealed={seal.sealed} "
|
|
12381
13396
|
f"barrier_remaining={seal.remaining} history={history.outcome} "
|
|
12382
|
-
f"history_candidates={history.candidates} history_traced={history.traced}"
|
|
13397
|
+
f"history_candidates={history.candidates} history_traced={history.traced} "
|
|
13398
|
+
f"heal={heal.outcome} heal_units={heal.units} "
|
|
13399
|
+
f"heal_degraded={heal.degraded}")
|
|
12383
13400
|
return 0
|
|
12384
13401
|
|
|
12385
13402
|
|
|
@@ -12483,6 +13500,15 @@ def _sync_codex_cache_locked(
|
|
|
12483
13500
|
_on_first_file_rollback: Callable[[], None] | None = None,
|
|
12484
13501
|
_on_file_committed: Callable[[str], None] | None = None,
|
|
12485
13502
|
_on_file_prepared: Callable[[str], None] | None = None,
|
|
13503
|
+
_walk_share: "_CodexWalkShare | None" = None,
|
|
13504
|
+
_ingest_limit: "int | None" = None,
|
|
13505
|
+
_heal_census: "frozenset[str] | None" = None,
|
|
13506
|
+
_heal_visited: "frozenset[str]" = frozenset(),
|
|
13507
|
+
_heal_visited_rows: "dict[str, tuple | None] | None" = None,
|
|
13508
|
+
_defer_finalization: bool = False,
|
|
13509
|
+
_heal_transition: bool = False,
|
|
13510
|
+
_replay_stage: "str | None" = None,
|
|
13511
|
+
_heal_generation: "str | None" = None,
|
|
12486
13512
|
) -> CodexIngestStats:
|
|
12487
13513
|
"""Read-through delta ingest of ~/.codex/sessions/**/*.jsonl.
|
|
12488
13514
|
|
|
@@ -12520,6 +13546,66 @@ def _sync_codex_cache_locked(
|
|
|
12520
13546
|
duplicated cost — measured at roughly half of every growing turn. Every
|
|
12521
13547
|
other caller keeps ``"auto"``, because the sync-internal reconcile is what
|
|
12522
13548
|
keeps the projection current for the dashboard and ``cache-sync``.
|
|
13549
|
+
|
|
13550
|
+
The private ``_walk_share`` … ``_heal_transition`` arguments belong to the
|
|
13551
|
+
heal executor (#982 spec 4.1–4.3, ``run_codex_admitted_heal``):
|
|
13552
|
+
|
|
13553
|
+
* ``_walk_share``: this call is an executor unit. Its resolver and ledger
|
|
13554
|
+
path set come from the share (bound under the locks, sealed after the
|
|
13555
|
+
last commit), and the heal fence's population hold and thread-row guard
|
|
13556
|
+
do not apply to it. Every other call holds the population.
|
|
13557
|
+
* ``_ingest_limit``: this call is one piece of a heal walk. After that
|
|
13558
|
+
many files were ingested or reset, the walk stops before the next file
|
|
13559
|
+
with bytes to ingest or reset; it and the rest stay unwalked, as at a
|
|
13560
|
+
budget stop (no deadline, no ``delta_append``), so the piece does not
|
|
13561
|
+
certify. A file that fails does not count.
|
|
13562
|
+
* ``_heal_census`` / ``_heal_visited``: the walk's census (the first
|
|
13563
|
+
piece's discovery set, ``stats.heal_discovered``; ``None`` on the first
|
|
13564
|
+
piece) and the census files earlier pieces examined successfully
|
|
13565
|
+
(``stats.heal_examined``). A piece skips visited files and files
|
|
13566
|
+
outside the census without examining them, counting them as walked
|
|
13567
|
+
(``files_skipped_unchanged``), exactly as the single walk treats a file
|
|
13568
|
+
it already visited or one created after its discovery.
|
|
13569
|
+
``stats.heal_walk_complete`` says the piece left no census file
|
|
13570
|
+
unwalked.
|
|
13571
|
+
* ``_defer_finalization``: the walk's own settlement runs only the
|
|
13572
|
+
``expand`` and ``model`` stages and the change-log prune stays owed.
|
|
13573
|
+
* ``_heal_visited_rows``: each visited file's committed cursor row as
|
|
13574
|
+
recorded when a piece visited it (``stats.heal_visit_rows``). The piece
|
|
13575
|
+
that completes the walk re-reads them under its own locks (revision
|
|
13576
|
+
7): a visited file whose row another writer changed since its visit
|
|
13577
|
+
still counts as walked, but stays in the backlog, makes the barrier
|
|
13578
|
+
census ``0`` and withholds the full-walk certificate
|
|
13579
|
+
(``stats.heal_visit_changed``); it is not re-examined.
|
|
13580
|
+
* ``_heal_transition``: the piece that completes the walk deletes a
|
|
13581
|
+
history heal record in its end-of-walk transaction, whether or not it
|
|
13582
|
+
failed a file or deferred one for a torn ``auth.json``: such a file
|
|
13583
|
+
stays owed exactly as the single walk leaves it
|
|
13584
|
+
(``stats.heal_transitioned``). A replay record is ended by the
|
|
13585
|
+
unchanged consumption block instead: deleted with the marker, or moved
|
|
13586
|
+
to phase ``certify`` when the replay is blocked.
|
|
13587
|
+
* ``_replay_stage="start"`` (#982 spec 4.5): the replay's start unit runs
|
|
13588
|
+
the byte-zero replay's preamble (D1 snapshot, preflight refusals, the
|
|
13589
|
+
clear, which writes the replay heal record, and the authoritative
|
|
13590
|
+
rehydration, which marks it ``rehydrated``) and returns before the
|
|
13591
|
+
walk. With a replay record already present nothing is cleared again,
|
|
13592
|
+
unless a later arming site marked it ``reclear``: the start unit of a
|
|
13593
|
+
new pass is the only executor unit that performs that fresh clear.
|
|
13594
|
+
* ``_heal_generation`` (spec 4.5a): the ``gen`` of the record the unit's
|
|
13595
|
+
pass started with, carried by every executor unit but the start unit.
|
|
13596
|
+
Under the writer locks a unit whose generation is not the record's, or
|
|
13597
|
+
that would have to perform a fresh clear (``reclear``, or an armed
|
|
13598
|
+
marker without a replay record), writes nothing and reports
|
|
13599
|
+
``stats.heal_generation_stale``; the transition and the consumption
|
|
13600
|
+
block act only when the generations match.
|
|
13601
|
+
|
|
13602
|
+
A replay heal record (spec 4.5) changes every other writer: the D1 test
|
|
13603
|
+
and the compressed-history ``known=`` read its persisted ``known`` set,
|
|
13604
|
+
nothing re-clears, a record not yet ``rehydrated`` is rehydrated first,
|
|
13605
|
+
a targeted call that is not an executor unit and every budgeted call
|
|
13606
|
+
decline ``replay_pending`` (the budgeted one records the deferral and
|
|
13607
|
+
sets ``stats.replay_refill``), and an unbudgeted ordinary call completes
|
|
13608
|
+
the remaining refill as an ordinary walk.
|
|
12523
13609
|
"""
|
|
12524
13610
|
if quota_reconcile not in ("auto", "defer"):
|
|
12525
13611
|
raise ValueError(
|
|
@@ -12575,7 +13661,14 @@ def _sync_codex_cache_locked(
|
|
|
12575
13661
|
rebuild_prune_scope: "_CodexPruneScope | None" = None
|
|
12576
13662
|
ordinary_prune_scope: "_CodexPruneScope | None" = None
|
|
12577
13663
|
raised = False
|
|
13664
|
+
# #982: True once this call bound the heal executor's share under its
|
|
13665
|
+
# locks; only such a call may seal (or must invalidate) it.
|
|
13666
|
+
share_bound = False
|
|
12578
13667
|
try:
|
|
13668
|
+
if _CODEX_PRE_STORE_LOCK_HOOK is not None:
|
|
13669
|
+
# Test seam (#982 R1): after admission returned, immediately
|
|
13670
|
+
# before this sync's own store-lock acquisition.
|
|
13671
|
+
_CODEX_PRE_STORE_LOCK_HOOK()
|
|
12579
13672
|
with _perf.phase("flock") as _p_flock:
|
|
12580
13673
|
acquired = acquire_cache_writer_flocks(
|
|
12581
13674
|
_cctally_core.CACHE_LOCK_PATH,
|
|
@@ -12588,6 +13681,11 @@ def _sync_codex_cache_locked(
|
|
|
12588
13681
|
if acquired is None:
|
|
12589
13682
|
eprint("[codex-cache] sync already in progress; using existing cache")
|
|
12590
13683
|
stats.lock_contended = True
|
|
13684
|
+
if deadline is not None:
|
|
13685
|
+
# #982 spec 4.4: tell the hook whether this tick would have
|
|
13686
|
+
# declined as a replay refill, so it publishes no waiting
|
|
13687
|
+
# request that the refill units could only wait out.
|
|
13688
|
+
stats.replay_refill = _codex_replay_refill_unlocked(conn)
|
|
12591
13689
|
return stats
|
|
12592
13690
|
if _after_locks is not None and _after_locks():
|
|
12593
13691
|
stats.lock_contended = stats.drain_yielded = True
|
|
@@ -12641,7 +13739,80 @@ def _sync_codex_cache_locked(
|
|
|
12641
13739
|
"SELECT 1 FROM cache_meta WHERE key=?",
|
|
12642
13740
|
(CODEX_REPLAY_FROM_ZERO_KEY,),
|
|
12643
13741
|
).fetchone() is not None
|
|
12644
|
-
if
|
|
13742
|
+
if _replay_stage not in (None, "start"):
|
|
13743
|
+
raise ValueError(
|
|
13744
|
+
"sync_codex_cache: _replay_stage must be None or 'start'")
|
|
13745
|
+
if _replay_stage is not None and (
|
|
13746
|
+
targeted or rebuild or deadline is not None):
|
|
13747
|
+
raise ValueError(
|
|
13748
|
+
"sync_codex_cache: _replay_stage is an unbudgeted whole-tree "
|
|
13749
|
+
"start unit")
|
|
13750
|
+
if _heal_generation is not None and (
|
|
13751
|
+
_walk_share is None or _replay_stage is not None or rebuild):
|
|
13752
|
+
raise ValueError(
|
|
13753
|
+
"sync_codex_cache: _heal_generation belongs to an executor "
|
|
13754
|
+
"unit that is not the start unit")
|
|
13755
|
+
# #982 spec 4.5: a replay heal record means the clear already
|
|
13756
|
+
# happened (its start unit committed). Every writer then refills
|
|
13757
|
+
# with the persisted D1 set instead of clearing again; only the
|
|
13758
|
+
# executor's units refill out of the single walk's order.
|
|
13759
|
+
heal_record = load_codex_heal_record(conn)
|
|
13760
|
+
replay_record = (heal_record if heal_record is not None
|
|
13761
|
+
and heal_record.get("mode") == "replay" else None)
|
|
13762
|
+
replay_active = replay_pending or replay_record is not None
|
|
13763
|
+
if _replay_stage is not None and not replay_active:
|
|
13764
|
+
# Nothing to start: another writer consumed the replay.
|
|
13765
|
+
return stats
|
|
13766
|
+
# #982 spec 4.5a (Generation): every executor unit but the start unit
|
|
13767
|
+
# belongs to the pass whose record generation it carries. A record
|
|
13768
|
+
# rewritten since (a fresh clear writes a new `gen`), a `reclear`
|
|
13769
|
+
# mark, or an armed marker without a replay record (a fresh clear
|
|
13770
|
+
# only a new pass's start unit may perform) ends that pass: this unit
|
|
13771
|
+
# writes nothing -- no ingest, clear, consumption, transition or
|
|
13772
|
+
# settlement, and no withdrawal of the full-walk certificate -- so no
|
|
13773
|
+
# piece ever skips a file it visited before a clear it did not see.
|
|
13774
|
+
if (_walk_share is not None and _replay_stage is None
|
|
13775
|
+
and _codex_heal_generation_stale(
|
|
13776
|
+
heal_record, replay_pending, _heal_generation)):
|
|
13777
|
+
stats.heal_generation_stale = True
|
|
13778
|
+
walk_marker_owed = False
|
|
13779
|
+
return stats
|
|
13780
|
+
# #982 spec 4.5a: a replay record re-armed by a later arming site
|
|
13781
|
+
# (`reclear`) is cleared again like an armed marker; a malformed one
|
|
13782
|
+
# (its D1 set or rehydration flag unreadable) declines every writer,
|
|
13783
|
+
# because clearing would recapture D1 from a partly refilled store.
|
|
13784
|
+
# The payload (population and D1 set) is parsed only by a call that
|
|
13785
|
+
# will use it, once, or taken from the executor's share. A `reclear`
|
|
13786
|
+
# mark acts only with the marker its arming site wrote in the same
|
|
13787
|
+
# transaction (`_codex_replay_reclear_due`); one whose marker is gone
|
|
13788
|
+
# (only a damaged store has it) is a record that no longer asks for a
|
|
13789
|
+
# replay, refilled with and retired like any other.
|
|
13790
|
+
replay_reclear = _codex_replay_reclear_due(replay_record,
|
|
13791
|
+
replay_pending)
|
|
13792
|
+
replay_payload: "CodexHealPayload | None" = None
|
|
13793
|
+
if replay_record is not None and not replay_record.get("malformed") \
|
|
13794
|
+
and deadline is None and not (targeted and _walk_share is None):
|
|
13795
|
+
replay_payload = (
|
|
13796
|
+
_walk_share.heal_payload(conn, replay_record)
|
|
13797
|
+
if _walk_share is not None
|
|
13798
|
+
else load_codex_heal_payload(conn, replay_record))
|
|
13799
|
+
replay_malformed = replay_record is not None and (
|
|
13800
|
+
bool(replay_record.get("malformed"))
|
|
13801
|
+
or (deadline is None and not (targeted and _walk_share is None)
|
|
13802
|
+
and replay_payload is None))
|
|
13803
|
+
refill_record = (replay_record if replay_record is not None
|
|
13804
|
+
and not replay_reclear and not replay_malformed
|
|
13805
|
+
else None)
|
|
13806
|
+
if replay_active and (
|
|
13807
|
+
# Spec 4.5a (revision 10): every writer, an explicit
|
|
13808
|
+
# `cache-sync --rebuild` included, fails closed on a malformed
|
|
13809
|
+
# record. Its persisted D1 set may be the only evidence for a
|
|
13810
|
+
# rollout not yet refilled, and a fresh capture cannot recover
|
|
13811
|
+
# it, so no clear may run over it.
|
|
13812
|
+
replay_malformed
|
|
13813
|
+
or deadline is not None
|
|
13814
|
+
or (targeted and (_walk_share is None
|
|
13815
|
+
or refill_record is None))):
|
|
12645
13816
|
# A live-tail tick must DEFER, never raise through the
|
|
12646
13817
|
# `targeted and rebuild` guard below.
|
|
12647
13818
|
#
|
|
@@ -12668,13 +13839,54 @@ def _sync_codex_cache_locked(
|
|
|
12668
13839
|
# and Codex ingest froze permanently. So the budgeted decline
|
|
12669
13840
|
# RECORDS itself (doctor reads the record) and hands the unbudgeted
|
|
12670
13841
|
# drain to a detached worker.
|
|
13842
|
+
#
|
|
13843
|
+
# #982 spec 4.5: with a replay heal record (the clear committed)
|
|
13844
|
+
# a targeted call proceeds only as an executor unit, and a
|
|
13845
|
+
# budgeted one still declines: ingesting outside the executor's
|
|
13846
|
+
# order would change thread rows and anchors relative to the
|
|
13847
|
+
# single walk. `replay_refill` tells the hook that the store is
|
|
13848
|
+
# cleared and partly refilled (no projection, no budget leg): a
|
|
13849
|
+
# record in phase `units`, or one too malformed to tell. A
|
|
13850
|
+
# blocked replay's `certify` record ends the fence, and its hooks
|
|
13851
|
+
# project and run the budget leg as for a blocked replay today.
|
|
12671
13852
|
if deadline is not None:
|
|
12672
13853
|
_record_codex_replay_deferral(conn)
|
|
13854
|
+
stats.replay_refill = replay_record is not None and (
|
|
13855
|
+
bool(replay_record.get("malformed"))
|
|
13856
|
+
or replay_record.get("phase") == "units")
|
|
12673
13857
|
stats.deferred_reason = "replay_pending"
|
|
12674
13858
|
return stats
|
|
12675
13859
|
# Both incompatible combinations already returned at the decline above,
|
|
12676
|
-
# so the marker-OR'd rebuild needs no second guard.
|
|
12677
|
-
|
|
13860
|
+
# so the marker-OR'd rebuild needs no second guard. A replay whose
|
|
13861
|
+
# start unit committed (a replay record) never clears again: the
|
|
13862
|
+
# caller refills with the record's persisted D1 set. An explicit
|
|
13863
|
+
# caller rebuild still clears (it is its own single walk) and keeps
|
|
13864
|
+
# the persisted set in its snapshot (a malformed record declined
|
|
13865
|
+
# above, whatever the caller asked).
|
|
13866
|
+
rebuild = rebuild or (replay_pending and (replay_record is None
|
|
13867
|
+
or replay_reclear))
|
|
13868
|
+
replay_refill = refill_record is not None and not rebuild
|
|
13869
|
+
if _walk_share is not None and deadline is not None:
|
|
13870
|
+
raise ValueError(
|
|
13871
|
+
"sync_codex_cache: _walk_share is an unbudgeted heal unit")
|
|
13872
|
+
# #982 spec 4.3: a history heal record in phase `units` is the fence.
|
|
13873
|
+
# A sync without the executor's share holds its population (skips
|
|
13874
|
+
# those files, exactly as unsettled history is held) and requests the
|
|
13875
|
+
# worker, which resumes a dead executor from the record. A rebuild is
|
|
13876
|
+
# itself the single walk the heal stands in for: it ingests every
|
|
13877
|
+
# file and retires the record in its clear transaction below.
|
|
13878
|
+
heal_history = (heal_record is not None
|
|
13879
|
+
and heal_record.get("mode") == "history")
|
|
13880
|
+
heal_population: "frozenset[str]" = frozenset()
|
|
13881
|
+
if (heal_history and heal_record.get("phase") == "units"
|
|
13882
|
+
and _walk_share is None and not rebuild):
|
|
13883
|
+
# The one parse of the payload in this call. Unreadable holds
|
|
13884
|
+
# nothing more (the fence itself still stands, fail closed).
|
|
13885
|
+
history_payload = load_codex_heal_payload(conn, heal_record)
|
|
13886
|
+
if history_payload is not None:
|
|
13887
|
+
heal_population = history_payload.population
|
|
13888
|
+
stats.drain_requested = tuple(sorted(
|
|
13889
|
+
set(stats.drain_requested) | {CODEX_REPLAY_DRAIN_COMMAND}))
|
|
12678
13890
|
# #927 spec §4.2 B4: the ordinary prune's protection needs every
|
|
12679
13891
|
# retained identity at a discovered path the scope refuses, including
|
|
12680
13892
|
# path-level refusal under a recognized root and child-only
|
|
@@ -12715,7 +13927,10 @@ def _sync_codex_cache_locked(
|
|
|
12715
13927
|
# design rejects, and it is what breaks acceptance criterion 4. Captured
|
|
12716
13928
|
# BEFORE the clear because the clear is what erases the evidence. A
|
|
12717
13929
|
# rollout first seen DURING a rebuild is absent here and takes a normal
|
|
12718
|
-
# first-ingest decision.
|
|
13930
|
+
# first-ingest decision. The clear below persists this set in its own
|
|
13931
|
+
# transaction for EVERY clear (the replay record's `known`, #982 4.5;
|
|
13932
|
+
# an explicit rebuild arms the replay there, #1051), so an interrupt
|
|
13933
|
+
# between the clear's commit and the walk's completion never loses it.
|
|
12719
13934
|
# Keyed on the DURABLE file identity, never on `path`. `path` holds the
|
|
12720
13935
|
# first configured candidate spelling, which is unstable across
|
|
12721
13936
|
# `$CODEX_HOME` reordering and symlink respelling (review F12) — a
|
|
@@ -12736,6 +13951,20 @@ def _sync_codex_cache_locked(
|
|
|
12736
13951
|
except (ValueError, TypeError, OSError):
|
|
12737
13952
|
continue
|
|
12738
13953
|
rebuild_known_identities |= _codex_history_identities(conn)
|
|
13954
|
+
if replay_payload is not None:
|
|
13955
|
+
# #982 spec 4.5: an explicit rebuild (or a re-armed replay's
|
|
13956
|
+
# fresh clear, spec 4.5a) over a refill in progress keeps
|
|
13957
|
+
# every identity the replay's own snapshot captured before its
|
|
13958
|
+
# clear (the cursors it erased).
|
|
13959
|
+
rebuild_known_identities |= replay_payload.known
|
|
13960
|
+
elif replay_refill:
|
|
13961
|
+
# #982 spec 4.5: the snapshot the replay's start unit captured
|
|
13962
|
+
# before its clear, persisted in the record. Never recaptured: the
|
|
13963
|
+
# cursors it was taken from are gone. Shared, never copied: a
|
|
13964
|
+
# refill only reads it.
|
|
13965
|
+
rebuild_known_identities = replay_payload.known
|
|
13966
|
+
# Which reads use the D1 snapshot: a clearing rebuild's and a refill's.
|
|
13967
|
+
d1_snapshot = rebuild or replay_refill
|
|
12739
13968
|
if rebuild and not targeted:
|
|
12740
13969
|
# #485: rebuild used to clear first and discover second. With an
|
|
12741
13970
|
# empty/missing/misconfigured root that made the explicit repair
|
|
@@ -12748,12 +13977,30 @@ def _sync_codex_cache_locked(
|
|
|
12748
13977
|
preflight_sources, _preflight_root_keys = (
|
|
12749
13978
|
_collect_retained_codex_paths_and_roots(conn)
|
|
12750
13979
|
)
|
|
13980
|
+
preflight_roots = _codex_provider_roots()
|
|
13981
|
+
# #983 spec 5.2: a pre-#294 S1 row carries no source_root_key. The
|
|
13982
|
+
# snapshot above skipped its cursor, and the partition would never
|
|
13983
|
+
# check the configured root that holds it, so resolve each one to
|
|
13984
|
+
# its owning root before either guard reads it. A discovered file
|
|
13985
|
+
# joins the D1 snapshot here, before the clear and the walk.
|
|
13986
|
+
if any(root_key is None for _path, root_key in preflight_sources):
|
|
13987
|
+
discovered_by_identity = {
|
|
13988
|
+
str(item.physical_path): item for item in preflight_files}
|
|
13989
|
+
resolved_sources: list[tuple[str, str | None]] = []
|
|
13990
|
+
for _path, root_key in preflight_sources:
|
|
13991
|
+
if root_key is None:
|
|
13992
|
+
root_key, owner_file = _codex_legacy_null_root_owner(
|
|
13993
|
+
_path, discovered_by_identity, preflight_roots)
|
|
13994
|
+
if owner_file is not None:
|
|
13995
|
+
rebuild_known_identities.add(
|
|
13996
|
+
codex_file_identity(owner_file))
|
|
13997
|
+
resolved_sources.append((_path, root_key))
|
|
13998
|
+
preflight_sources = resolved_sources
|
|
12751
13999
|
preflight_source_root_keys = {
|
|
12752
14000
|
root_key
|
|
12753
14001
|
for _path, root_key in preflight_sources
|
|
12754
14002
|
if root_key is not None
|
|
12755
14003
|
}
|
|
12756
|
-
preflight_roots = _codex_provider_roots()
|
|
12757
14004
|
(
|
|
12758
14005
|
_safe_sources,
|
|
12759
14006
|
refused_sources,
|
|
@@ -12797,12 +14044,45 @@ def _sync_codex_cache_locked(
|
|
|
12797
14044
|
if rebuild:
|
|
12798
14045
|
# Clear INSIDE the lock — see sync_cache() for the full
|
|
12799
14046
|
# rationale. Done before the existing SELECT so delta
|
|
12800
|
-
# detection sees an empty baseline.
|
|
14047
|
+
# detection sees an empty baseline. The clear, the full
|
|
14048
|
+
# finalization debt, the replay marker and the replay record
|
|
14049
|
+
# with the D1 set commit together in this one transaction
|
|
14050
|
+
# (#1051): there is no intermediate commit.
|
|
12801
14051
|
if _clear_codex_derived_rows(conn):
|
|
12802
14052
|
_bump_codex_physical_mutation_seq(conn)
|
|
12803
14053
|
# #980 D3: full re-derivation restores the unattributed state, so
|
|
12804
14054
|
# every root's whole range is owed, in the wipe's own transaction.
|
|
12805
14055
|
add_codex_finalization_debt(conn, full=True)
|
|
14056
|
+
if heal_history:
|
|
14057
|
+
# #982: this walk re-reads every rollout in discovery order.
|
|
14058
|
+
_delete_codex_heal_record(conn)
|
|
14059
|
+
if not replay_pending:
|
|
14060
|
+
# #1051: every clear runs as a byte-zero replay, an explicit
|
|
14061
|
+
# `cache-sync --rebuild` included. Arming in the clear's own
|
|
14062
|
+
# transaction is what persists the D1 snapshot below: an
|
|
14063
|
+
# explicit rebuild interrupted after this commit and before
|
|
14064
|
+
# its walk completes leaves the marker and the replay record,
|
|
14065
|
+
# so every retry refills with the persisted `known` instead of
|
|
14066
|
+
# sending pre-mechanism history to the live `auth.json`. The
|
|
14067
|
+
# predicate is marker absence, not `not replay_active`, so a
|
|
14068
|
+
# replay record that lost its marker gets it back too. From
|
|
14069
|
+
# here on this call runs the replay's lifecycle: the
|
|
14070
|
+
# rehydration marks the record, a clean complete walk consumes
|
|
14071
|
+
# both, a blocked one moves the record to `certify`.
|
|
14072
|
+
arm_codex_replay_from_zero(conn)
|
|
14073
|
+
replay_pending = True
|
|
14074
|
+
replay_active = True
|
|
14075
|
+
if replay_active:
|
|
14076
|
+
# #982 spec 4.5: the clear's own transaction writes the replay
|
|
14077
|
+
# heal record with the D1 snapshot, so no later writer clears
|
|
14078
|
+
# again or recaptures it from the cursors this just erased.
|
|
14079
|
+
# The rehydration's transaction below marks it `rehydrated`.
|
|
14080
|
+
_store_codex_heal_record(
|
|
14081
|
+
conn, {"mode": "replay", "phase": "units",
|
|
14082
|
+
"rehydrated": False},
|
|
14083
|
+
population=[str(item.source_path)
|
|
14084
|
+
for item in preflight_files],
|
|
14085
|
+
known=rebuild_known_identities)
|
|
12806
14086
|
conn.commit()
|
|
12807
14087
|
eprint("[cache-sync] rebuild: cleared Codex cached entries")
|
|
12808
14088
|
# #416 spec §3.4: rehydrate the attribution map from the journal BEFORE
|
|
@@ -12847,8 +14127,16 @@ def _sync_codex_cache_locked(
|
|
|
12847
14127
|
except sqlite3.DatabaseError:
|
|
12848
14128
|
_cursor_row = None
|
|
12849
14129
|
_cursor_text = _cursor_row[0] if _cursor_row else None
|
|
14130
|
+
# #982 spec 4.5: a replay record whose rehydration never committed (a
|
|
14131
|
+
# crash between the clear's commit and the rehydration's) is
|
|
14132
|
+
# rehydrated authoritatively first, under these locks, before any
|
|
14133
|
+
# rollout is read; the same transaction marks it `rehydrated`.
|
|
14134
|
+
mark_rehydrated = replay_active and (
|
|
14135
|
+
rebuild or (refill_record is not None
|
|
14136
|
+
and not refill_record.get("rehydrated")))
|
|
14137
|
+
authoritative_rehydration = bool(rebuild) or mark_rehydrated
|
|
12850
14138
|
_since = None
|
|
12851
|
-
if _cursor_text and not
|
|
14139
|
+
if _cursor_text and not authoritative_rehydration:
|
|
12852
14140
|
_seg, _sep, _off = str(_cursor_text).rpartition(":")
|
|
12853
14141
|
if _seg and _off.isdigit():
|
|
12854
14142
|
_since = (_seg, int(_off))
|
|
@@ -12869,10 +14157,10 @@ def _sync_codex_cache_locked(
|
|
|
12869
14157
|
# `database is locked` trigger #297 documents.
|
|
12870
14158
|
_rehydration = _jr.rehydrate_codex_journal_families(
|
|
12871
14159
|
conn,
|
|
12872
|
-
authoritative=
|
|
14160
|
+
authoritative=authoritative_rehydration,
|
|
12873
14161
|
file_account_since=_since,
|
|
12874
14162
|
window_attribution_since=(
|
|
12875
|
-
None if
|
|
14163
|
+
None if authoritative_rehydration
|
|
12876
14164
|
else load_codex_window_attribution_cursor(conn)),
|
|
12877
14165
|
caller="sync_codex_cache(rebuild=True)",
|
|
12878
14166
|
)
|
|
@@ -12903,7 +14191,15 @@ def _sync_codex_cache_locked(
|
|
|
12903
14191
|
# without the caller having to model that function's write
|
|
12904
14192
|
# conditions. Getting this wrong strands an open transaction across
|
|
12905
14193
|
# the whole walk.
|
|
12906
|
-
if
|
|
14194
|
+
if mark_rehydrated:
|
|
14195
|
+
rehydrated_record = load_codex_heal_record(conn)
|
|
14196
|
+
if (rehydrated_record is not None
|
|
14197
|
+
and rehydrated_record.get("mode") == "replay"
|
|
14198
|
+
and not rehydrated_record.get("malformed")):
|
|
14199
|
+
rehydrated_record["rehydrated"] = True
|
|
14200
|
+
_store_codex_heal_fence(conn, rehydrated_record)
|
|
14201
|
+
if (authoritative_rehydration or _new_cursor != _cursor_text
|
|
14202
|
+
or conn.in_transaction):
|
|
12907
14203
|
conn.commit()
|
|
12908
14204
|
# Both reports come AFTER the commit (closeout review C5): the
|
|
12909
14205
|
# `except` below rolls back, so anything printed before it would
|
|
@@ -12930,6 +14226,10 @@ def _sync_codex_cache_locked(
|
|
|
12930
14226
|
"[cache-sync] could not rehydrate Codex attribution decisions "
|
|
12931
14227
|
f"or window attributions: {exc}; deferring the Codex walk")
|
|
12932
14228
|
return stats
|
|
14229
|
+
if _replay_stage == "start":
|
|
14230
|
+
# #982 spec 4.5: the start unit ends with the rehydration; the
|
|
14231
|
+
# executor's units refill in the single walk's order.
|
|
14232
|
+
return stats
|
|
12933
14233
|
|
|
12934
14234
|
# #980 D2: a budgeted call settles the finalization debt it inherited
|
|
12935
14235
|
# BEFORE walking, under the same admission rules, so older debt cannot
|
|
@@ -13225,8 +14525,14 @@ def _sync_codex_cache_locked(
|
|
|
13225
14525
|
# #980 F3: a budgeted walk bounds the resolver's group seed read with
|
|
13226
14526
|
# its walk deadline (late-bound clock, so the walk's test seam drives
|
|
13227
14527
|
# it); an unbudgeted call is unchanged.
|
|
13228
|
-
|
|
13229
|
-
|
|
14528
|
+
if _walk_share is not None:
|
|
14529
|
+
# #982 spec 4.2: the single walk's own resolver, shared across the
|
|
14530
|
+
# heal's units; kept only while no other writer mutated quota.
|
|
14531
|
+
anchor_resolver = _walk_share.bind_walk(conn)
|
|
14532
|
+
share_bound = True
|
|
14533
|
+
else:
|
|
14534
|
+
anchor_resolver = CodexResetAnchorResolver(
|
|
14535
|
+
conn, deadline=deadline, clock=lambda: _walk_clock())
|
|
13230
14536
|
# #901 §5.3c (W11, Q14): every call without the hook's budget prepares
|
|
13231
14537
|
# its delta with no write transaction open and applies it in one
|
|
13232
14538
|
# transaction, each file in its own savepoint, at most
|
|
@@ -13246,6 +14552,16 @@ def _sync_codex_cache_locked(
|
|
|
13246
14552
|
# Whole-transaction losses per file, so a persistent one ends.
|
|
13247
14553
|
w11_losses: "dict[str, int]" = {}
|
|
13248
14554
|
walk_queue = collections.deque(enumerate(walk))
|
|
14555
|
+
# #982 spec 4.1 steps 4 and 7: a heal walk piece's census (its own
|
|
14556
|
+
# discovery set on the first piece) and the census files it found
|
|
14557
|
+
# unchanged; ``None`` outside a heal walk.
|
|
14558
|
+
heal_census: "frozenset[str] | None" = None
|
|
14559
|
+
heal_unchanged: "set[str]" = set()
|
|
14560
|
+
if _ingest_limit is not None:
|
|
14561
|
+
stats.heal_discovered = frozenset(
|
|
14562
|
+
str(item.source_path) for item in files)
|
|
14563
|
+
heal_census = (stats.heal_discovered if _heal_census is None
|
|
14564
|
+
else frozenset(_heal_census))
|
|
13249
14565
|
|
|
13250
14566
|
def _w11_batch_bytes() -> int:
|
|
13251
14567
|
return sum(item.new_input for item in w11_batch)
|
|
@@ -13495,7 +14811,9 @@ def _sync_codex_cache_locked(
|
|
|
13495
14811
|
strike_marks: list[tuple[str, int]] = []
|
|
13496
14812
|
barrier_holds: list = []
|
|
13497
14813
|
barrier_touched: set[str] = set()
|
|
13498
|
-
history = _CodexHistoryContext(
|
|
14814
|
+
history = _CodexHistoryContext(
|
|
14815
|
+
conn, ledger_source=(None if _walk_share is None
|
|
14816
|
+
else _walk_share.ledger_paths))
|
|
13499
14817
|
for _walk_index, discovered in _walk_items():
|
|
13500
14818
|
# The budget is checked BEFORE a file is opened, so a tick either
|
|
13501
14819
|
# commits a file whole (or to a recorded partial offset) or does not
|
|
@@ -13506,12 +14824,28 @@ def _sync_codex_cache_locked(
|
|
|
13506
14824
|
break
|
|
13507
14825
|
jp = discovered.source_path
|
|
13508
14826
|
path_str = str(jp)
|
|
14827
|
+
if heal_census is not None and (
|
|
14828
|
+
path_str in _heal_visited or path_str not in heal_census):
|
|
14829
|
+
# #982 spec 4.1: an earlier piece of this walk examined the
|
|
14830
|
+
# file (bytes appended since belong to later syncs), or it was
|
|
14831
|
+
# created after the census. Either way the single walk would
|
|
14832
|
+
# not read it again, so it is walked, never examined.
|
|
14833
|
+
stats.files_skipped_unchanged += 1
|
|
14834
|
+
continue
|
|
13509
14835
|
if path_str in protected_paths:
|
|
13510
14836
|
# The file is still visible, but its configured root did not
|
|
13511
14837
|
# yield recognizable Codex data. Treat that as unknown
|
|
13512
14838
|
# filesystem state: neither truncation/requalification reset
|
|
13513
14839
|
# nor append ingestion may replace the retained generation.
|
|
13514
14840
|
continue
|
|
14841
|
+
if path_str in heal_population:
|
|
14842
|
+
# #982 spec 4.3: only the heal executor ingests population
|
|
14843
|
+
# files, in the single walk's order; held like unsettled
|
|
14844
|
+
# history (the worker was requested above).
|
|
14845
|
+
stats.container_reads_held += 1
|
|
14846
|
+
stats.container_skipped["verification-pending"] = (
|
|
14847
|
+
stats.container_skipped.get("verification-pending", 0) + 1)
|
|
14848
|
+
continue
|
|
13515
14849
|
selected = _select_codex_container(jp)
|
|
13516
14850
|
if selected is None:
|
|
13517
14851
|
eprint(f"[codex-cache] stat failed for {jp}: no container")
|
|
@@ -13541,7 +14875,7 @@ def _sync_codex_cache_locked(
|
|
|
13541
14875
|
# included, is classified like compressed history.
|
|
13542
14876
|
verdict = history.classify(
|
|
13543
14877
|
conn, codex_file_identity(discovered), path_str,
|
|
13544
|
-
known=rebuild_known_identities if
|
|
14878
|
+
known=rebuild_known_identities if d1_snapshot else (),
|
|
13545
14879
|
archive=selected.location == _rc.ARCHIVE)
|
|
13546
14880
|
if verdict == "unknown":
|
|
13547
14881
|
# Untraced compressed history waits for the off-hook
|
|
@@ -13721,6 +15055,8 @@ def _sync_codex_cache_locked(
|
|
|
13721
15055
|
and size == prev_size
|
|
13722
15056
|
):
|
|
13723
15057
|
stats.files_skipped_unchanged += 1
|
|
15058
|
+
if heal_census is not None:
|
|
15059
|
+
heal_unchanged.add(path_str)
|
|
13724
15060
|
continue
|
|
13725
15061
|
elif (
|
|
13726
15062
|
not requalified and not replaced and not incomplete
|
|
@@ -13746,6 +15082,32 @@ def _sync_codex_cache_locked(
|
|
|
13746
15082
|
initial_total_tokens = 0
|
|
13747
15083
|
prev_total_tokens = None
|
|
13748
15084
|
|
|
15085
|
+
# #982 spec 4.1 steps 4 and 7: a heal walk piece ingests (or
|
|
15086
|
+
# resets) at most `_ingest_limit` files. Reaching the next file
|
|
15087
|
+
# with bytes to ingest or reset stops the piece BEFORE it is read:
|
|
15088
|
+
# it and the rest stay unwalked, as at a budget stop, and the
|
|
15089
|
+
# piece does not certify.
|
|
15090
|
+
if (_ingest_limit is not None
|
|
15091
|
+
and path_str not in stats.heal_counted_paths
|
|
15092
|
+
and len(stats.heal_counted_paths) >= _ingest_limit):
|
|
15093
|
+
requeue = (_w11_flush(final=False) if w11 and w11_batch
|
|
15094
|
+
else [])
|
|
15095
|
+
# W11 counts a file when it is prepared; one whose batch then
|
|
15096
|
+
# failed ingested nothing, so it does not count (the next
|
|
15097
|
+
# piece re-examines it) and this piece goes on.
|
|
15098
|
+
stats.heal_counted_paths &= set(committed_state)
|
|
15099
|
+
if len(stats.heal_counted_paths) < _ingest_limit:
|
|
15100
|
+
if requeue:
|
|
15101
|
+
walk_queue.extendleft(
|
|
15102
|
+
reversed(requeue + [(_walk_index, discovered)]))
|
|
15103
|
+
continue
|
|
15104
|
+
else:
|
|
15105
|
+
stats.heal_limit_stopped = True
|
|
15106
|
+
owed = ({_walk_index}
|
|
15107
|
+
| {index for index, _item in requeue}
|
|
15108
|
+
| {index for index, _item in walk_queue})
|
|
15109
|
+
unwalked = [walk[index] for index in sorted(owed)]
|
|
15110
|
+
break
|
|
13749
15111
|
# #901 §5.3c (W11): a file's new input is known before it is read.
|
|
13750
15112
|
# Preparation stops before the file that would take the batch past
|
|
13751
15113
|
# either bound; that batch is applied and committed, and
|
|
@@ -13870,7 +15232,7 @@ def _sync_codex_cache_locked(
|
|
|
13870
15232
|
# an ascending list, so the appended low offset would then shadow
|
|
13871
15233
|
# the real decision for every later byte in the file.
|
|
13872
15234
|
file_account_key = None
|
|
13873
|
-
elif
|
|
15235
|
+
elif d1_snapshot and file_identity in rebuild_known_identities:
|
|
13874
15236
|
# Spec D1: history that was never durably stamped becomes
|
|
13875
15237
|
# unattributed; nothing is inferred. A rebuild re-reads bytes
|
|
13876
15238
|
# that were ingested BEFORE this mechanism existed, and reading
|
|
@@ -14463,6 +15825,7 @@ def _sync_codex_cache_locked(
|
|
|
14463
15825
|
inode=st.st_ino,
|
|
14464
15826
|
container=container_acceptance,
|
|
14465
15827
|
barrier=barrier_read,
|
|
15828
|
+
thread_guard=heal_population,
|
|
14466
15829
|
),
|
|
14467
15830
|
merges=anchor_resolver.take_pending_merges(),
|
|
14468
15831
|
reset_counted=bool(truncated or requalified),
|
|
@@ -14477,6 +15840,8 @@ def _sync_codex_cache_locked(
|
|
|
14477
15840
|
skip_reasons=dict(iter_state.skip_reasons),
|
|
14478
15841
|
))
|
|
14479
15842
|
_codex_w11_trace("prepare_end", path_str, new_input)
|
|
15843
|
+
if _ingest_limit is not None:
|
|
15844
|
+
stats.heal_counted_paths.add(path_str)
|
|
14480
15845
|
# Private test seam: after a file is prepared and before the
|
|
14481
15846
|
# next file's preparation, which under W11 is where a later
|
|
14482
15847
|
# target's turn begins (its commit comes with its batch's).
|
|
@@ -14535,6 +15900,7 @@ def _sync_codex_cache_locked(
|
|
|
14535
15900
|
inode=st.st_ino,
|
|
14536
15901
|
container=container_acceptance,
|
|
14537
15902
|
barrier=barrier_read,
|
|
15903
|
+
thread_guard=heal_population,
|
|
14538
15904
|
)
|
|
14539
15905
|
except sqlite3.DatabaseError as exc:
|
|
14540
15906
|
conn.rollback()
|
|
@@ -14567,6 +15933,8 @@ def _sync_codex_cache_locked(
|
|
|
14567
15933
|
anchor_resolver.discard_uncommitted_file()
|
|
14568
15934
|
continue
|
|
14569
15935
|
anchor_resolver.mark_file_committed()
|
|
15936
|
+
if _ingest_limit is not None:
|
|
15937
|
+
stats.heal_counted_paths.add(path_str)
|
|
14570
15938
|
# #901 §5.3c (W9): the size trigger is evaluated after each commit.
|
|
14571
15939
|
_wal_checkpoint.after_commit(conn)
|
|
14572
15940
|
committed_state[path_str] = (
|
|
@@ -14625,6 +15993,24 @@ def _sync_codex_cache_locked(
|
|
|
14625
15993
|
# points: the pre-open deadline check fills `unwalked`, while a stop
|
|
14626
15994
|
# INSIDE a file only sets the flag.
|
|
14627
15995
|
walk_complete = not stats.budget_exhausted and not unwalked
|
|
15996
|
+
# #982 spec 4.1 steps 4 and 7 (revision 7): the piece that completes
|
|
15997
|
+
# a heal walk certifies a file an earlier piece visited only while its
|
|
15998
|
+
# committed cursor row still equals the one recorded at that visit.
|
|
15999
|
+
# A file another writer changed since (a hook that ingested part of an
|
|
16000
|
+
# append and stopped at its budget, say) still counts as walked, but
|
|
16001
|
+
# stays in this piece's backlog, makes the barrier census 0 and
|
|
16002
|
+
# withholds the full-walk certificate, exactly as a single walk
|
|
16003
|
+
# followed by that writer would leave them; it is not re-examined.
|
|
16004
|
+
visit_changed: "list[CodexDiscoveredFile]" = []
|
|
16005
|
+
if heal_census is not None and walk_complete and _heal_visited_rows:
|
|
16006
|
+
current_rows = _load_codex_session_files_rows(
|
|
16007
|
+
conn, sorted(_heal_visited_rows))
|
|
16008
|
+
changed = frozenset(
|
|
16009
|
+
path for path, row in _heal_visited_rows.items()
|
|
16010
|
+
if current_rows.get(path) != row)
|
|
16011
|
+
stats.heal_visit_changed = changed
|
|
16012
|
+
visit_changed = [item for item in files
|
|
16013
|
+
if str(item.source_path) in changed]
|
|
14628
16014
|
_p_walk.set_count(stats.files_processed)
|
|
14629
16015
|
_p_walk.set_meta(skipped=stats.files_skipped_unchanged,
|
|
14630
16016
|
rows=stats.rows_changed)
|
|
@@ -14671,7 +16057,8 @@ def _sync_codex_cache_locked(
|
|
|
14671
16057
|
else (unwalked[0] if unwalked else None))
|
|
14672
16058
|
backlog_files, backlog_bytes = _codex_backlog_after(
|
|
14673
16059
|
partial_files + unwalked
|
|
14674
|
-
+ [item.discovered for item in withdrawn]
|
|
16060
|
+
+ [item.discovered for item in withdrawn]
|
|
16061
|
+
+ visit_changed,
|
|
14675
16062
|
existing, committed_state)
|
|
14676
16063
|
stats.backlog_files = backlog_files
|
|
14677
16064
|
stats.backlog_bytes = backlog_bytes
|
|
@@ -14750,8 +16137,32 @@ def _sync_codex_cache_locked(
|
|
|
14750
16137
|
elif walk_complete:
|
|
14751
16138
|
conn.execute("DELETE FROM cache_meta WHERE key = ?",
|
|
14752
16139
|
("codex_torn_auth_deferred",))
|
|
16140
|
+
# #982 spec 4.1 step 5: the transition. The fenced walk's
|
|
16141
|
+
# piece that leaves no census file unwalked ends a history
|
|
16142
|
+
# heal's fence here, in its own end-of-walk transaction
|
|
16143
|
+
# (before its settlement, which the executor defers to the
|
|
16144
|
+
# finalization units anyway), whether or not it failed a file
|
|
16145
|
+
# or deferred one for a torn `auth.json`: such a file stays
|
|
16146
|
+
# owed exactly as the single walk leaves it, and the census
|
|
16147
|
+
# below, set from this piece's counters, keeps it blocking
|
|
16148
|
+
# new seals. (A replay heal keeps the consumption conditions
|
|
16149
|
+
# of the block after this one.) A run that later withdraws
|
|
16150
|
+
# its last batch repeats this step with `walk_complete` False
|
|
16151
|
+
# after the rollback undid the delete.
|
|
16152
|
+
if _heal_transition:
|
|
16153
|
+
stats.heal_transitioned = False
|
|
16154
|
+
if walk_complete:
|
|
16155
|
+
transition_record = load_codex_heal_record(conn)
|
|
16156
|
+
if (transition_record is not None
|
|
16157
|
+
and transition_record.get("mode") == "history"
|
|
16158
|
+
and transition_record.get("gen")
|
|
16159
|
+
== _heal_generation):
|
|
16160
|
+
_delete_codex_heal_record(conn)
|
|
16161
|
+
stats.heal_transitioned = True
|
|
14753
16162
|
# Consume the byte-zero replay marker only after a clean full walk,
|
|
14754
|
-
# and only when THIS call observed it
|
|
16163
|
+
# and only when THIS call observed it or armed it itself (#1051:
|
|
16164
|
+
# every clear arms the marker in its own transaction, so an
|
|
16165
|
+
# explicit rebuild consumes what it armed). A contended call returned
|
|
14755
16166
|
# long before here, and a walk that failed or deferred a file leaves
|
|
14756
16167
|
# the marker standing — a surviving marker is what makes the repair
|
|
14757
16168
|
# retry on the next sync, and it is also what keeps
|
|
@@ -14769,6 +16180,19 @@ def _sync_codex_cache_locked(
|
|
|
14769
16180
|
# file. It is defense in depth today — a budgeted tick declines a
|
|
14770
16181
|
# pending replay outright above — but the two clears must keep the
|
|
14771
16182
|
# same shape.
|
|
16183
|
+
# #982 spec 4.5a (Generation): an executor walk consumes the
|
|
16184
|
+
# marker, ends a replay record or moves it to `certify` only
|
|
16185
|
+
# while the record is still its pass's, re-read here under the
|
|
16186
|
+
# same locks (the preamble already returned otherwise).
|
|
16187
|
+
heal_generation_ok = True
|
|
16188
|
+
if _walk_share is not None:
|
|
16189
|
+
heal_generation_ok = not _codex_heal_generation_stale(
|
|
16190
|
+
load_codex_heal_record(conn),
|
|
16191
|
+
conn.execute(
|
|
16192
|
+
"SELECT 1 FROM cache_meta WHERE key=? LIMIT 1",
|
|
16193
|
+
(CODEX_REPLAY_FROM_ZERO_KEY,)).fetchone()
|
|
16194
|
+
is not None,
|
|
16195
|
+
_heal_generation)
|
|
14772
16196
|
if (
|
|
14773
16197
|
walk_complete
|
|
14774
16198
|
and stats.files_failed == 0
|
|
@@ -14786,17 +16210,26 @@ def _sync_codex_cache_locked(
|
|
|
14786
16210
|
)
|
|
14787
16211
|
):
|
|
14788
16212
|
_clear_codex_prune_refusal(conn)
|
|
14789
|
-
if replay_pending:
|
|
16213
|
+
if replay_pending and heal_generation_ok:
|
|
14790
16214
|
conn.execute("DELETE FROM cache_meta WHERE key = ?",
|
|
14791
16215
|
(CODEX_REPLAY_FROM_ZERO_KEY,))
|
|
14792
|
-
|
|
14793
|
-
|
|
14794
|
-
|
|
14795
|
-
|
|
14796
|
-
|
|
14797
|
-
|
|
14798
|
-
|
|
14799
|
-
|
|
16216
|
+
# #982 spec 4.1 step 5: a replay heal record ends with
|
|
16217
|
+
# the marker, in the same transaction.
|
|
16218
|
+
consumed_record = load_codex_heal_record(conn)
|
|
16219
|
+
if (heal_generation_ok and consumed_record is not None
|
|
16220
|
+
and consumed_record.get("mode") == "replay"):
|
|
16221
|
+
_delete_codex_heal_record(conn)
|
|
16222
|
+
stats.heal_transitioned = _heal_transition
|
|
16223
|
+
if heal_generation_ok:
|
|
16224
|
+
conn.execute("DELETE FROM cache_meta WHERE key = ?",
|
|
16225
|
+
(CODEX_REPLAY_BLOCKED_KEY,))
|
|
16226
|
+
# The budgeted-decline record is cleared by the same
|
|
16227
|
+
# walk, for the same reason: an unbudgeted walk
|
|
16228
|
+
# reaching here is exactly the caller whose absence
|
|
16229
|
+
# the record was reporting.
|
|
16230
|
+
conn.execute("DELETE FROM cache_meta WHERE key = ?",
|
|
16231
|
+
(CODEX_REPLAY_DEFERRED_KEY,))
|
|
16232
|
+
elif replay_pending and walk_complete and heal_generation_ok:
|
|
14800
16233
|
# A full walk ran and could NOT consume the marker, so the
|
|
14801
16234
|
# replay — and with it every Codex transcript ingest, which
|
|
14802
16235
|
# defers behind this marker — is stalled rather than merely
|
|
@@ -14808,6 +16241,15 @@ def _sync_codex_cache_locked(
|
|
|
14808
16241
|
"files_failed": stats.files_failed,
|
|
14809
16242
|
"files_deferred_torn": stats.files_deferred_torn,
|
|
14810
16243
|
}, sort_keys=True))
|
|
16244
|
+
# #982 spec 4.1 step 5: a blocked replay ends its fence
|
|
16245
|
+
# (phase `certify`) and keeps `known`; nothing re-clears.
|
|
16246
|
+
blocked_record = load_codex_heal_record(conn)
|
|
16247
|
+
if (blocked_record is not None
|
|
16248
|
+
and blocked_record.get("mode") == "replay"
|
|
16249
|
+
and blocked_record.get("phase") == "units"):
|
|
16250
|
+
blocked_record["phase"] = "certify"
|
|
16251
|
+
_store_codex_heal_fence(conn, blocked_record)
|
|
16252
|
+
stats.heal_transitioned = _heal_transition
|
|
14811
16253
|
conn.execute(_CODEX_FINALIZATION_RELEASE)
|
|
14812
16254
|
# #980 D3/D4: settle the persisted finalization debt — the debt
|
|
14813
16255
|
# this call's batches wrote and any debt inherited — in order:
|
|
@@ -14837,15 +16279,21 @@ def _sync_codex_cache_locked(
|
|
|
14837
16279
|
# a later call (or the executor) revisits it; a budgeted call
|
|
14838
16280
|
# admits a unit only while the budget leaves room, and a unit that
|
|
14839
16281
|
# reaches the deadline rolls back and is never reported as a zero.
|
|
16282
|
+
settle_stages = (
|
|
16283
|
+
{"stages": _CODEX_HEAL_FENCED_STAGES} if _defer_finalization
|
|
16284
|
+
else {})
|
|
14840
16285
|
outcome = _codex_settle_finalization(
|
|
14841
16286
|
conn, deadline=budget_deadline, clock=_finalization_clock,
|
|
14842
|
-
core_model=True, strike_state=strike_state)
|
|
16287
|
+
core_model=True, strike_state=strike_state, **settle_stages)
|
|
14843
16288
|
for message in outcome.errors:
|
|
14844
16289
|
eprint(message)
|
|
14845
16290
|
if (outcome.model_failed or outcome.adoption_failed
|
|
14846
16291
|
or outcome.reconcile_failed):
|
|
14847
16292
|
stats.maintenance_failed = True
|
|
14848
16293
|
if outcome.cancelled or outcome.deferred_to_worker or (
|
|
16294
|
+
# #982: a fenced or deferred settlement leaves debt owed.
|
|
16295
|
+
outcome.fenced or (
|
|
16296
|
+
_defer_finalization and not outcome.completed)) or (
|
|
14849
16297
|
budget_deadline is not None and not outcome.completed):
|
|
14850
16298
|
# Cancelled, handed to the worker, or a unit the remaining
|
|
14851
16299
|
# budget could not admit: the persisted debt stays owed.
|
|
@@ -14870,7 +16318,11 @@ def _sync_codex_cache_locked(
|
|
|
14870
16318
|
# prunes; when the prune is due it records it as owed maintenance
|
|
14871
16319
|
# for the drain executor instead.
|
|
14872
16320
|
with _codex_finalization_stage(conn):
|
|
14873
|
-
|
|
16321
|
+
# #982 spec 4.3: during the heal fence (and in a heal round
|
|
16322
|
+
# that defers its settlement) a due prune stays owed for the
|
|
16323
|
+
# executor instead.
|
|
16324
|
+
if (deadline is None and not _defer_finalization
|
|
16325
|
+
and not _codex_heal_fence_active(conn)):
|
|
14874
16326
|
_prune_codex_accounting_change_log(conn)
|
|
14875
16327
|
elif _codex_change_log_prune_due(conn):
|
|
14876
16328
|
record_codex_owed_maintenance(conn, change_log_prune=True)
|
|
@@ -14889,6 +16341,13 @@ def _sync_codex_cache_locked(
|
|
|
14889
16341
|
stats.maintenance_failed = False
|
|
14890
16342
|
finalization_notes.clear()
|
|
14891
16343
|
_finalization_steps(False, w11_withdrawn)
|
|
16344
|
+
if heal_census is not None:
|
|
16345
|
+
# #982 spec 4.1: committed and unchanged census files are
|
|
16346
|
+
# visited; a withdrawn batch's files are not (`committed_state`
|
|
16347
|
+
# forgot them), so the next piece examines them again.
|
|
16348
|
+
stats.heal_examined = frozenset(
|
|
16349
|
+
(heal_unchanged | set(committed_state)) & heal_census)
|
|
16350
|
+
stats.heal_walk_complete = walk_complete and not w11_lost
|
|
14892
16351
|
# Projection intentionally runs only after this function releases the
|
|
14893
16352
|
# Codex cache flock in ``finally`` below. cache.db and stats.db are not
|
|
14894
16353
|
# cross-database atomic: after this committed ingest, a projection
|
|
@@ -14975,6 +16434,7 @@ def _sync_codex_cache_locked(
|
|
|
14975
16434
|
and not stats.files_deferred_reset
|
|
14976
16435
|
and not stats.container_pending
|
|
14977
16436
|
and not stats.container_reads_held
|
|
16437
|
+
and not stats.heal_visit_changed
|
|
14978
16438
|
and (
|
|
14979
16439
|
stats.files_processed + stats.files_skipped_unchanged
|
|
14980
16440
|
+ sum(stats.container_skipped.values())
|
|
@@ -15016,6 +16476,7 @@ def _sync_codex_cache_locked(
|
|
|
15016
16476
|
barrier_census = bool(
|
|
15017
16477
|
walk_complete and stats.files_failed == 0
|
|
15018
16478
|
and stats.files_deferred_torn == 0 and not w11_lost
|
|
16479
|
+
and not stats.heal_visit_changed
|
|
15019
16480
|
and stats.files_processed + stats.files_skipped_unchanged
|
|
15020
16481
|
+ sum(stats.container_skipped.values())
|
|
15021
16482
|
+ stats.container_pending == stats.files_total)
|
|
@@ -15083,6 +16544,11 @@ def _sync_codex_cache_locked(
|
|
|
15083
16544
|
# one recorded the drain as owed above.
|
|
15084
16545
|
if deadline is None:
|
|
15085
16546
|
_maybe_truncate_wal(conn, _cctally_core.CACHE_DB_PATH)
|
|
16547
|
+
if heal_census is not None and stats.heal_examined:
|
|
16548
|
+
# Revision 7: what this piece visited, as committed, read after
|
|
16549
|
+
# its last write and before its locks are released.
|
|
16550
|
+
stats.heal_visit_rows = _load_codex_session_files_rows(
|
|
16551
|
+
conn, sorted(stats.heal_examined))
|
|
15086
16552
|
except BaseException:
|
|
15087
16553
|
raised = True
|
|
15088
16554
|
raise
|
|
@@ -15126,7 +16592,16 @@ def _sync_codex_cache_locked(
|
|
|
15126
16592
|
raise
|
|
15127
16593
|
if read_policy is not None and raised:
|
|
15128
16594
|
read_policy.invalidate("cache_unavailable")
|
|
16595
|
+
if share_bound and not raised:
|
|
16596
|
+
# #982 spec 4.2: after this unit's last commit, before its
|
|
16597
|
+
# locks are released: the next unit reuses the memo only if
|
|
16598
|
+
# nobody mutated quota in between.
|
|
16599
|
+
_walk_share.seal(conn)
|
|
16600
|
+
share_bound = False
|
|
15129
16601
|
finally:
|
|
16602
|
+
if share_bound:
|
|
16603
|
+
# An exception (or a failure above): the next unit reseeds.
|
|
16604
|
+
_walk_share.invalidate()
|
|
15130
16605
|
# #981 R4: released on every path, whatever the pin raised.
|
|
15131
16606
|
release_cache_writer_flocks(held_writer_flocks)
|
|
15132
16607
|
|
|
@@ -15342,9 +16817,299 @@ class FinalizationOutcome:
|
|
|
15342
16817
|
restored: int = 0
|
|
15343
16818
|
readopted: int = 0
|
|
15344
16819
|
deferred_to_worker: bool = False
|
|
16820
|
+
#: #982 spec 4.3: the heal fence restricted this settlement to the
|
|
16821
|
+
#: ``expand`` and ``model`` stages; callers treat it like a deferred unit.
|
|
16822
|
+
fenced: bool = False
|
|
15345
16823
|
errors: list = field(default_factory=list)
|
|
15346
16824
|
|
|
15347
16825
|
|
|
16826
|
+
# --------------------------------------------------------------------------
|
|
16827
|
+
# #982 / #975 — the Codex heal record and its fence (spec 4.3)
|
|
16828
|
+
# --------------------------------------------------------------------------
|
|
16829
|
+
#
|
|
16830
|
+
# The heal record is two ``cache_meta`` keys written and retired together:
|
|
16831
|
+
#
|
|
16832
|
+
# * ``codex_heal_walk``, the small FENCE key ``{"v", "gen", "mode", "phase"}``
|
|
16833
|
+
# plus, in replay mode, ``"rehydrated"`` and, once a later arming site
|
|
16834
|
+
# re-arms the replay, ``"reclear"``. Every fence check reads only this key.
|
|
16835
|
+
# * ``codex_heal_walk_payload``, the history-sized half ``{"v", "gen",
|
|
16836
|
+
# "population"}`` plus, in replay mode, ``"known"`` (the D1 snapshot). It is
|
|
16837
|
+
# parsed at most once per sync call, or once per executor pass (the walk
|
|
16838
|
+
# share hands it to the units), and bound to the fence key by ``gen``.
|
|
16839
|
+
#
|
|
16840
|
+
# Both keys are written once by the heal executor's start unit (or the
|
|
16841
|
+
# replay's clear) in one transaction and deleted together once by the
|
|
16842
|
+
# transition (the first fenced certification round that walks clean and
|
|
16843
|
+
# complete); they are never rewritten per unit. Only the fence key changes in
|
|
16844
|
+
# between: the rehydration sets ``rehydrated``, a blocked replay moves it to
|
|
16845
|
+
# phase ``certify``, and ``arm_codex_replay_from_zero`` sets ``reclear``.
|
|
16846
|
+
# While the fence key holds phase ``units`` (the fence):
|
|
16847
|
+
#
|
|
16848
|
+
# * only the executor's units (syncs carrying its ``_walk_share``) ingest a
|
|
16849
|
+
# population file; every other sync holds them and requests the worker;
|
|
16850
|
+
# * every one-way stamping path is restricted: the settlement chokepoint runs
|
|
16851
|
+
# only ``expand`` and ``model`` and reports ``fenced``, and the two stamping
|
|
16852
|
+
# kernels stamp nothing (or raise ``CodexHealFenced`` when strict);
|
|
16853
|
+
# * a non-executor thread-row upsert never replaces a population file's row;
|
|
16854
|
+
# * the accounting change-log prune deletes nothing and stays owed.
|
|
16855
|
+
|
|
16856
|
+
CODEX_HEAL_WALK_KEY = "codex_heal_walk"
|
|
16857
|
+
CODEX_HEAL_PAYLOAD_KEY = "codex_heal_walk_payload"
|
|
16858
|
+
CODEX_HEAL_WALK_LOCK_NAME = "codex-heal-walk.lock"
|
|
16859
|
+
CODEX_HEAL_FENCED_TOKEN = "deferred_heal"
|
|
16860
|
+
_CODEX_HEAL_RECORD_VERSION = 1
|
|
16861
|
+
_CODEX_HEAL_FENCED_STAGES = ("expand", "model")
|
|
16862
|
+
|
|
16863
|
+
|
|
16864
|
+
class CodexHealFenced(sqlite3.DatabaseError):
|
|
16865
|
+
"""A strict stamping kernel refused while the heal fence holds.
|
|
16866
|
+
|
|
16867
|
+
A ``sqlite3.DatabaseError`` whose text is the status token
|
|
16868
|
+
``deferred_heal``, mirroring ``deferred_model``: ``account attribute``'s
|
|
16869
|
+
all-or-nothing apply rolls its cache transaction back and reaches its
|
|
16870
|
+
existing ``recordedPending`` outcome, and a rerun after the heal
|
|
16871
|
+
completes it."""
|
|
16872
|
+
|
|
16873
|
+
|
|
16874
|
+
class CodexHealPayload(NamedTuple):
|
|
16875
|
+
"""The history-sized half of a heal record, parsed once."""
|
|
16876
|
+
|
|
16877
|
+
gen: str
|
|
16878
|
+
population: "frozenset[str]"
|
|
16879
|
+
known: "frozenset[str] | None" # replay mode only (the D1 snapshot)
|
|
16880
|
+
|
|
16881
|
+
|
|
16882
|
+
def _codex_heal_meta_value(conn: sqlite3.Connection, key: str):
|
|
16883
|
+
try:
|
|
16884
|
+
row = conn.execute("SELECT value FROM cache_meta WHERE key=?",
|
|
16885
|
+
(key,)).fetchone()
|
|
16886
|
+
except sqlite3.OperationalError as exc:
|
|
16887
|
+
if "no such table" in str(exc):
|
|
16888
|
+
return None
|
|
16889
|
+
raise
|
|
16890
|
+
return None if row is None else row[0]
|
|
16891
|
+
|
|
16892
|
+
|
|
16893
|
+
def load_codex_heal_record(conn: sqlite3.Connection) -> "dict | None":
|
|
16894
|
+
"""The heal record's small fence key, or None. Never the payload.
|
|
16895
|
+
|
|
16896
|
+
Unreadable fails closed. A record nobody can parse is a fence over an
|
|
16897
|
+
unknown population marked ``malformed``: in history mode (the next
|
|
16898
|
+
executor's transition removes it), or in replay mode when the record
|
|
16899
|
+
says so or the byte-zero replay marker is armed, because a replay record
|
|
16900
|
+
that lost its D1 set must never be cleared again with a recaptured one
|
|
16901
|
+
(spec 4.5a): writers decline and the executor ends ``degraded``."""
|
|
16902
|
+
text = _codex_heal_meta_value(conn, CODEX_HEAL_WALK_KEY)
|
|
16903
|
+
if text is None:
|
|
16904
|
+
return None
|
|
16905
|
+
try:
|
|
16906
|
+
record = json.loads(str(text))
|
|
16907
|
+
except (TypeError, ValueError):
|
|
16908
|
+
record = None
|
|
16909
|
+
if (isinstance(record, dict)
|
|
16910
|
+
and record.get("v") == _CODEX_HEAL_RECORD_VERSION
|
|
16911
|
+
and isinstance(record.get("gen"), str)
|
|
16912
|
+
and record.get("mode") in ("history", "replay")
|
|
16913
|
+
and record.get("phase") in ("units", "certify")
|
|
16914
|
+
and isinstance(record.get("reclear", False), bool)
|
|
16915
|
+
and (record.get("mode") != "replay"
|
|
16916
|
+
or isinstance(record.get("rehydrated"), bool))):
|
|
16917
|
+
return record
|
|
16918
|
+
replay = (isinstance(record, dict) and record.get("mode") == "replay")
|
|
16919
|
+
if not replay:
|
|
16920
|
+
try:
|
|
16921
|
+
replay = conn.execute(
|
|
16922
|
+
"SELECT 1 FROM cache_meta WHERE key=? LIMIT 1",
|
|
16923
|
+
(CODEX_REPLAY_FROM_ZERO_KEY,)).fetchone() is not None
|
|
16924
|
+
except sqlite3.OperationalError:
|
|
16925
|
+
replay = False
|
|
16926
|
+
return {"v": _CODEX_HEAL_RECORD_VERSION, "gen": None,
|
|
16927
|
+
"mode": "replay" if replay else "history", "phase": "units",
|
|
16928
|
+
"malformed": True}
|
|
16929
|
+
|
|
16930
|
+
|
|
16931
|
+
def _codex_replay_refill_unlocked(conn: sqlite3.Connection) -> bool:
|
|
16932
|
+
"""Would a budgeted sync decline as a replay refill (#982 spec 4.4)?
|
|
16933
|
+
|
|
16934
|
+
Read without the store locks by a budgeted call that met contention: a
|
|
16935
|
+
replay heal record in phase ``units``, or one too malformed to tell, is
|
|
16936
|
+
the state in which no budgeted tick ingests, projects or evaluates a
|
|
16937
|
+
budget (the predicate the decline sets ``replay_refill`` from). A waiting
|
|
16938
|
+
request could then only make the refill units idle for its lifetime.
|
|
16939
|
+
A marker without a record (the start unit has not committed) and a
|
|
16940
|
+
blocked replay's ``certify`` record answer False: their hooks still
|
|
16941
|
+
project, so their requests are kept. Unreadable answers False, so the
|
|
16942
|
+
hook publishes its request, as today."""
|
|
16943
|
+
try:
|
|
16944
|
+
record = load_codex_heal_record(conn)
|
|
16945
|
+
except sqlite3.Error:
|
|
16946
|
+
return False
|
|
16947
|
+
return (record is not None and record.get("mode") == "replay"
|
|
16948
|
+
and (bool(record.get("malformed"))
|
|
16949
|
+
or record.get("phase") == "units"))
|
|
16950
|
+
|
|
16951
|
+
|
|
16952
|
+
def load_codex_heal_payload(
|
|
16953
|
+
conn: sqlite3.Connection,
|
|
16954
|
+
record: "dict | None") -> "CodexHealPayload | None":
|
|
16955
|
+
"""Parse the record's payload: the population and, in replay mode, the
|
|
16956
|
+
D1 set. None when the record is malformed or the payload is missing,
|
|
16957
|
+
malformed or another record's (its ``gen`` differs); callers fail closed
|
|
16958
|
+
(a history fence over an empty population; a replay that declines)."""
|
|
16959
|
+
if record is None or record.get("malformed"):
|
|
16960
|
+
return None
|
|
16961
|
+
text = _codex_heal_meta_value(conn, CODEX_HEAL_PAYLOAD_KEY)
|
|
16962
|
+
if text is None:
|
|
16963
|
+
return None
|
|
16964
|
+
try:
|
|
16965
|
+
raw = json.loads(str(text))
|
|
16966
|
+
except (TypeError, ValueError):
|
|
16967
|
+
return None
|
|
16968
|
+
if not (isinstance(raw, dict)
|
|
16969
|
+
and raw.get("v") == _CODEX_HEAL_RECORD_VERSION
|
|
16970
|
+
and raw.get("gen") == record.get("gen")):
|
|
16971
|
+
return None
|
|
16972
|
+
population = raw.get("population")
|
|
16973
|
+
if not (isinstance(population, list)
|
|
16974
|
+
and all(isinstance(p, str) for p in population)):
|
|
16975
|
+
return None
|
|
16976
|
+
known = None
|
|
16977
|
+
if record.get("mode") == "replay":
|
|
16978
|
+
known = raw.get("known")
|
|
16979
|
+
if not (isinstance(known, list)
|
|
16980
|
+
and all(isinstance(k, str) for k in known)):
|
|
16981
|
+
return None
|
|
16982
|
+
known = frozenset(known)
|
|
16983
|
+
return CodexHealPayload(str(raw["gen"]), frozenset(population), known)
|
|
16984
|
+
|
|
16985
|
+
|
|
16986
|
+
def _codex_heal_fence_active(conn: sqlite3.Connection) -> bool:
|
|
16987
|
+
"""The fence: a heal record in phase ``units``."""
|
|
16988
|
+
record = load_codex_heal_record(conn)
|
|
16989
|
+
return record is not None and record.get("phase") == "units"
|
|
16990
|
+
|
|
16991
|
+
|
|
16992
|
+
def _codex_heal_fence_json(record: dict) -> str:
|
|
16993
|
+
fence = {key: value for key, value in record.items()
|
|
16994
|
+
if key not in ("malformed", "population", "known")}
|
|
16995
|
+
return json.dumps(fence, sort_keys=True, separators=(",", ":"))
|
|
16996
|
+
|
|
16997
|
+
|
|
16998
|
+
def _store_codex_heal_record(conn: sqlite3.Connection, record: dict, *,
|
|
16999
|
+
population, known=None) -> dict:
|
|
17000
|
+
"""Write a NEW record, both keys under a fresh ``gen``, on the caller's
|
|
17001
|
+
transaction (the history start unit, the replay clear). Returns the
|
|
17002
|
+
fence key as written."""
|
|
17003
|
+
fence = {key: value for key, value in record.items()
|
|
17004
|
+
if key not in ("malformed", "population", "known", "reclear")}
|
|
17005
|
+
fence["v"] = _CODEX_HEAL_RECORD_VERSION
|
|
17006
|
+
fence["gen"] = uuid.uuid4().hex
|
|
17007
|
+
payload = {"v": _CODEX_HEAL_RECORD_VERSION, "gen": fence["gen"],
|
|
17008
|
+
"population": sorted(set(str(p) for p in population))}
|
|
17009
|
+
if fence.get("mode") == "replay":
|
|
17010
|
+
payload["known"] = sorted(set(known or ()))
|
|
17011
|
+
conn.execute(
|
|
17012
|
+
"INSERT OR REPLACE INTO cache_meta(key, value) VALUES (?, ?)",
|
|
17013
|
+
(CODEX_HEAL_PAYLOAD_KEY,
|
|
17014
|
+
json.dumps(payload, sort_keys=True, separators=(",", ":"))))
|
|
17015
|
+
conn.execute(
|
|
17016
|
+
"INSERT OR REPLACE INTO cache_meta(key, value) VALUES (?, ?)",
|
|
17017
|
+
(CODEX_HEAL_WALK_KEY, _codex_heal_fence_json(fence)))
|
|
17018
|
+
return fence
|
|
17019
|
+
|
|
17020
|
+
|
|
17021
|
+
def _store_codex_heal_fence(conn: sqlite3.Connection, record: dict) -> None:
|
|
17022
|
+
"""Rewrite only the fence key of an existing record (its ``gen`` and
|
|
17023
|
+
payload unchanged): ``rehydrated``, phase ``certify``, ``reclear``."""
|
|
17024
|
+
conn.execute(
|
|
17025
|
+
"INSERT OR REPLACE INTO cache_meta(key, value) VALUES (?, ?)",
|
|
17026
|
+
(CODEX_HEAL_WALK_KEY, _codex_heal_fence_json(record)))
|
|
17027
|
+
|
|
17028
|
+
|
|
17029
|
+
def _delete_codex_heal_record(conn: sqlite3.Connection) -> None:
|
|
17030
|
+
conn.execute("DELETE FROM cache_meta WHERE key IN (?, ?)",
|
|
17031
|
+
(CODEX_HEAL_WALK_KEY, CODEX_HEAL_PAYLOAD_KEY))
|
|
17032
|
+
|
|
17033
|
+
|
|
17034
|
+
def arm_codex_replay_from_zero(conn: sqlite3.Connection) -> None:
|
|
17035
|
+
"""Arm the guarded byte-zero Codex replay, on the caller's transaction.
|
|
17036
|
+
|
|
17037
|
+
Every arming site goes through here (spec 4.5a). Without a replay heal
|
|
17038
|
+
record this writes ``CODEX_REPLAY_FROM_ZERO_KEY`` and nothing else. With
|
|
17039
|
+
one (a refill in progress or a blocked replay's ``certify`` record) the
|
|
17040
|
+
clear already happened, so a bare marker would only be refilled and the
|
|
17041
|
+
repair it was armed for would never re-derive the rows already refilled.
|
|
17042
|
+
The record is marked ``reclear`` instead: the next writer clears again
|
|
17043
|
+
exactly as an armed marker does today, with the fresh D1 capture united
|
|
17044
|
+
with the record's persisted ``known`` (a rollout not yet refilled keeps
|
|
17045
|
+
its pre-mechanism identity, I7), and writes a new record. A malformed
|
|
17046
|
+
record is left as it is: it already fails closed.
|
|
17047
|
+
"""
|
|
17048
|
+
_set_cache_meta(conn, CODEX_REPLAY_FROM_ZERO_KEY, "1")
|
|
17049
|
+
record = load_codex_heal_record(conn)
|
|
17050
|
+
if (record is not None and record.get("mode") == "replay"
|
|
17051
|
+
and not record.get("malformed") and not record.get("reclear")):
|
|
17052
|
+
record["reclear"] = True
|
|
17053
|
+
_store_codex_heal_fence(conn, record)
|
|
17054
|
+
|
|
17055
|
+
|
|
17056
|
+
def _codex_replay_reclear_due(record: "dict | None",
|
|
17057
|
+
replay_armed: bool) -> bool:
|
|
17058
|
+
"""Whether a replay record's ``reclear`` mark asks for a fresh clear.
|
|
17059
|
+
|
|
17060
|
+
``arm_codex_replay_from_zero`` writes the mark with the replay marker in
|
|
17061
|
+
one transaction, and the consumption block deletes the marker with the
|
|
17062
|
+
record, so the mark acts only while its marker is armed. A mark whose
|
|
17063
|
+
marker is gone (only a damaged ``cache_meta`` has one) asks for nothing:
|
|
17064
|
+
every writer treats the record as one that no longer needs a replay,
|
|
17065
|
+
refills with its persisted D1 set and retires it, and none clears."""
|
|
17066
|
+
return bool(replay_armed and record is not None
|
|
17067
|
+
and record.get("mode") == "replay" and record.get("reclear"))
|
|
17068
|
+
|
|
17069
|
+
|
|
17070
|
+
def _codex_heal_generation_stale(record: "dict | None", replay_armed: bool,
|
|
17071
|
+
generation: "str | None") -> bool:
|
|
17072
|
+
"""#982 spec 4.5a (Generation): whether an executor unit of the pass
|
|
17073
|
+
that started with record generation ``generation`` must write nothing.
|
|
17074
|
+
|
|
17075
|
+
True when the store now asks for a fresh clear, which only the start
|
|
17076
|
+
unit of a new pass (or an unbudgeted ordinary sync outside any pass) may
|
|
17077
|
+
perform: the replay marker is armed without a replay record, or the
|
|
17078
|
+
replay record is marked ``reclear`` with that marker armed
|
|
17079
|
+
(``_codex_replay_reclear_due``). Also true when the record is not
|
|
17080
|
+
the pass's (another ``gen``). An unreadable record has no ``gen``, so it
|
|
17081
|
+
is the pass's only when the pass started from it (a malformed history
|
|
17082
|
+
record, which that pass's transition removes; a malformed replay record
|
|
17083
|
+
still declines every writer below). A missing record with no armed
|
|
17084
|
+
marker is not stale: the pass's transition (or another writer's
|
|
17085
|
+
complete walk) ended it, and the finalization units and the closing walk
|
|
17086
|
+
run after that."""
|
|
17087
|
+
if replay_armed and (record is None or record.get("mode") != "replay"):
|
|
17088
|
+
return True
|
|
17089
|
+
if record is None:
|
|
17090
|
+
return False
|
|
17091
|
+
return (_codex_replay_reclear_due(record, replay_armed)
|
|
17092
|
+
or record.get("gen") != generation)
|
|
17093
|
+
|
|
17094
|
+
|
|
17095
|
+
def _codex_heal_fenced_kernel(conn: sqlite3.Connection, strict: bool) -> bool:
|
|
17096
|
+
"""The stamping kernels' own check (spec 4.3): True means "stamp nothing";
|
|
17097
|
+
strict callers get ``CodexHealFenced`` instead."""
|
|
17098
|
+
try:
|
|
17099
|
+
fenced = _codex_heal_fence_active(conn)
|
|
17100
|
+
except sqlite3.DatabaseError as exc:
|
|
17101
|
+
if (isinstance(exc, sqlite3.OperationalError)
|
|
17102
|
+
and _is_sqlite_interrupt(exc)):
|
|
17103
|
+
# A cancellation, never a swallowable read error (D4).
|
|
17104
|
+
raise CodexFinalizationCancelled("interrupt") from exc
|
|
17105
|
+
if strict:
|
|
17106
|
+
raise
|
|
17107
|
+
return True
|
|
17108
|
+
if fenced and strict:
|
|
17109
|
+
raise CodexHealFenced(CODEX_HEAL_FENCED_TOKEN)
|
|
17110
|
+
return fenced
|
|
17111
|
+
|
|
17112
|
+
|
|
15348
17113
|
def _codex_malformed_debt() -> CodexFinalizationDebt:
|
|
15349
17114
|
return CodexFinalizationDebt(
|
|
15350
17115
|
full=True, model=True, attribution=True, malformed=True)
|
|
@@ -15993,6 +17758,13 @@ def _codex_settle_finalization(
|
|
|
15993
17758
|
"""
|
|
15994
17759
|
outcome = FinalizationOutcome()
|
|
15995
17760
|
budgeted = deadline is not None
|
|
17761
|
+
if _codex_heal_fence_active(conn):
|
|
17762
|
+
# #982 spec 4.3: while a heal holds its population, adoption and
|
|
17763
|
+
# reconciliation (one-way NULL -> account stamps) would decide from a
|
|
17764
|
+
# partly ingested store. Expansion and model resolution are safe:
|
|
17765
|
+
# model resolution reads only the observation's own rollout.
|
|
17766
|
+
stages = tuple(s for s in stages if s in _CODEX_HEAL_FENCED_STAGES)
|
|
17767
|
+
outcome.fenced = True
|
|
15996
17768
|
if strike_state is None:
|
|
15997
17769
|
# One call, one strike: a caller settling in several steps (the sync's
|
|
15998
17770
|
# pre-walk settle and its finalization) shares one state instead.
|
|
@@ -16359,6 +18131,9 @@ def apply_codex_window_spend_adoption(
|
|
|
16359
18131
|
high = max(span[1] for span in spans.values())
|
|
16360
18132
|
reset_bounds = (low - _CODEX_ACCOUNT_WEEK, high + _CODEX_ACCOUNT_WEEK)
|
|
16361
18133
|
candidate_bounds = (low - _CODEX_ACCOUNT_WEEK, high)
|
|
18134
|
+
# #982 spec 4.3: no one-way stamp while a heal holds its population.
|
|
18135
|
+
if _codex_heal_fenced_kernel(conn, strict):
|
|
18136
|
+
return 0
|
|
16362
18137
|
|
|
16363
18138
|
from _cctally_quota import load_codex_quota_observations
|
|
16364
18139
|
|
|
@@ -16501,6 +18276,59 @@ def _codex_stored_identity_path(
|
|
|
16501
18276
|
return _codex_identity_path(selected, _codex_root_for_logical(logical, roots))
|
|
16502
18277
|
|
|
16503
18278
|
|
|
18279
|
+
def _codex_legacy_null_root_owner(
|
|
18280
|
+
path: str,
|
|
18281
|
+
discovered_by_identity: "dict[str, CodexDiscoveredFile]",
|
|
18282
|
+
roots: "list[CodexProviderRoot]",
|
|
18283
|
+
) -> "tuple[str | None, CodexDiscoveredFile | None]":
|
|
18284
|
+
"""#983 spec 5.2: ``(root_key, discovered_file | None)`` for a stored path
|
|
18285
|
+
whose row carries no ``source_root_key`` (pre-#294 S1), or ``(None, None)``.
|
|
18286
|
+
|
|
18287
|
+
Ownership follows discovery's own answer, never literal containment of the
|
|
18288
|
+
stored spelling, because a symlinked spelling, overlapping roots and the
|
|
18289
|
+
#970 archive release all defeat that. In order:
|
|
18290
|
+
|
|
18291
|
+
1. the stored identity names a discovered file: that file's root, and the
|
|
18292
|
+
file itself so the rebuild's D1 snapshot gains exactly the identity the
|
|
18293
|
+
walk computes for it;
|
|
18294
|
+
2. otherwise the first root in configured order whose CANONICALIZED walk
|
|
18295
|
+
or archive scope contains the identity (``_codex_scope_root`` would
|
|
18296
|
+
compare the configured spelling, which misses a symlinked root);
|
|
18297
|
+
3. otherwise a configured root ``_codex_provider_roots`` omitted (an
|
|
18298
|
+
unmounted volume, a missing directory), keyed as ``_codex_prune_scope``
|
|
18299
|
+
keys it, so the scope refuses it as configured but unrecognizable;
|
|
18300
|
+
4. otherwise no owner: the path is outside every configured root and stays
|
|
18301
|
+
removable, as a root-keyed row whose root left the configured set is.
|
|
18302
|
+
"""
|
|
18303
|
+
try:
|
|
18304
|
+
phys = _codex_stored_identity_path(path, roots)
|
|
18305
|
+
except (OSError, TypeError, ValueError):
|
|
18306
|
+
return None, None
|
|
18307
|
+
discovered = discovered_by_identity.get(str(phys))
|
|
18308
|
+
if discovered is not None:
|
|
18309
|
+
return discovered.source_root_key, discovered
|
|
18310
|
+
for root in roots:
|
|
18311
|
+
scopes = [root.walk_root]
|
|
18312
|
+
if root.archive_root is not None:
|
|
18313
|
+
scopes.append(root.archive_root)
|
|
18314
|
+
for scope in scopes:
|
|
18315
|
+
try:
|
|
18316
|
+
if phys.is_relative_to(_canonical_codex_path(scope)):
|
|
18317
|
+
return root.source_root_key, None
|
|
18318
|
+
except (OSError, TypeError, ValueError):
|
|
18319
|
+
continue
|
|
18320
|
+
walked = {root.source_root_key for root in roots}
|
|
18321
|
+
for raw_root in _cctally()._codex_home_roots():
|
|
18322
|
+
try:
|
|
18323
|
+
canonical = _canonical_codex_path(raw_root)
|
|
18324
|
+
key = source_root_key(str(canonical))
|
|
18325
|
+
if key not in walked and phys.is_relative_to(canonical):
|
|
18326
|
+
return key, None
|
|
18327
|
+
except (OSError, TypeError, ValueError):
|
|
18328
|
+
continue
|
|
18329
|
+
return None, None
|
|
18330
|
+
|
|
18331
|
+
|
|
16504
18332
|
|
|
16505
18333
|
def codex_file_key_for_entry_path(
|
|
16506
18334
|
source_root_key: str, source_path: str, *,
|
|
@@ -16620,9 +18448,9 @@ class _CodexHistoryContext:
|
|
|
16620
18448
|
"""One walk's provenance state, read once before the walk loop."""
|
|
16621
18449
|
|
|
16622
18450
|
__slots__ = ("payload", "traced", "settle_on_finish", "ledger_paths",
|
|
16623
|
-
"consulted", "archives_covered")
|
|
18451
|
+
"consulted", "archives_covered", "ledger_source")
|
|
16624
18452
|
|
|
16625
|
-
def __init__(self, conn):
|
|
18453
|
+
def __init__(self, conn, *, ledger_source=None):
|
|
16626
18454
|
import _cctally_journal as _jr
|
|
16627
18455
|
self.payload = _codex_history_marker(conn)
|
|
16628
18456
|
self.archives_covered = _codex_history_marker_covers_archives(
|
|
@@ -16635,6 +18463,9 @@ class _CodexHistoryContext:
|
|
|
16635
18463
|
and _jr.journal_high_water() is None)
|
|
16636
18464
|
self.ledger_paths = None
|
|
16637
18465
|
self.consulted = False
|
|
18466
|
+
# #982 spec 4.2: a heal unit reads the ledger path set through its
|
|
18467
|
+
# share (one full read per pass, then only new rows).
|
|
18468
|
+
self.ledger_source = ledger_source
|
|
16638
18469
|
|
|
16639
18470
|
def classify(self, conn, identity: str, path: str, *, known=(),
|
|
16640
18471
|
archive: bool = False) -> str:
|
|
@@ -16657,7 +18488,9 @@ class _CodexHistoryContext:
|
|
|
16657
18488
|
self.consulted = True
|
|
16658
18489
|
return "history"
|
|
16659
18490
|
if self.ledger_paths is None:
|
|
16660
|
-
self.ledger_paths = _codex_ledger_paths(conn)
|
|
18491
|
+
self.ledger_paths = (_codex_ledger_paths(conn)
|
|
18492
|
+
if self.ledger_source is None
|
|
18493
|
+
else self.ledger_source(conn))
|
|
16661
18494
|
if _codex_has_history_provenance(conn, identity, path,
|
|
16662
18495
|
ledger_paths=self.ledger_paths):
|
|
16663
18496
|
return "history"
|
|
@@ -16721,6 +18554,40 @@ def _codex_history_candidates(*, archive_only: bool = False) -> "list[str]":
|
|
|
16721
18554
|
conn.close()
|
|
16722
18555
|
|
|
16723
18556
|
|
|
18557
|
+
def _codex_provenance_identity_write(
|
|
18558
|
+
share: _CodexWalkShare, path: str, apply, *, wait_for_requests: bool):
|
|
18559
|
+
"""One provenance identity's admitted transaction over ``share``.
|
|
18560
|
+
|
|
18561
|
+
``apply(conn, anchors)`` receives the identity's resolver, built from
|
|
18562
|
+
copies of the share's committed-evidence base (#975, spec 4.2), and
|
|
18563
|
+
returns whether it applied. After the commit, still under the writer
|
|
18564
|
+
locks, the base absorbs that identity's stored rows and the token is
|
|
18565
|
+
sealed. A refusal, an exception or a rollback adds nothing and drops the
|
|
18566
|
+
base, so the next identity seeds from the committed store.
|
|
18567
|
+
"""
|
|
18568
|
+
applied: list = []
|
|
18569
|
+
|
|
18570
|
+
def run(conn):
|
|
18571
|
+
result = apply(conn, share.identity_resolver(conn))
|
|
18572
|
+
applied.append(bool(result))
|
|
18573
|
+
return result
|
|
18574
|
+
|
|
18575
|
+
def absorb(conn) -> None:
|
|
18576
|
+
if applied and applied[0]:
|
|
18577
|
+
share.absorb_identity(conn, path)
|
|
18578
|
+
share.seal(conn)
|
|
18579
|
+
|
|
18580
|
+
try:
|
|
18581
|
+
admitted, result = _codex_admitted_cache_write(
|
|
18582
|
+
run, wait_for_requests=wait_for_requests, after_commit=absorb)
|
|
18583
|
+
except BaseException:
|
|
18584
|
+
share.invalidate()
|
|
18585
|
+
raise
|
|
18586
|
+
if not admitted or not result:
|
|
18587
|
+
share.invalidate()
|
|
18588
|
+
return admitted, result
|
|
18589
|
+
|
|
18590
|
+
|
|
16724
18591
|
def run_codex_history_provenance_drain(
|
|
16725
18592
|
*, wait_for_requests: bool = True) -> ProvenanceDrainResult:
|
|
16726
18593
|
"""Settle provenance and retained quota before historical raw ingestion.
|
|
@@ -16753,6 +18620,9 @@ def run_codex_history_provenance_drain(
|
|
|
16753
18620
|
for path, item in prepared.items()}
|
|
16754
18621
|
traced, high_water, quotas, decisions = _jr._codex_journal_history_evidence(
|
|
16755
18622
|
candidates, identities=identities.values(), capture=True)
|
|
18623
|
+
# #975 RC2: one committed-evidence seed for the whole pass, so each
|
|
18624
|
+
# anchor group is read once rather than once per identity.
|
|
18625
|
+
share = _CodexWalkShare()
|
|
16756
18626
|
for path in candidates:
|
|
16757
18627
|
item, identity = prepared[path], identities[path]
|
|
16758
18628
|
# Codex legacy normalization is independent of Claude's cutover
|
|
@@ -16761,7 +18631,7 @@ def run_codex_history_provenance_drain(
|
|
|
16761
18631
|
records = list(_jr._decoded_quota_stream(quotas.get(path, ()), None))
|
|
16762
18632
|
file_ops = decisions.get(identity, ())
|
|
16763
18633
|
|
|
16764
|
-
def apply(conn, path=path, item=item, identity=identity,
|
|
18634
|
+
def apply(conn, anchors, path=path, item=item, identity=identity,
|
|
16765
18635
|
records=records, file_ops=file_ops):
|
|
16766
18636
|
live = _qualify_codex_targets({path})
|
|
16767
18637
|
if (len(live) != 1
|
|
@@ -16774,7 +18644,8 @@ def run_codex_history_provenance_drain(
|
|
|
16774
18644
|
return False # another writer moved the captured population
|
|
16775
18645
|
before = conn.total_changes
|
|
16776
18646
|
_jr._apply_file_account_records(conn, file_ops)
|
|
16777
|
-
_jr._apply_quota_records(conn, records, quiet=True
|
|
18647
|
+
_jr._apply_quota_records(conn, records, quiet=True,
|
|
18648
|
+
anchors=anchors)
|
|
16778
18649
|
if conn.total_changes != before:
|
|
16779
18650
|
# Preserve triggers/change ledgers; no partial recovery
|
|
16780
18651
|
# mints journal coverage or leaves old coverage standing.
|
|
@@ -16782,8 +18653,8 @@ def run_codex_history_provenance_drain(
|
|
|
16782
18653
|
_bump_codex_physical_mutation_seq(conn)
|
|
16783
18654
|
return True
|
|
16784
18655
|
|
|
16785
|
-
admitted, applied =
|
|
16786
|
-
apply, wait_for_requests=wait_for_requests)
|
|
18656
|
+
admitted, applied = _codex_provenance_identity_write(
|
|
18657
|
+
share, path, apply, wait_for_requests=wait_for_requests)
|
|
16787
18658
|
if not admitted or not applied:
|
|
16788
18659
|
return ProvenanceDrainResult("yielded", len(candidates), len(traced))
|
|
16789
18660
|
# A changed inventory is retried from one new captured census; never
|
|
@@ -16819,6 +18690,511 @@ def run_codex_history_provenance_drain(
|
|
|
16819
18690
|
_release_worker_lock(worker)
|
|
16820
18691
|
|
|
16821
18692
|
|
|
18693
|
+
def _codex_replay_heal_due(conn: sqlite3.Connection) -> bool:
|
|
18694
|
+
"""An armed byte-zero replay or a replay heal record (#982 spec 4.6)."""
|
|
18695
|
+
if conn.execute("SELECT 1 FROM cache_meta WHERE key=? LIMIT 1",
|
|
18696
|
+
(CODEX_REPLAY_FROM_ZERO_KEY,)).fetchone() is not None:
|
|
18697
|
+
return True
|
|
18698
|
+
record = load_codex_heal_record(conn)
|
|
18699
|
+
return record is not None and record.get("mode") == "replay"
|
|
18700
|
+
|
|
18701
|
+
|
|
18702
|
+
class HealResult(NamedTuple):
|
|
18703
|
+
"""What one ``run_codex_admitted_heal`` pass did (#982 spec 4.1)."""
|
|
18704
|
+
|
|
18705
|
+
outcome: str # "done", "busy", "yielded", "contended", "degraded",
|
|
18706
|
+
# "refused", "replay_due", "restart" or "none"
|
|
18707
|
+
mode: "str | None"
|
|
18708
|
+
units: int # admitted targeted units: population and owed pass
|
|
18709
|
+
owed_units: int # of them, the owed pass's
|
|
18710
|
+
degraded: int # population files still owed after their attempts (U6)
|
|
18711
|
+
finalization_units: int
|
|
18712
|
+
certification_rounds: int # fenced and closing full rounds
|
|
18713
|
+
#: The last admitted sync's stats (the start unit's when it refused, the
|
|
18714
|
+
#: last walk piece's otherwise) with every work counter of the summary
|
|
18715
|
+
#: line (`_CODEX_HEAL_SUMMED_COUNTERS`) summed over every unit and piece
|
|
18716
|
+
#: of the pass, except the skipped count: the files the pass only found
|
|
18717
|
+
#: unchanged, each once. The failed and torn-deferred counts stay the
|
|
18718
|
+
#: last piece's. Each file is reported once, as the single walk does.
|
|
18719
|
+
#: ``None`` when no unit ran.
|
|
18720
|
+
stats: "CodexIngestStats | None" = None
|
|
18721
|
+
|
|
18722
|
+
|
|
18723
|
+
#: The work counters a heal pass reports summed over its units and pieces.
|
|
18724
|
+
_CODEX_HEAL_SUMMED_COUNTERS = (
|
|
18725
|
+
"files_processed", "files_skipped_unchanged", "files_reset_truncated",
|
|
18726
|
+
"rows_changed", "lines_malformed", "token_events_skipped")
|
|
18727
|
+
|
|
18728
|
+
|
|
18729
|
+
class _CodexHealYield(Exception):
|
|
18730
|
+
"""A fresh hook request made a non-waiting executor give up its pass."""
|
|
18731
|
+
|
|
18732
|
+
|
|
18733
|
+
class _CodexHealContended(Exception):
|
|
18734
|
+
"""Another writer held the store locks for longer than the replay worker
|
|
18735
|
+
has always waited for them (``CODEX_REPLAY_DRAIN_LOCK_TIMEOUT_SECONDS``)."""
|
|
18736
|
+
|
|
18737
|
+
|
|
18738
|
+
class _CodexHealRestart(Exception):
|
|
18739
|
+
"""A unit found its pass generation stale (spec 4.5a): the record was
|
|
18740
|
+
rewritten, or a fresh clear only a new pass's start unit may perform is
|
|
18741
|
+
due. The unit wrote nothing; the pass ends ``restart``."""
|
|
18742
|
+
|
|
18743
|
+
|
|
18744
|
+
def _codex_heal_owed_files(
|
|
18745
|
+
conn: sqlite3.Connection,
|
|
18746
|
+
files: "list[CodexDiscoveredFile]") -> "list[CodexDiscoveredFile]":
|
|
18747
|
+
"""The discovered files whose bytes ``_codex_backlog_after`` still owes:
|
|
18748
|
+
no cursor, an incomplete cursor, or a size beyond the cursor."""
|
|
18749
|
+
existing = _load_codex_session_files_rows(
|
|
18750
|
+
conn, [str(item.source_path) for item in files])
|
|
18751
|
+
return [item for item in files
|
|
18752
|
+
if _codex_backlog_after([item], existing)[0]]
|
|
18753
|
+
|
|
18754
|
+
|
|
18755
|
+
def run_codex_admitted_heal(
|
|
18756
|
+
*, wait_for_requests: bool = True,
|
|
18757
|
+
population: "list[str] | None" = None) -> HealResult:
|
|
18758
|
+
"""Run a Codex history heal as admitted per-file units (#982 spec 4.1).
|
|
18759
|
+
|
|
18760
|
+
Single-flight through ``codex-heal-walk.lock`` (busy returns ``busy``).
|
|
18761
|
+
No writer lock is held between units, and every unit is admitted the way
|
|
18762
|
+
``_apply_codex_verdict`` admits a verdict: the store is opened before
|
|
18763
|
+
admission, ``admit`` takes only the gate (the sync owns its locks, tried
|
|
18764
|
+
without waiting), and the sync rechecks for a fresh request once it holds
|
|
18765
|
+
them (``_after_locks``). A contended or rechecked unit changed nothing;
|
|
18766
|
+
with ``wait_for_requests`` the executor waits out every fresh request and
|
|
18767
|
+
retries the same unit, otherwise it returns ``yielded``. Store locks that
|
|
18768
|
+
another writer holds continuously for ``CODEX_REPLAY_DRAIN_LOCK_TIMEOUT_SECONDS``
|
|
18769
|
+
(the replay worker's lock wait) end the pass ``contended``.
|
|
18770
|
+
|
|
18771
|
+
History mode runs when ``population`` (the census a settled provenance
|
|
18772
|
+
pass just produced) is given or a history heal record exists, in order:
|
|
18773
|
+
|
|
18774
|
+
1. the start unit writes the record (phase ``units``: the fence);
|
|
18775
|
+
2. one targeted unit per population file, in discovery order;
|
|
18776
|
+
3. the owed pass: every discovered file still owed, at most twice each;
|
|
18777
|
+
4. the fenced walk: one full walk in admitted pieces (``_ingest_limit=1``,
|
|
18778
|
+
``_heal_census``, ``_heal_visited``, settlement deferred), each
|
|
18779
|
+
ingesting or resetting at most one file, until a piece leaves no
|
|
18780
|
+
census file unwalked; that piece makes the transition, deleting the
|
|
18781
|
+
record, whether or not a file failed (it stays owed);
|
|
18782
|
+
5. the #980 executor's unbudgeted finalization units until the debt is
|
|
18783
|
+
empty (a failed or idle unit hands the rest to ``_codex-prune-drain``);
|
|
18784
|
+
6. the closing walk: a second walk in pieces, with its own census and
|
|
18785
|
+
visited set and with ordinary settlement and sealing; its last piece
|
|
18786
|
+
certifies the walk.
|
|
18787
|
+
|
|
18788
|
+
A piece that neither completes its walk nor adds a census file to the
|
|
18789
|
+
visited set (only an existing refusal such as ``quota_index_pending``
|
|
18790
|
+
does) ends the pass as ``degraded``: the record stays, and the worker is
|
|
18791
|
+
requested again. Every other piece but the last adds a visited file, so a
|
|
18792
|
+
walk takes at most one piece more than its census has files.
|
|
18793
|
+
|
|
18794
|
+
Replay mode (spec 4.5) runs when ``CODEX_REPLAY_FROM_ZERO_KEY`` or a
|
|
18795
|
+
replay record exists, whatever ``population`` says. Its start unit
|
|
18796
|
+
(``_replay_stage="start"``) captures the D1 snapshot, clears and writes
|
|
18797
|
+
the record (population: every discovered file), then rehydrates; a
|
|
18798
|
+
resumed pass whose record is not yet ``rehydrated`` reruns it, which
|
|
18799
|
+
rehydrates without clearing again. A refused start (``rebuild_refused``
|
|
18800
|
+
or a prune refusal) clears nothing and returns ``refused``. Steps 2-6
|
|
18801
|
+
follow exactly as in history mode, except that the fenced walk's last
|
|
18802
|
+
piece ends the record through the unchanged consumption block: deleted
|
|
18803
|
+
with the marker, or moved to phase ``certify`` when the replay is
|
|
18804
|
+
blocked (a failed or torn-deferred file). A ``certify`` record resumes at
|
|
18805
|
+
step 5: its fence is over, and the closing walk retries the blocked file
|
|
18806
|
+
and consumes the marker once it walks clean.
|
|
18807
|
+
|
|
18808
|
+
Generation (spec 4.5a): every unit after the start unit carries the
|
|
18809
|
+
``gen`` of the record the pass started with (``_heal_generation``), and
|
|
18810
|
+
each finalization unit checks it under its store locks too. A unit that
|
|
18811
|
+
finds another ``gen``, a ``reclear`` mark or an armed marker without a
|
|
18812
|
+
replay record writes nothing; the pass then ends ``restart``, discarding
|
|
18813
|
+
its census, visited set and cached payload, and requests the worker under
|
|
18814
|
+
the short retry throttle. Only the next pass's start unit performs the
|
|
18815
|
+
fresh clear, so no piece skips a file it visited before that clear.
|
|
18816
|
+
"""
|
|
18817
|
+
import _lib_codex_drain_admission as adm
|
|
18818
|
+
worker = _worker_lock(CODEX_HEAL_WALK_LOCK_NAME)
|
|
18819
|
+
if worker is None:
|
|
18820
|
+
return HealResult("busy", None, 0, 0, 0, 0, 0)
|
|
18821
|
+
counts = {"units": 0, "owed": 0, "degraded": 0, "final": 0, "rounds": 0}
|
|
18822
|
+
summed = collections.Counter()
|
|
18823
|
+
last: dict = {"stats": None, "mode": "history"}
|
|
18824
|
+
# The summary reports each file at most once per pass, as the single
|
|
18825
|
+
# walk does: a file any unit processed (or reset) counts as processed
|
|
18826
|
+
# only, and one the pass only found unchanged counts as skipped once,
|
|
18827
|
+
# however many walks re-find it. A piece's skips of visited and
|
|
18828
|
+
# out-of-census files stay in its own `files_skipped_unchanged` (the
|
|
18829
|
+
# barrier census sum needs them) and never reach the pass's count.
|
|
18830
|
+
pass_files: "dict[str, set[str]]" = {"processed": set(), "unchanged": set()}
|
|
18831
|
+
# A worker requested by this pass (`contended`, `degraded`) is spawned
|
|
18832
|
+
# after the heal lock is released: `_defer_codex_replay_drain` spawns
|
|
18833
|
+
# nothing while a pass holds it.
|
|
18834
|
+
worker_owed = {"replay": False}
|
|
18835
|
+
|
|
18836
|
+
def request_worker() -> None:
|
|
18837
|
+
worker_owed["replay"] = True
|
|
18838
|
+
|
|
18839
|
+
def count_pass_files(only_paths, stats: CodexIngestStats) -> None:
|
|
18840
|
+
if only_paths is not None:
|
|
18841
|
+
if stats.files_processed or stats.files_reset_truncated:
|
|
18842
|
+
pass_files["processed"].update(only_paths)
|
|
18843
|
+
elif stats.files_skipped_unchanged:
|
|
18844
|
+
pass_files["unchanged"].update(only_paths)
|
|
18845
|
+
return
|
|
18846
|
+
counted = set(stats.heal_counted_paths)
|
|
18847
|
+
pass_files["processed"] |= counted
|
|
18848
|
+
pass_files["unchanged"] |= set(stats.heal_examined) - counted
|
|
18849
|
+
|
|
18850
|
+
def result(outcome: str, mode: "str | None" = None) -> HealResult:
|
|
18851
|
+
stats = last["stats"]
|
|
18852
|
+
if stats is not None:
|
|
18853
|
+
stats = _dc_replace(stats, **{
|
|
18854
|
+
**{name: summed[name] for name in _CODEX_HEAL_SUMMED_COUNTERS},
|
|
18855
|
+
"files_skipped_unchanged": len(
|
|
18856
|
+
pass_files["unchanged"] - pass_files["processed"])})
|
|
18857
|
+
return HealResult(outcome, last["mode"] if mode is None else mode,
|
|
18858
|
+
counts["units"], counts["owed"],
|
|
18859
|
+
counts["degraded"], counts["final"],
|
|
18860
|
+
counts["rounds"], stats)
|
|
18861
|
+
|
|
18862
|
+
share = _CodexWalkShare()
|
|
18863
|
+
# The generation of the record this pass works on (spec 4.5a), set once
|
|
18864
|
+
# the start unit (or the probe of a resumed record) has it.
|
|
18865
|
+
pass_state: dict = {"gen": None}
|
|
18866
|
+
|
|
18867
|
+
def recheck() -> bool:
|
|
18868
|
+
return bool(adm.fresh_requests(_cctally_core.APP_DIR))
|
|
18869
|
+
|
|
18870
|
+
def admitted_sync(kind: str, **kwargs) -> CodexIngestStats:
|
|
18871
|
+
# The store locks held by another writer (not a hook request, which
|
|
18872
|
+
# expires) are waited for as long as the replay worker's single sync
|
|
18873
|
+
# always waited for them, then the pass gives up as `contended`.
|
|
18874
|
+
contended_since: "float | None" = None
|
|
18875
|
+
while True:
|
|
18876
|
+
stats = None
|
|
18877
|
+
conn = open_cache_db()
|
|
18878
|
+
try:
|
|
18879
|
+
admission = adm.admit(_cctally_core.APP_DIR,
|
|
18880
|
+
lambda: "sync-owns-locks",
|
|
18881
|
+
lambda token: None)
|
|
18882
|
+
if admission.admitted:
|
|
18883
|
+
try:
|
|
18884
|
+
with _perf.phase("codex_heal_unit") as _p_unit:
|
|
18885
|
+
_p_unit.set_meta(kind=kind)
|
|
18886
|
+
if kind != "start":
|
|
18887
|
+
kwargs["_heal_generation"] = pass_state["gen"]
|
|
18888
|
+
stats = sync_codex_cache(
|
|
18889
|
+
conn, maintenance="caller", lock_timeout=0,
|
|
18890
|
+
_walk_share=share, _after_locks=recheck,
|
|
18891
|
+
**kwargs)
|
|
18892
|
+
finally:
|
|
18893
|
+
adm.finish(admission, lambda token: None)
|
|
18894
|
+
finally:
|
|
18895
|
+
conn.close()
|
|
18896
|
+
if stats is not None and stats.heal_generation_stale:
|
|
18897
|
+
raise _CodexHealRestart()
|
|
18898
|
+
if stats is not None and not stats.lock_contended:
|
|
18899
|
+
last["stats"] = stats
|
|
18900
|
+
for name in _CODEX_HEAL_SUMMED_COUNTERS:
|
|
18901
|
+
summed[name] += int(getattr(stats, name) or 0)
|
|
18902
|
+
count_pass_files(kwargs.get("only_paths"), stats)
|
|
18903
|
+
return stats
|
|
18904
|
+
if not wait_for_requests:
|
|
18905
|
+
raise _CodexHealYield()
|
|
18906
|
+
if (stats is not None and not stats.drain_yielded
|
|
18907
|
+
and not adm.fresh_requests(_cctally_core.APP_DIR)):
|
|
18908
|
+
now = time.monotonic()
|
|
18909
|
+
if contended_since is None:
|
|
18910
|
+
contended_since = now
|
|
18911
|
+
if now - contended_since >= CODEX_REPLAY_DRAIN_LOCK_TIMEOUT_SECONDS:
|
|
18912
|
+
raise _CodexHealContended()
|
|
18913
|
+
else:
|
|
18914
|
+
# An admission refusal, a drain yield, or a raw refusal met
|
|
18915
|
+
# while a request (a Codex hook's, or the Claude waiter,
|
|
18916
|
+
# #1055 spec 4.3) is fresh: a deliberate wait, not continuous
|
|
18917
|
+
# contention.
|
|
18918
|
+
contended_since = None
|
|
18919
|
+
_drain_backoff(adm)
|
|
18920
|
+
|
|
18921
|
+
def run_finalization_units() -> None:
|
|
18922
|
+
"""Step 5: unbudgeted finalization units until the debt is empty.
|
|
18923
|
+
Store locks another writer holds continuously are waited for as long
|
|
18924
|
+
as every other unit waits for them, then the pass is `contended`."""
|
|
18925
|
+
contended_since: "float | None" = None
|
|
18926
|
+
while True:
|
|
18927
|
+
conn = open_cache_db()
|
|
18928
|
+
try:
|
|
18929
|
+
if load_codex_finalization_debt(conn).empty:
|
|
18930
|
+
break
|
|
18931
|
+
conn.rollback()
|
|
18932
|
+
admission = adm.admit(_cctally_core.APP_DIR, _cache_store_locks,
|
|
18933
|
+
_release_cache_store_locks)
|
|
18934
|
+
if not admission.admitted:
|
|
18935
|
+
if not wait_for_requests:
|
|
18936
|
+
raise _CodexHealYield()
|
|
18937
|
+
if adm.fresh_requests(_cctally_core.APP_DIR):
|
|
18938
|
+
contended_since = None
|
|
18939
|
+
else:
|
|
18940
|
+
now = time.monotonic()
|
|
18941
|
+
if contended_since is None:
|
|
18942
|
+
contended_since = now
|
|
18943
|
+
if (now - contended_since
|
|
18944
|
+
>= CODEX_REPLAY_DRAIN_LOCK_TIMEOUT_SECONDS):
|
|
18945
|
+
raise _CodexHealContended()
|
|
18946
|
+
_drain_backoff(adm)
|
|
18947
|
+
continue
|
|
18948
|
+
contended_since = None
|
|
18949
|
+
try:
|
|
18950
|
+
# Spec 4.5a: under the store locks, the unit belongs to
|
|
18951
|
+
# this pass's record generation or writes nothing.
|
|
18952
|
+
if _codex_heal_generation_stale(
|
|
18953
|
+
load_codex_heal_record(conn),
|
|
18954
|
+
conn.execute(
|
|
18955
|
+
"SELECT 1 FROM cache_meta WHERE key=? LIMIT 1",
|
|
18956
|
+
(CODEX_REPLAY_FROM_ZERO_KEY,)).fetchone()
|
|
18957
|
+
is not None,
|
|
18958
|
+
pass_state["gen"]):
|
|
18959
|
+
raise _CodexHealRestart()
|
|
18960
|
+
with _perf.phase("codex_heal_unit") as _p_unit:
|
|
18961
|
+
_p_unit.set_meta(kind="finalization")
|
|
18962
|
+
ok, model_cleared = _run_codex_executor_unit(
|
|
18963
|
+
conn, "finalization")
|
|
18964
|
+
except BaseException:
|
|
18965
|
+
conn.rollback()
|
|
18966
|
+
raise
|
|
18967
|
+
finally:
|
|
18968
|
+
adm.finish(admission, _release_cache_store_locks)
|
|
18969
|
+
finally:
|
|
18970
|
+
conn.close()
|
|
18971
|
+
if model_cleared:
|
|
18972
|
+
try:
|
|
18973
|
+
from _cctally_quota import _defer_codex_quota_verification
|
|
18974
|
+
_defer_codex_quota_verification()
|
|
18975
|
+
except Exception:
|
|
18976
|
+
pass
|
|
18977
|
+
if not ok:
|
|
18978
|
+
# The prune drain's executor owns this debt with its
|
|
18979
|
+
# `blocked` rule; the closing round's settlement retries too.
|
|
18980
|
+
_request_codex_drain_worker(CODEX_PRUNE_DRAIN_COMMAND)
|
|
18981
|
+
break
|
|
18982
|
+
counts["final"] += 1
|
|
18983
|
+
|
|
18984
|
+
def probe_state() -> "tuple[dict | None, bool]":
|
|
18985
|
+
"""The record's fence key and whether the replay marker is armed."""
|
|
18986
|
+
conn = open_cache_db()
|
|
18987
|
+
try:
|
|
18988
|
+
return (load_codex_heal_record(conn), conn.execute(
|
|
18989
|
+
"SELECT 1 FROM cache_meta WHERE key=? LIMIT 1",
|
|
18990
|
+
(CODEX_REPLAY_FROM_ZERO_KEY,)).fetchone() is not None)
|
|
18991
|
+
finally:
|
|
18992
|
+
conn.close()
|
|
18993
|
+
|
|
18994
|
+
try:
|
|
18995
|
+
record, replay = probe_state()
|
|
18996
|
+
replay_mode = replay or (record is not None
|
|
18997
|
+
and record.get("mode") == "replay")
|
|
18998
|
+
if replay_mode:
|
|
18999
|
+
last["mode"] = "replay"
|
|
19000
|
+
if (record is not None and record.get("mode") == "replay"
|
|
19001
|
+
and record.get("malformed")):
|
|
19002
|
+
# Spec 4.5a: a replay record whose D1 set cannot be read is
|
|
19003
|
+
# never cleared again (every writer declines); nothing here
|
|
19004
|
+
# can make progress without it.
|
|
19005
|
+
return result("degraded")
|
|
19006
|
+
# 1. The replay's start unit: the clear (which writes the record)
|
|
19007
|
+
# and the rehydration. Rerun on a resumed record that never
|
|
19008
|
+
# rehydrated; it clears nothing once the record exists, unless a
|
|
19009
|
+
# later arming site re-armed it (`reclear`, spec 4.5a).
|
|
19010
|
+
if (record is None or record.get("mode") != "replay"
|
|
19011
|
+
or not record.get("rehydrated")
|
|
19012
|
+
or _codex_replay_reclear_due(record, replay)):
|
|
19013
|
+
start_stats = admitted_sync("start", _replay_stage="start")
|
|
19014
|
+
record, replay = probe_state()
|
|
19015
|
+
if record is None or record.get("mode") != "replay":
|
|
19016
|
+
if not replay:
|
|
19017
|
+
return result("none") # consumed by another writer
|
|
19018
|
+
if start_stats.rebuild_refused or start_stats.prune_refused:
|
|
19019
|
+
return result("refused")
|
|
19020
|
+
request_worker()
|
|
19021
|
+
return result("degraded")
|
|
19022
|
+
if record.get("malformed"):
|
|
19023
|
+
return result("degraded")
|
|
19024
|
+
if _codex_replay_reclear_due(record, replay):
|
|
19025
|
+
# The fresh clear did not run (refused or deferred).
|
|
19026
|
+
if start_stats.rebuild_refused or start_stats.prune_refused:
|
|
19027
|
+
return result("refused")
|
|
19028
|
+
request_worker()
|
|
19029
|
+
return result("degraded")
|
|
19030
|
+
if not record.get("rehydrated"):
|
|
19031
|
+
# The rehydration failed and rolled back; a later
|
|
19032
|
+
# worker reruns it before any rollout is read.
|
|
19033
|
+
request_worker()
|
|
19034
|
+
return result("degraded")
|
|
19035
|
+
else:
|
|
19036
|
+
if record is None and not population:
|
|
19037
|
+
return result("none", None)
|
|
19038
|
+
if record is not None and record.get("phase") != "units":
|
|
19039
|
+
return result("none")
|
|
19040
|
+
if record is not None:
|
|
19041
|
+
pass_state["gen"] = record.get("gen")
|
|
19042
|
+
|
|
19043
|
+
# 1. The start unit.
|
|
19044
|
+
if record is None:
|
|
19045
|
+
fresh_population = sorted(set(str(p) for p in population))
|
|
19046
|
+
|
|
19047
|
+
def start(conn):
|
|
19048
|
+
current = load_codex_heal_record(conn)
|
|
19049
|
+
if current is not None:
|
|
19050
|
+
return current
|
|
19051
|
+
written = _store_codex_heal_record(
|
|
19052
|
+
conn, {"mode": "history", "phase": "units"},
|
|
19053
|
+
population=fresh_population)
|
|
19054
|
+
share.remember_heal_payload(CodexHealPayload(
|
|
19055
|
+
written["gen"], frozenset(fresh_population), None))
|
|
19056
|
+
return written
|
|
19057
|
+
|
|
19058
|
+
admitted, record = _codex_admitted_cache_write(
|
|
19059
|
+
start, wait_for_requests=wait_for_requests,
|
|
19060
|
+
contention_bound=CODEX_REPLAY_DRAIN_LOCK_TIMEOUT_SECONDS)
|
|
19061
|
+
if not admitted:
|
|
19062
|
+
return result("yielded")
|
|
19063
|
+
if record.get("mode") != "history" or record.get("phase") != "units":
|
|
19064
|
+
return result("none", record.get("mode"))
|
|
19065
|
+
pass_state["gen"] = record.get("gen")
|
|
19066
|
+
|
|
19067
|
+
def walk_in_pieces(kind: str, **kwargs) -> bool:
|
|
19068
|
+
"""Steps 4 and 7: one full walk in admitted pieces. True when a
|
|
19069
|
+
piece left no census file unwalked; False when a piece neither
|
|
19070
|
+
completed the walk nor visited a file (no progress). Each
|
|
19071
|
+
visited file's committed row is kept for the last piece's
|
|
19072
|
+
revalidation (revision 7)."""
|
|
19073
|
+
census: "frozenset[str] | None" = None
|
|
19074
|
+
visited: "dict[str, tuple | None]" = {}
|
|
19075
|
+
while True:
|
|
19076
|
+
stats = admitted_sync(
|
|
19077
|
+
kind, _ingest_limit=1, _heal_census=census,
|
|
19078
|
+
_heal_visited=frozenset(visited),
|
|
19079
|
+
_heal_visited_rows=dict(visited), **kwargs)
|
|
19080
|
+
counts["rounds"] += 1
|
|
19081
|
+
if census is None and stats.heal_discovered is not None:
|
|
19082
|
+
census = stats.heal_discovered
|
|
19083
|
+
added = set(stats.heal_examined) - set(visited)
|
|
19084
|
+
for path in added:
|
|
19085
|
+
visited[path] = stats.heal_visit_rows.get(path)
|
|
19086
|
+
if stats.heal_walk_complete:
|
|
19087
|
+
return True
|
|
19088
|
+
if not added:
|
|
19089
|
+
return False
|
|
19090
|
+
|
|
19091
|
+
if record.get("phase") == "units":
|
|
19092
|
+
# The pass's one parse of the population (and, in replay mode,
|
|
19093
|
+
# the D1 set the units then take from the share).
|
|
19094
|
+
conn = open_cache_db()
|
|
19095
|
+
try:
|
|
19096
|
+
payload = share.heal_payload(conn, record)
|
|
19097
|
+
finally:
|
|
19098
|
+
conn.close()
|
|
19099
|
+
if payload is None and last["mode"] == "replay":
|
|
19100
|
+
return result("degraded") # unreadable D1 set: fail closed
|
|
19101
|
+
members = (frozenset() if payload is None
|
|
19102
|
+
else payload.population)
|
|
19103
|
+
|
|
19104
|
+
# 2. Population units, in the single walk's (discovery) order.
|
|
19105
|
+
attempts: "collections.Counter[str]" = collections.Counter()
|
|
19106
|
+
for item in _discover_codex_files_with_roots():
|
|
19107
|
+
path = str(item.source_path)
|
|
19108
|
+
if path not in members:
|
|
19109
|
+
continue
|
|
19110
|
+
attempts[path] += 1
|
|
19111
|
+
admitted_sync("population", only_paths={path})
|
|
19112
|
+
counts["units"] += 1
|
|
19113
|
+
|
|
19114
|
+
# 3. The owed pass: appearances after the census, tails another
|
|
19115
|
+
# writer left, and population files a targeted unit declined.
|
|
19116
|
+
while True:
|
|
19117
|
+
files = _discover_codex_files_with_roots()
|
|
19118
|
+
conn = open_cache_db()
|
|
19119
|
+
try:
|
|
19120
|
+
owed = _codex_heal_owed_files(conn, files)
|
|
19121
|
+
finally:
|
|
19122
|
+
conn.close()
|
|
19123
|
+
todo = [item for item in owed
|
|
19124
|
+
if attempts[str(item.source_path)] < 2]
|
|
19125
|
+
if not todo:
|
|
19126
|
+
# Only population files: a file that appeared after the
|
|
19127
|
+
# census is a later sync's, as for the single walk.
|
|
19128
|
+
counts["degraded"] = sum(
|
|
19129
|
+
1 for item in owed if str(item.source_path) in members)
|
|
19130
|
+
break
|
|
19131
|
+
for item in todo:
|
|
19132
|
+
path = str(item.source_path)
|
|
19133
|
+
attempts[path] += 1
|
|
19134
|
+
admitted_sync("owed", only_paths={path})
|
|
19135
|
+
counts["units"] += 1
|
|
19136
|
+
counts["owed"] += 1
|
|
19137
|
+
|
|
19138
|
+
# Spec 4.5a: a replay armed meanwhile (or re-armed over this
|
|
19139
|
+
# replay) makes every piece below clear and refill one file;
|
|
19140
|
+
# end the pass instead, without a transition. The next pass
|
|
19141
|
+
# starts in replay mode, whose clear retires a history record.
|
|
19142
|
+
current, armed = probe_state()
|
|
19143
|
+
if ((last["mode"] == "history" and armed)
|
|
19144
|
+
or _codex_replay_reclear_due(current, armed)):
|
|
19145
|
+
return result("replay_due")
|
|
19146
|
+
|
|
19147
|
+
# 4. The fenced walk, whose last piece makes the transition
|
|
19148
|
+
# (replay: the unchanged consumption block deletes the record
|
|
19149
|
+
# with the marker, or moves it to `certify` when blocked).
|
|
19150
|
+
walk_in_pieces("certification", _defer_finalization=True,
|
|
19151
|
+
_heal_transition=True)
|
|
19152
|
+
conn = open_cache_db()
|
|
19153
|
+
try:
|
|
19154
|
+
current = load_codex_heal_record(conn)
|
|
19155
|
+
finally:
|
|
19156
|
+
conn.close()
|
|
19157
|
+
if current is not None and current.get("phase") == "units":
|
|
19158
|
+
# No progress (or a complete walk that could not end the
|
|
19159
|
+
# record): the fence stays, and every hook keeps requesting
|
|
19160
|
+
# the worker while the record exists. Without a record (or
|
|
19161
|
+
# with a `certify` one), the last piece made the transition
|
|
19162
|
+
# or another walk (a rebuild) retired it meanwhile.
|
|
19163
|
+
request_worker()
|
|
19164
|
+
return result("degraded")
|
|
19165
|
+
|
|
19166
|
+
# 5. Finalization units of the #980 executor, now unfenced.
|
|
19167
|
+
run_finalization_units()
|
|
19168
|
+
|
|
19169
|
+
# 6. The closing walk: ordinary settlement; it certifies the walk.
|
|
19170
|
+
# The fence is gone, so a refusal here is the ordinary syncs' to
|
|
19171
|
+
# retry (the refusing piece requested its own worker).
|
|
19172
|
+
if not walk_in_pieces("closing"):
|
|
19173
|
+
return result("degraded")
|
|
19174
|
+
return result("done")
|
|
19175
|
+
except _CodexHealYield:
|
|
19176
|
+
return result("yielded")
|
|
19177
|
+
except _CodexHealContended:
|
|
19178
|
+
# The record (and with it the fence) stays: request the worker, which
|
|
19179
|
+
# the record's short retry throttle lets through within minutes.
|
|
19180
|
+
request_worker()
|
|
19181
|
+
return result("contended")
|
|
19182
|
+
except _CodexHealRestart:
|
|
19183
|
+
# Spec 4.5a: the census and visited set died with the walk's frame;
|
|
19184
|
+
# the cached payload is another generation's. The next pass starts
|
|
19185
|
+
# over from its own start unit (the fresh clear, when one is due).
|
|
19186
|
+
share.forget_heal_payload()
|
|
19187
|
+
# Spawns nothing while this pass holds the heal lock (every caller
|
|
19188
|
+
# re-runs the executor inline on `restart`), which keeps a re-run and
|
|
19189
|
+
# a spawned worker from racing for the lock.
|
|
19190
|
+
_defer_codex_replay_drain(short=True)
|
|
19191
|
+
return result("restart")
|
|
19192
|
+
finally:
|
|
19193
|
+
_release_worker_lock(worker)
|
|
19194
|
+
if worker_owed["replay"]:
|
|
19195
|
+
_request_codex_drain_worker(CODEX_REPLAY_DRAIN_COMMAND)
|
|
19196
|
+
|
|
19197
|
+
|
|
16822
19198
|
class _CodexFileBaselineResolver:
|
|
16823
19199
|
"""Per-row ``codex_file_accounts`` baselines, memoised per file.
|
|
16824
19200
|
|
|
@@ -16916,6 +19292,10 @@ def reconcile_codex_window_attribution_spend(
|
|
|
16916
19292
|
"""
|
|
16917
19293
|
from _cctally_quota import resolve_codex_window_attributions_with_evidence
|
|
16918
19294
|
|
|
19295
|
+
# #982 spec 4.3: `account attribute` calls this directly, so the kernel
|
|
19296
|
+
# checks the heal fence itself (strict: `CodexHealFenced`).
|
|
19297
|
+
if _codex_heal_fenced_kernel(conn, strict):
|
|
19298
|
+
return (0, 0)
|
|
16919
19299
|
try:
|
|
16920
19300
|
with _codex_cancellable_read(conn, deadline, clock):
|
|
16921
19301
|
active = load_active_window_attributions(conn)
|
|
@@ -21355,6 +23735,7 @@ def _claude_batch_committed(conn, released, sessions, stats, touched_sessions,
|
|
|
21355
23735
|
_CONVERSATION_RESIDUAL_FIELDS = frozenset({
|
|
21356
23736
|
"failed_paths", "barrier_deferred_keys", "barrier_deferred_targets",
|
|
21357
23737
|
"barrier_published_paths", "barrier_retry_causes",
|
|
23738
|
+
"barrier_terminal_keys", # #1052
|
|
21358
23739
|
})
|
|
21359
23740
|
|
|
21360
23741
|
|
|
@@ -22125,6 +24506,32 @@ def _cache_side_replay_pending(conn: sqlite3.Connection) -> bool:
|
|
|
22125
24506
|
return False
|
|
22126
24507
|
|
|
22127
24508
|
|
|
24509
|
+
def _cache_side_history_heal_pending(conn: sqlite3.Connection) -> bool:
|
|
24510
|
+
"""Whether cache.db holds a history heal fence (#982 spec 4.3, R10).
|
|
24511
|
+
|
|
24512
|
+
Per-file heal units commit one population file at a time, and a
|
|
24513
|
+
cursorless contributor's membership is unknown until its own unit reads
|
|
24514
|
+
it, so the barrier cannot hold a conversation whose other contributor is
|
|
24515
|
+
still to come. The pass therefore publishes nothing while the fence holds;
|
|
24516
|
+
the heal's transition ends it once every population file is accounted.
|
|
24517
|
+
Qualified through ``cache_db`` like ``_cache_side_replay_pending``, and a
|
|
24518
|
+
connection without the attachment reports False.
|
|
24519
|
+
"""
|
|
24520
|
+
try:
|
|
24521
|
+
row = conn.execute(
|
|
24522
|
+
"SELECT value FROM cache_db.cache_meta WHERE key=?",
|
|
24523
|
+
(CODEX_HEAL_WALK_KEY,)).fetchone()
|
|
24524
|
+
except sqlite3.OperationalError:
|
|
24525
|
+
return False
|
|
24526
|
+
if row is None:
|
|
24527
|
+
return False
|
|
24528
|
+
try:
|
|
24529
|
+
record = json.loads(str(row[0]))
|
|
24530
|
+
except (TypeError, ValueError):
|
|
24531
|
+
return True # malformed fails closed, as the fence
|
|
24532
|
+
return not (isinstance(record, dict) and record.get("phase") != "units")
|
|
24533
|
+
|
|
24534
|
+
|
|
22128
24535
|
def _clear_codex_conversation_store(conn: sqlite3.Connection) -> None:
|
|
22129
24536
|
"""Clear only the re-derivable Codex transcript families."""
|
|
22130
24537
|
conn.execute("DELETE FROM codex_conversation_events")
|
|
@@ -22201,6 +24608,75 @@ def _codex_conversation_cursor_row(conn, path: str):
|
|
|
22201
24608
|
return None if row is None else tuple(row[1:])
|
|
22202
24609
|
|
|
22203
24610
|
|
|
24611
|
+
def _codex_conversation_accepted(row) -> _AcceptedContainer:
|
|
24612
|
+
"""The container evidence of a conversations cursor row (``row[1:]``)."""
|
|
24613
|
+
return _accepted_container(row, ext=14, mtime=1, device=12, inode=13, size=0)
|
|
24614
|
+
|
|
24615
|
+
|
|
24616
|
+
def _codex_cursor_unverified(row) -> bool:
|
|
24617
|
+
"""A conversations cursor row whose state is explicitly ``unverified``."""
|
|
24618
|
+
return row is not None and _codex_conversation_accepted(row).state == "unverified"
|
|
24619
|
+
|
|
24620
|
+
|
|
24621
|
+
_CURSOR_UNREAD = object()
|
|
24622
|
+
|
|
24623
|
+
|
|
24624
|
+
def _classify_codex_unit_file(conn, discovered, *, executor_paths,
|
|
24625
|
+
protected_paths, requal_held, unit, targeted,
|
|
24626
|
+
cursor=_CURSOR_UNREAD) -> str:
|
|
24627
|
+
"""``"blocked"``, ``"terminal"`` or ``"quiet"`` for one unit file (#1052
|
|
24628
|
+
spec 5.1), by the accessors and in the order of the conversation loop's
|
|
24629
|
+
own checks. Never decodes: a cursorless compressed container, whose
|
|
24630
|
+
decision would validate it, is blocked before the decision is reached,
|
|
24631
|
+
and an executor path, the only other decoding entrance, is blocked first.
|
|
24632
|
+
|
|
24633
|
+
Terminal is the conversations store's terminal verdict exactly: an
|
|
24634
|
+
explicitly ``unverified`` cursor whose decision is the quiet
|
|
24635
|
+
``skip/verification-unverified``. The decision maps every non-accepted,
|
|
24636
|
+
non-pending state to that reason, so the reason alone is not enough.
|
|
24637
|
+
|
|
24638
|
+
Quiet is a ``read`` decision the loop would not refuse before reading a
|
|
24639
|
+
byte of it. The loop refuses two such files, and so does this (#1052
|
|
24640
|
+
review F2): in a targeted pass, a retained cursor the file no longer
|
|
24641
|
+
continues (truncated, replaced, or under another root), and, in ``unit``,
|
|
24642
|
+
a certificate that names another generation of the file. ``cursor`` is
|
|
24643
|
+
the row the caller already read, if any."""
|
|
24644
|
+
path_str = str(discovered.source_path)
|
|
24645
|
+
if path_str in protected_paths or path_str in executor_paths:
|
|
24646
|
+
return "blocked"
|
|
24647
|
+
selected = _select_codex_container(discovered.source_path)
|
|
24648
|
+
if selected is None:
|
|
24649
|
+
return "blocked"
|
|
24650
|
+
prev = (_codex_conversation_cursor_row(conn, path_str)
|
|
24651
|
+
if cursor is _CURSOR_UNREAD else cursor)
|
|
24652
|
+
if requal_held and _codex_held_by_requalification(
|
|
24653
|
+
requal_held, frozenset(), discovered, selected.location,
|
|
24654
|
+
first_sight=prev is None):
|
|
24655
|
+
return "blocked"
|
|
24656
|
+
if prev is None and selected.kind == _rc.ZSTD:
|
|
24657
|
+
return "blocked"
|
|
24658
|
+
decision = _codex_container_decision(
|
|
24659
|
+
selected, None if prev is None else _codex_conversation_accepted(prev),
|
|
24660
|
+
executor_path=False, should_stop=None)
|
|
24661
|
+
if decision.action == "read":
|
|
24662
|
+
fp = selected.fingerprint
|
|
24663
|
+
size = int(decision.size)
|
|
24664
|
+
if targeted and prev is not None and (
|
|
24665
|
+
size < prev[0] or prev[3] != discovered.source_root_key
|
|
24666
|
+
or _ingest_frontier.source_identity_replaced(
|
|
24667
|
+
prev[12], prev[13], fp.device, fp.inode)):
|
|
24668
|
+
return "blocked"
|
|
24669
|
+
if unit is not None and unit.cap_for(path_str, _rc.generation_token(
|
|
24670
|
+
fp, size if selected.kind == _rc.ZSTD else None)) is None:
|
|
24671
|
+
return "blocked"
|
|
24672
|
+
return "quiet"
|
|
24673
|
+
if (decision.action == "skip"
|
|
24674
|
+
and decision.reason == "verification-unverified"
|
|
24675
|
+
and _codex_cursor_unverified(prev)):
|
|
24676
|
+
return "terminal"
|
|
24677
|
+
return "blocked"
|
|
24678
|
+
|
|
24679
|
+
|
|
22204
24680
|
def _codex_conversation_population_subset(conn, keys) -> "set[str]":
|
|
22205
24681
|
found: set[str] = set()
|
|
22206
24682
|
for chunk in _codex_barrier_chunks({k for k in keys if k}):
|
|
@@ -22291,15 +24767,24 @@ class _CodexPublicationWork:
|
|
|
22291
24767
|
files as ``(discovered, unit)`` and ``(None, unit)`` (the loop commits the
|
|
22292
24768
|
unit). Planning captures one read snapshot between the two phases and
|
|
22293
24769
|
releases it before any publication transaction begins.
|
|
24770
|
+
|
|
24771
|
+
#1052 spec 5.1: each planned unit is classified before any of its files
|
|
24772
|
+
is yielded. A terminal deferral (a terminal contributor and no blocked
|
|
24773
|
+
file) yields nothing, so nothing of it is read, decoded or published.
|
|
22294
24774
|
"""
|
|
22295
24775
|
|
|
22296
|
-
def __init__(self, conn, files, *, only_paths, stats
|
|
24776
|
+
def __init__(self, conn, files, *, only_paths, stats,
|
|
24777
|
+
executor_paths=frozenset(), protected_paths=frozenset(),
|
|
24778
|
+
requal_held=None):
|
|
22297
24779
|
self.conn = conn
|
|
22298
24780
|
self.files = list(files)
|
|
22299
24781
|
self.files_by_path = {str(d.source_path): d for d in self.files}
|
|
22300
24782
|
self.only_paths = (None if only_paths is None
|
|
22301
24783
|
else {str(path) for path in only_paths})
|
|
22302
24784
|
self.stats = stats
|
|
24785
|
+
self.executor_paths = frozenset(executor_paths)
|
|
24786
|
+
self.protected_paths = frozenset(protected_paths)
|
|
24787
|
+
self.requal_held = requal_held or {}
|
|
22303
24788
|
self.population_present = conn.execute(
|
|
22304
24789
|
"SELECT 1 FROM cache_db.codex_barrier_population LIMIT 1"
|
|
22305
24790
|
).fetchone() is not None
|
|
@@ -22314,10 +24799,49 @@ class _CodexPublicationWork:
|
|
|
22314
24799
|
yield None, None
|
|
22315
24800
|
self.units = _plan_codex_publication_units(self.conn, self)
|
|
22316
24801
|
for unit in self.units:
|
|
24802
|
+
terminal = self._terminal_paths(unit)
|
|
24803
|
+
if terminal:
|
|
24804
|
+
# #1052 spec 5.2: deferred, never retried. A retry flag raised
|
|
24805
|
+
# elsewhere is neither set nor cleared here.
|
|
24806
|
+
unit.failed = True
|
|
24807
|
+
self.stats.barrier_keys_deferred += len(unit.keys)
|
|
24808
|
+
self.stats.barrier_keys_terminal += len(unit.keys)
|
|
24809
|
+
named = self.stats.barrier_terminal_keys
|
|
24810
|
+
for path in terminal:
|
|
24811
|
+
named[path] = named.get(path, frozenset()) | frozenset(unit.keys)
|
|
24812
|
+
continue
|
|
22317
24813
|
for discovered in unit.files:
|
|
22318
24814
|
yield discovered, unit
|
|
22319
24815
|
yield None, unit
|
|
22320
24816
|
|
|
24817
|
+
def _terminal_paths(self, unit) -> "tuple[str, ...]":
|
|
24818
|
+
"""The terminal files of a terminal deferral; ``()`` for every other
|
|
24819
|
+
unit, which the loop then treats exactly as before (#1052 spec 5.1).
|
|
24820
|
+
|
|
24821
|
+
A unit none of whose files has an ``unverified`` conversations cursor
|
|
24822
|
+
is not classified further. Otherwise one blocked file makes the whole
|
|
24823
|
+
unit an ordinary one, so its own refusal (G10, pending verification,
|
|
24824
|
+
a protected or executor path, an unread new identity) still reaches
|
|
24825
|
+
the loop's retryable failure."""
|
|
24826
|
+
cursors = [
|
|
24827
|
+
(discovered, _codex_conversation_cursor_row(
|
|
24828
|
+
self.conn, str(discovered.source_path)))
|
|
24829
|
+
for discovered in unit.files]
|
|
24830
|
+
if not any(_codex_cursor_unverified(row) for _d, row in cursors):
|
|
24831
|
+
return ()
|
|
24832
|
+
terminal = []
|
|
24833
|
+
for discovered, row in cursors:
|
|
24834
|
+
verdict = _classify_codex_unit_file(
|
|
24835
|
+
self.conn, discovered, executor_paths=self.executor_paths,
|
|
24836
|
+
protected_paths=self.protected_paths,
|
|
24837
|
+
requal_held=self.requal_held, unit=unit,
|
|
24838
|
+
targeted=self.only_paths is not None, cursor=row)
|
|
24839
|
+
if verdict == "blocked":
|
|
24840
|
+
return ()
|
|
24841
|
+
if verdict == "terminal":
|
|
24842
|
+
terminal.append(str(discovered.source_path))
|
|
24843
|
+
return tuple(terminal)
|
|
24844
|
+
|
|
22321
24845
|
def _defer(self, discovered, keys, *, generation, size) -> bool:
|
|
22322
24846
|
path = str(discovered.source_path)
|
|
22323
24847
|
prior = self.deferred.get(path)
|
|
@@ -22792,6 +25316,38 @@ def _rollback_codex_file_savepoint(conn: sqlite3.Connection) -> bool:
|
|
|
22792
25316
|
return False
|
|
22793
25317
|
|
|
22794
25318
|
|
|
25319
|
+
class _CodexConversationCensus:
|
|
25320
|
+
"""One conversation pass's container skips and held reads (#1052 5.3).
|
|
25321
|
+
|
|
25322
|
+
The census is phase 1's: its skips are ``container_skipped``. A
|
|
25323
|
+
unit-phase skip of a path phase 1 did not skip is counted once in
|
|
25324
|
+
``barrier_container_skipped`` instead, so certification never counts one
|
|
25325
|
+
file twice. ``container_reads_held`` counts distinct logical paths across
|
|
25326
|
+
both phases."""
|
|
25327
|
+
|
|
25328
|
+
def __init__(self, stats):
|
|
25329
|
+
self.stats = stats
|
|
25330
|
+
self.skipped_phase1: "set[str]" = set()
|
|
25331
|
+
self.skipped_units: "set[str]" = set()
|
|
25332
|
+
self.held: "set[str]" = set()
|
|
25333
|
+
|
|
25334
|
+
def skip(self, path: str, reason: str, *, unit_phase: bool) -> None:
|
|
25335
|
+
stats = self.stats
|
|
25336
|
+
if not unit_phase:
|
|
25337
|
+
self.skipped_phase1.add(path)
|
|
25338
|
+
stats.container_skipped[reason] = (
|
|
25339
|
+
stats.container_skipped.get(reason, 0) + 1)
|
|
25340
|
+
elif path not in self.skipped_phase1 and path not in self.skipped_units:
|
|
25341
|
+
self.skipped_units.add(path)
|
|
25342
|
+
stats.barrier_container_skipped[reason] = (
|
|
25343
|
+
stats.barrier_container_skipped.get(reason, 0) + 1)
|
|
25344
|
+
|
|
25345
|
+
def hold(self, path: str) -> None:
|
|
25346
|
+
if path not in self.held:
|
|
25347
|
+
self.held.add(path)
|
|
25348
|
+
self.stats.container_reads_held += 1
|
|
25349
|
+
|
|
25350
|
+
|
|
22795
25351
|
def _sync_codex_conversations_locked(
|
|
22796
25352
|
conn: sqlite3.Connection,
|
|
22797
25353
|
*,
|
|
@@ -22879,6 +25435,9 @@ def _sync_codex_conversations_locked(
|
|
|
22879
25435
|
if _cache_side_replay_pending(conn):
|
|
22880
25436
|
stats.deferred_reason = "cache_replay_pending"
|
|
22881
25437
|
return stats
|
|
25438
|
+
if _cache_side_history_heal_pending(conn):
|
|
25439
|
+
stats.deferred_reason = "cache_heal_pending"
|
|
25440
|
+
return stats
|
|
22882
25441
|
# #968 spec 5.11: the barrier extends this admission point. Its state
|
|
22883
25442
|
# or attachment missing fails closed rather than reading as "no
|
|
22884
25443
|
# population".
|
|
@@ -23078,7 +25637,10 @@ def _sync_codex_conversations_locked(
|
|
|
23078
25637
|
location_marks: "list[tuple[str, _rc.SelectedContainer, _AcceptedContainer]]" = []
|
|
23079
25638
|
batch = _CodexConversationBatch()
|
|
23080
25639
|
work = _CodexPublicationWork(
|
|
23081
|
-
conn, files, only_paths=only_paths, stats=stats
|
|
25640
|
+
conn, files, only_paths=only_paths, stats=stats,
|
|
25641
|
+
executor_paths=executor_paths, protected_paths=protected_paths,
|
|
25642
|
+
requal_held=requal_held)
|
|
25643
|
+
census = _CodexConversationCensus(stats)
|
|
23082
25644
|
for discovered, unit in work:
|
|
23083
25645
|
if discovered is None:
|
|
23084
25646
|
if unit is None: # the phase boundary
|
|
@@ -23118,7 +25680,7 @@ def _sync_codex_conversations_locked(
|
|
|
23118
25680
|
requal_held, frozenset(), discovered, selected.location,
|
|
23119
25681
|
first_sight=prev is None):
|
|
23120
25682
|
# G10: held until both stores retire the direct-era spelling.
|
|
23121
|
-
|
|
25683
|
+
census.hold(path_str)
|
|
23122
25684
|
stats.drain_requested = tuple(sorted(
|
|
23123
25685
|
set(stats.drain_requested) | {CODEX_PRUNE_DRAIN_COMMAND}))
|
|
23124
25686
|
if unit is not None:
|
|
@@ -23127,8 +25689,8 @@ def _sync_codex_conversations_locked(
|
|
|
23127
25689
|
continue
|
|
23128
25690
|
_fp = selected.fingerprint
|
|
23129
25691
|
st = _CodexContainerStat(_fp.size, _fp.mtime_ns, _fp.device, _fp.inode)
|
|
23130
|
-
accepted = (None if prev is None
|
|
23131
|
-
|
|
25692
|
+
accepted = (None if prev is None
|
|
25693
|
+
else _codex_conversation_accepted(prev))
|
|
23132
25694
|
container_decision = _codex_container_decision(
|
|
23133
25695
|
selected, accepted, executor_path=path_str in executor_paths, should_stop=None)
|
|
23134
25696
|
if container_decision.action == "adopt-location":
|
|
@@ -23145,12 +25707,15 @@ def _sync_codex_conversations_locked(
|
|
|
23145
25707
|
continue
|
|
23146
25708
|
if container_decision.action == "skip":
|
|
23147
25709
|
reason = container_decision.reason
|
|
23148
|
-
|
|
25710
|
+
census.skip(path_str, reason, unit_phase=unit is not None)
|
|
23149
25711
|
if selected.kind == _rc.ZSTD and prev is None:
|
|
23150
|
-
|
|
25712
|
+
census.hold(path_str)
|
|
23151
25713
|
if unit is not None:
|
|
23152
25714
|
# A failed publication is still due, even when its failure
|
|
23153
25715
|
# was an unreadable skip rather than pending verification.
|
|
25716
|
+
# #1052: a unit reaching a terminal contributor here also
|
|
25717
|
+
# carries a blocked file (or its cursor changed after
|
|
25718
|
+
# classification), so it stays retryable.
|
|
23154
25719
|
batch = _fail_codex_publication_unit(
|
|
23155
25720
|
conn, batch, stats, retry=True)
|
|
23156
25721
|
_report_conversation_progress(progress, "ingest", stats)
|
|
@@ -23357,9 +25922,9 @@ def _sync_codex_conversations_locked(
|
|
|
23357
25922
|
raise _rc.ContainerError("unstable-stat")
|
|
23358
25923
|
except _rc.ContainerError as exc:
|
|
23359
25924
|
reason = _container_skip_reason(exc.category)
|
|
23360
|
-
|
|
25925
|
+
census.skip(path_str, reason, unit_phase=unit is not None)
|
|
23361
25926
|
if selected.kind == _rc.ZSTD and prev is None:
|
|
23362
|
-
|
|
25927
|
+
census.hold(path_str)
|
|
23363
25928
|
if unit is not None:
|
|
23364
25929
|
batch = _fail_codex_publication_unit(conn, batch, stats, retry=True)
|
|
23365
25930
|
_report_conversation_progress(progress, "ingest", stats)
|
|
@@ -23564,6 +26129,11 @@ def _sync_codex_conversations_locked(
|
|
|
23564
26129
|
_commit_codex_conversation_batch(conn, batch, stats, progress)
|
|
23565
26130
|
batch = _CodexConversationBatch()
|
|
23566
26131
|
_commit_codex_conversation_batch(conn, batch, stats, progress)
|
|
26132
|
+
if only_paths is None:
|
|
26133
|
+
# #972 §5.4: the rollups the walk did not rewrite still follow the
|
|
26134
|
+
# current thread metadata and filesystem. Failed files do not stop
|
|
26135
|
+
# it; a targeted pass never runs it.
|
|
26136
|
+
_reconcile_codex_rollup_attribution(conn, stats)
|
|
23567
26137
|
if (work.seal_seq is not None and work.backlog_complete
|
|
23568
26138
|
and not stats.barrier_retry):
|
|
23569
26139
|
# Every certificate sealed up to this token was considered. Units
|
|
@@ -23581,7 +26151,8 @@ def _sync_codex_conversations_locked(
|
|
|
23581
26151
|
conn, "codex_conversation_source_files", path, adopted, snap)
|
|
23582
26152
|
conn.commit()
|
|
23583
26153
|
if (stats.container_pending
|
|
23584
|
-
or stats.container_skipped.get("verification-pending", 0)
|
|
26154
|
+
or stats.container_skipped.get("verification-pending", 0)
|
|
26155
|
+
or stats.barrier_container_skipped.get("verification-pending", 0)):
|
|
23585
26156
|
stats.drain_requested = ("_codex-replay-drain",)
|
|
23586
26157
|
run_codex_find_projection_backfill(conn)
|
|
23587
26158
|
_report_conversation_progress(progress, "finalize", stats)
|
|
@@ -23626,7 +26197,7 @@ def _sync_codex_conversations_locked(
|
|
|
23626
26197
|
and not stats.prune_refused
|
|
23627
26198
|
and not stats.files_deferred_torn
|
|
23628
26199
|
and not stats.container_pending
|
|
23629
|
-
and not any(stats.
|
|
26200
|
+
and not any(stats.all_container_skipped().values())
|
|
23630
26201
|
and stats.files_processed + stats.files_skipped_unchanged
|
|
23631
26202
|
== stats.files_total),
|
|
23632
26203
|
cursor_table="codex_conversation_source_files")
|
|
@@ -23805,7 +26376,9 @@ def _run_transcript_rebuild_worker(
|
|
|
23805
26376
|
emit({
|
|
23806
26377
|
"event": "result",
|
|
23807
26378
|
"stats": {
|
|
23808
|
-
name: value
|
|
26379
|
+
name: (sorted(value) if isinstance(value, (set, frozenset))
|
|
26380
|
+
else value)
|
|
26381
|
+
for name, value in asdict(stats).items()
|
|
23809
26382
|
if name not in _CONVERSATION_RESIDUAL_FIELDS
|
|
23810
26383
|
},
|
|
23811
26384
|
"statsType": type(stats).__name__,
|
|
@@ -24136,6 +26709,21 @@ def cmd_cache_sync(args: argparse.Namespace) -> int:
|
|
|
24136
26709
|
def _sync_codex_leg(
|
|
24137
26710
|
active_conn: sqlite3.Connection,
|
|
24138
26711
|
) -> CodexIngestStats:
|
|
26712
|
+
if not args.rebuild and _codex_replay_heal_due(active_conn):
|
|
26713
|
+
# #982 spec 4.6: an armed byte-zero replay (or one a dead
|
|
26714
|
+
# executor left) clears and refills through the admitted heal
|
|
26715
|
+
# executor, before provenance runs, as the first sync did.
|
|
26716
|
+
with _perf.phase("codex_heal") as _p_heal:
|
|
26717
|
+
heal = run_codex_admitted_heal(wait_for_requests=True)
|
|
26718
|
+
if heal.outcome in ("replay_due", "restart"):
|
|
26719
|
+
# Spec 4.5a: re-armed during the pass (or a unit
|
|
26720
|
+
# found its generation stale); replay again.
|
|
26721
|
+
heal = run_codex_admitted_heal(wait_for_requests=True)
|
|
26722
|
+
_p_heal.set_meta(outcome=heal.outcome, units=heal.units,
|
|
26723
|
+
mode=heal.mode)
|
|
26724
|
+
if heal.outcome in ("busy", "contended"):
|
|
26725
|
+
return CodexIngestStats(lock_contended=True)
|
|
26726
|
+
return heal.stats or CodexIngestStats()
|
|
24139
26727
|
with _perf.phase("sync_codex_cache"):
|
|
24140
26728
|
return sync_codex_cache(
|
|
24141
26729
|
active_conn,
|
|
@@ -24233,14 +26821,7 @@ def cmd_cache_sync(args: argparse.Namespace) -> int:
|
|
|
24233
26821
|
)
|
|
24234
26822
|
contended = True
|
|
24235
26823
|
elif not stats.lock_contended:
|
|
24236
|
-
|
|
24237
|
-
f"[cache-sync] codex done: {stats.files_processed} processed, "
|
|
24238
|
-
f"{stats.files_skipped_unchanged} skipped, "
|
|
24239
|
-
f"{stats.files_reset_truncated} reset, "
|
|
24240
|
-
f"{stats.rows_changed} rows changed, "
|
|
24241
|
-
f"{stats.lines_malformed} malformed, "
|
|
24242
|
-
f"{stats.token_events_skipped} drift-skipped"
|
|
24243
|
-
)
|
|
26824
|
+
_print_codex_done_line(stats)
|
|
24244
26825
|
# #416 review B4: emitted on EVERY branch (including a contended or
|
|
24245
26826
|
# incomplete rebuild) — a deferral means Codex spend and quota stopped
|
|
24246
26827
|
# updating, which the "done" line's zeroes look identical to. Exit code
|
|
@@ -24256,14 +26837,41 @@ def cmd_cache_sync(args: argparse.Namespace) -> int:
|
|
|
24256
26837
|
codex_stats = stats
|
|
24257
26838
|
if not stats.lock_contended:
|
|
24258
26839
|
history = run_codex_history_provenance_drain(wait_for_requests=True)
|
|
24259
|
-
|
|
24260
|
-
|
|
24261
|
-
|
|
24262
|
-
|
|
24263
|
-
|
|
24264
|
-
|
|
24265
|
-
|
|
26840
|
+
heal_record = load_codex_heal_record(conn)
|
|
26841
|
+
# A replay the first leg left blocked (or unfinished) keeps the
|
|
26842
|
+
# executor in replay mode; the history heal waits for it.
|
|
26843
|
+
if not _codex_replay_heal_due(conn) and (
|
|
26844
|
+
(history.outcome == "settled" and history.candidates)
|
|
26845
|
+
or (heal_record is not None
|
|
26846
|
+
and heal_record.get("mode") == "history")):
|
|
26847
|
+
# #982 spec 4.6: the now-settled historical inputs are read by
|
|
26848
|
+
# the admitted heal executor, one file per unit, in place of a
|
|
26849
|
+
# second whole-tree sync under one writer-lock hold.
|
|
26850
|
+
with _perf.phase("codex_heal") as _p_heal:
|
|
26851
|
+
heal = run_codex_admitted_heal(
|
|
26852
|
+
wait_for_requests=True,
|
|
26853
|
+
population=(None if heal_record is not None
|
|
26854
|
+
else _codex_history_candidates()))
|
|
26855
|
+
if heal.outcome in ("replay_due", "restart"):
|
|
26856
|
+
# Spec 4.5a: a replay armed during this heal ended
|
|
26857
|
+
# its pass before the fenced walk; replay now, as the
|
|
26858
|
+
# first leg would have.
|
|
26859
|
+
heal = run_codex_admitted_heal(wait_for_requests=True)
|
|
26860
|
+
_p_heal.set_meta(outcome=heal.outcome, units=heal.units)
|
|
26861
|
+
if heal.stats is not None and heal.stats.rebuild_refused:
|
|
26862
|
+
# The inline re-run's replay was refused by the #485/#968
|
|
26863
|
+
# preflight: reported exactly as the first leg reports a
|
|
26864
|
+
# refused replay (every marker was kept).
|
|
26865
|
+
conn.close()
|
|
26866
|
+
eprint(CODEX_REBUILD_REFUSED_MESSAGE)
|
|
26867
|
+
_p_root.__exit__(None, None, None)
|
|
26868
|
+
return 3
|
|
26869
|
+
if heal.outcome in ("busy", "contended"):
|
|
26870
|
+
codex_stats = CodexIngestStats(lock_contended=True)
|
|
24266
26871
|
contended = True
|
|
26872
|
+
elif heal.stats is not None:
|
|
26873
|
+
# The heal's summary, once, in the first leg's format.
|
|
26874
|
+
_print_codex_done_line(heal.stats)
|
|
24267
26875
|
if not codex_stats.lock_contended:
|
|
24268
26876
|
run_codex_barrier_membership_scan(wait_for_requests=True)
|
|
24269
26877
|
run_codex_barrier_seal_drain(wait_for_requests=True)
|