cctally 1.92.0 → 1.92.2

package/bin/_lib_journal_router.py

@@ -0,0 +1,234 @@
+"""Pure record-routing kernel for the stats rebuild's journal read pass (#496 S4).
+
+The rebuild used to materialize the whole journal twice: `_read_range` built a
+list of every raw line, and the decode loop built a second list of every parsed
+record while the first was still referenced. On the maintainer's install that is
+1,954,007 lines and 1.72 GB producing an 8.08 GiB peak, of which the stats fold
+consumes 99,289 records (5.08%).
+
+`bin/_cctally_doctor.py`'s conflict scan already established the shape this
+kernel generalizes: retain only what the effective selector consumes and drop
+everything else as it is decoded (#374 review, measured at 4.3 GB of peak RSS
+for an identical result).
+
+No I/O and no imports from `_cctally_journal`, so the rules here are unit
+testable without a journal on disk.
+"""
+from __future__ import annotations
+
+import hashlib
+
+
+#: The record types the stats rebuild retains decoded. `resolve_effective_events`
+#: acts only on evt / correction / correction_batch and the protocol-resolution
+#: op; the op-fold stream takes only ops whose kind is in `FOLD_APPLIERS`.
+#: Deliberately identical to `_cctally_doctor._CONFLICT_SCAN_RECORD_TYPES` — the
+#: two must not drift, because both feed the same shared selector.
+RETAINED_RECORD_TYPES = frozenset({"evt", "correction", "correction_batch", "op"})
+
+
+class LastSeenAccumulator:
+    """Reproduce `_derive_account_last_seen`'s contribution set from a stream.
+
+    The rebuild normalizes every record and then takes `_account_of`, which
+    reads a top-level ``account`` or an ``account_observe`` op's
+    ``payload.account_key``. `_normalize_legacy_account_stamp` writes a
+    top-level ``account`` ONLY for ``t == "obs"``; a legacy evt or op instead
+    gets ``payload.account_key``, which `_account_of` does not read.
+
+    So exactly three classes contribute, and a provider-wide maximum over every
+    legacy line would over-count — advancing `last_seen_utc` from legacy events
+    and vendor-tagged budget events that contribute nothing today. The Claude
+    legacy bucket is deferred because the cutover account is not known until the
+    stream reaches it, at 92.9% of a production journal.
+    """
+
+    __slots__ = ("stamped", "legacy_claude_at", "legacy_codex_at")
+
+    def __init__(self) -> None:
+        self.stamped: dict = {}
+        self.legacy_claude_at = None
+        self.legacy_codex_at = None
+
+    def observe(self, record, provider_of_legacy) -> None:
+        """Fold one record. `provider_of_legacy` is `classify_legacy_provider`."""
+        at = record.get("at")
+        if not at:
+            return
+        account = record.get("account")
+        if isinstance(account, str) and account:
+            self._bump(account, at)
+            return
+        record_type = record.get("t")
+        if record_type == "op":
+            payload = record.get("payload") or {}
+            if payload.get("kind") == "account_observe":
+                key = payload.get("account_key")
+                if isinstance(key, str) and key:
+                    self._bump(key, at)
+            return
+        if record_type != "obs":
+            # A legacy evt normalizes into `payload.account_key`, which
+            # `_account_of` ignores. Contributing here would move last-seen.
+            return
+        provider = provider_of_legacy(record)
+        if provider == "claude":
+            if self.legacy_claude_at is None or at > self.legacy_claude_at:
+                self.legacy_claude_at = at
+        elif provider == "codex":
+            if self.legacy_codex_at is None or at > self.legacy_codex_at:
+                self.legacy_codex_at = at
+
+    def _bump(self, key: str, at: str) -> None:
+        previous = self.stamped.get(key)
+        if previous is None or at > previous:
+            self.stamped[key] = at
+
+    def resolve(self, cutover_claude: str, unattributed: str) -> dict:
+        """Apply the deferred legacy buckets and return the final MAX map."""
+        out = dict(self.stamped)
+        if self.legacy_claude_at is not None:
+            previous = out.get(cutover_claude)
+            if previous is None or self.legacy_claude_at > previous:
+                out[cutover_claude] = self.legacy_claude_at
+        if self.legacy_codex_at is not None:
+            previous = out.get(unattributed)
+            if previous is None or self.legacy_codex_at > previous:
+                out[unattributed] = self.legacy_codex_at
+        return out
+
+
+class PrefixEvidenceUnavailable(LookupError):
+    """A protocol-evidence digest was asked for a prefix the stream dropped.
+
+    Never expected: an evidence point is always the end of the line immediately
+    preceding a `journal_protocol_resolution` op, so it is either inside the
+    segment being streamed or the boundary registered at the last segment
+    transition. Raised rather than silently degraded, because the alternative is
+    a rebuild that quietly disagrees with `journal_prefix_hash`.
+    """
+
+
+class PrefixHashAccumulator:
+    """`journal_prefix_hash` computed from the bytes the rebuild already read.
+
+    `journal_prefix_hash(prior_high_water)` does `path.read_bytes()[:size]` on
+    every segment through the prefix, so one `journal_protocol_resolution` op
+    re-reads the whole journal up to its own position and builds a full-segment
+    bytes transient. This accumulator reproduces the identical durable digest
+    from the bytes the single streaming pass is reading anyway (#496 S4 §5.2).
+
+    The framing is `journal_prefix_hash`'s, verbatim: per segment, the 4-byte
+    big-endian name length, the name, the 8-byte big-endian data length, then
+    the data. Completed segments are absorbed into a running `sha256`; only the
+    segment currently being streamed is buffered, so residency is bounded by one
+    segment. At a segment transition the caller passes the boundary the next
+    record's `prior_high_water` will name, which is the ONE offset in the
+    outgoing segment that can still be asked for; its digest is precomputed
+    before the buffer is released.
+    """
+
+    # A transition can only ever register one boundary, and consecutive empty
+    # segments re-register the same one. A handful of slots is therefore already
+    # generous; the cap exists so a pathological journal cannot grow this map.
+    # Eviction is safe because a registered boundary is only ever READ
+    # IMMEDIATELY AFTER the `begin_segment` that registered it: a resolution op
+    # can name a previous segment's end only when it is the first line of the
+    # new segment, and every later position falls in the `_current_name` branch
+    # of `digest_at`. So the cap bounds a map that never needs more than the
+    # most recent entry; widening it buys nothing and narrowing it below the
+    # runs of empty segments a journal can contain would start dropping the one
+    # entry that is still live.
+    _MAX_BOUNDARIES = 16
+
+    def __init__(self) -> None:
+        self._running = hashlib.sha256()
+        self._current_name = None
+        self._current = bytearray()
+        self._boundaries: dict = {}
+        self._boundary_order: list = []
+        #: Bytes fed into an evidence digest, reported as the `protocol_evidence`
+        #: traversal pass. Zero on any journal with no resolution op.
+        self.bytes_hashed = 0
+        self.digests_computed = 0
+
+    # -- feeding ----------------------------------------------------------
+
+    def begin_segment(self, name: str, boundary=None) -> None:
+        """Start `name`, absorbing whatever segment was open before it.
+
+        `boundary` is the `(segment, offset)` the next record's evidence would
+        name — i.e. the streaming loop's current `prior_high_water`. It is
+        resolved and cached HERE because the outgoing segment's bytes are gone
+        immediately afterwards.
+        """
+        if boundary is not None:
+            self._register_boundary(boundary)
+        if self._current_name is not None:
+            # `hashlib.update` accepts the buffer directly, so the outgoing
+            # segment is framed WITHOUT a copy of itself. The maintainer's
+            # largest segment is 410 MB, and copying it here briefly doubled
+            # that at every transition.
+            with memoryview(self._current) as data:
+                self._absorb(self._current_name, data)
+        self._current_name = name
+        self._current = bytearray()
+
+    def extend(self, data: bytes) -> None:
+        """Absorb raw bytes read from the segment currently open."""
+        self._current.extend(data)
+
+    # -- reading ----------------------------------------------------------
+
+    def digest_at(self, high_water) -> "str | None":
+        """`journal_prefix_hash(high_water)`, from the streamed bytes."""
+        if high_water is None:
+            return None
+        segment, offset = high_water
+        if segment == self._current_name:
+            digest = self._running.copy()
+            with memoryview(self._current)[:offset] as data:
+                self._frame(digest, segment, data)
+                self.bytes_hashed += len(data)
+            self.digests_computed += 1
+            return "sha256:" + digest.hexdigest()
+        cached = self._boundaries.get((segment, offset))
+        if cached is None:
+            raise PrefixEvidenceUnavailable(
+                f"no streamed prefix evidence for {segment}@{offset}"
+            )
+        self.bytes_hashed += cached[1]
+        self.digests_computed += 1
+        return cached[0]
+
+    # -- internals --------------------------------------------------------
+
+    def _register_boundary(self, boundary) -> None:
+        segment, offset = boundary
+        if segment != self._current_name:
+            # Already registered at an earlier transition (a run of empty
+            # segments leaves `prior_high_water` unchanged), or never streamed.
+            return
+        key = (segment, offset)
+        if key in self._boundaries:
+            return
+        digest = self._running.copy()
+        with memoryview(self._current)[:offset] as data:
+            self._frame(digest, segment, data)
+            size = len(data)
+        self._boundaries[key] = ("sha256:" + digest.hexdigest(), size)
+        self._boundary_order.append(key)
+        while len(self._boundary_order) > self._MAX_BOUNDARIES:
+            self._boundaries.pop(self._boundary_order.pop(0), None)
+
+    def _absorb(self, name, data) -> None:
+        self._frame(self._running, name, data)
+
+    @staticmethod
+    def _frame(digest, name: str, data) -> None:
+        """`data` is any bytes-like buffer; `hashlib` consumes it in place."""
+        encoded = name.encode("utf-8")
+        digest.update(len(encoded).to_bytes(4, "big"))
+        digest.update(encoded)
+        digest.update(len(data).to_bytes(8, "big"))
+        digest.update(data)

package/bin/_lib_stats_publish.py

@@ -0,0 +1,243 @@
+"""Pure planning kernel for in-place stats index publication (#496 S3).
+
+No I/O and no SQLite connection: every function takes plain data so the
+publication protocol's decisions can be tested without a database.
+"""
+
+from __future__ import annotations
+
+import dataclasses
+import re
+import sqlite3
+
+_INTERNAL = "sqlite_"
+_VIRTUAL = re.compile(r"CREATE\s+VIRTUAL\s+TABLE", re.IGNORECASE)
+# `INSERT INTO t SELECT * FROM src.t` is not valid for a generated column, so a
+# table carrying one is refused rather than mis-copied. Both spellings SQLite
+# accepts are matched, and the pattern is anchored on what actually precedes the
+# `AS (` of a generated column: either the `GENERATED ALWAYS` keywords, or the
+# end of a column's type/constraint text at a `,` or `(` boundary. A bare
+# `\bAS\s*\(` would also match a `CHECK (x AS (…))`-shaped expression or any
+# future DDL that merely contains those characters, and the refusal is a hard,
+# non-fallback-eligible raise — so a false positive would abort publication on a
+# table the copy handles perfectly well.
+_GENERATED = re.compile(
+    r"GENERATED\s+ALWAYS\s+AS\s*\("
+    r"|[(,]\s*\"?\w+\"?(?:[^,()]|\([^()]*\))*?\bAS\s*\(",
+    re.IGNORECASE,
+)
+
+# Drop order: dropping a table already removes its own indexes and triggers, so
+# the dependent classes go first and a precomputed flat list never touches an
+# object that no longer exists.
+_DROP_ORDER = (
+    ("view", "DROP VIEW IF EXISTS"),
+    ("trigger", "DROP TRIGGER IF EXISTS"),
+    ("index", "DROP INDEX IF EXISTS"),
+    ("table", "DROP TABLE IF EXISTS"),
+)
+# Create order for everything that is not a table: after the rows, so index
+# builds are single-pass, and indexes before triggers before views.
+_CREATE_ORDER = ("index", "trigger", "view")
+
+
+@dataclasses.dataclass(frozen=True)
+class GenerationSwapPlan:
+    """One transaction's worth of statements, in the order they must run."""
+
+    drop_statements: tuple
+    create_table_statements: tuple
+    copy_tables: tuple
+    # Indexes, then triggers, then views — every non-table object, created
+    # after the row copy. Named for the only class the stats schema uses today.
+    create_index_statements: tuple
+    rejected: tuple
+
+
+def _usable(objects):
+    """Objects a swap may act on: named by the user, and carrying their DDL.
+
+    `sqlite_sequence` cannot be dropped, and an automatic index created by a
+    UNIQUE constraint has `sql IS NULL` and no independent existence.
+    """
+    for kind, name, sql in objects:
+        if str(name).startswith(_INTERNAL):
+            continue
+        if sql is None:
+            continue
+        yield str(kind), str(name), str(sql)
+
+
+def _unsupported(kind: str, sql: str) -> bool:
+    if _VIRTUAL.search(sql):
+        return True
+    return kind == "table" and bool(_GENERATED.search(sql))
+
+
+def plan_generation_swap(dest_objects, src_objects) -> GenerationSwapPlan:
+    """Plan the swap of ``dest_objects`` for ``src_objects``.
+
+    Each argument is a sequence of ``(type, name, sql)`` triples exactly as
+    ``SELECT type, name, sql FROM sqlite_schema`` returns them. Deriving the
+    drop list from the destination and the create list from the source retires
+    whatever the live generation holds without a maintained table of names, so
+    a table added in a later epoch cannot be forgotten.
+    """
+    rejected = tuple(
+        name
+        for kind, name, sql in _usable(src_objects)
+        if _unsupported(kind, sql)
+    )
+
+    drops = []
+    for wanted, statement in _DROP_ORDER:
+        for kind, name, _sql in _usable(dest_objects):
+            if kind == wanted:
+                drops.append(f'{statement} "{name}"')
+
+    creates, copies = [], []
+    others: dict = {kind: [] for kind in _CREATE_ORDER}
+    for kind, name, sql in _usable(src_objects):
+        if _unsupported(kind, sql):
+            continue
+        if kind == "table":
+            creates.append(sql)
+            copies.append(name)
+        elif kind in others:
+            others[kind].append(sql)
+
+    post = [sql for kind in _CREATE_ORDER for sql in others[kind]]
+    return GenerationSwapPlan(
+        drop_statements=tuple(drops),
+        create_table_statements=tuple(creates),
+        copy_tables=tuple(copies),
+        create_index_statements=tuple(post),
+        rejected=rejected,
+    )
+
+
+# --------------------------------------------------------------------------
+# The publication phase machine (#496 S3 §4.1)
+# --------------------------------------------------------------------------
+#
+# "The transaction raised" is not a safe discriminator, because a failure
+# before the commit can roll back while a failure after it cannot, and a
+# commit-time I/O error leaves the outcome genuinely unknown. The publisher
+# therefore records which of these four phases it had reached when it failed.
+
+# Rollback is proven, the old generation is intact, and physical fallback is
+# legal.
+PRE_COMMIT = "pre_commit"
+# The commit's outcome is undetermined. Do not fall back; reopen and resolve
+# through the publication stamp.
+COMMIT_UNKNOWN = "commit_unknown"
+# The new generation is live. Physical fallback is never legal from here.
+COMMITTED = "committed"
+# The record and the marker agree with the bytes.
+VERDICT_SETTLED = "verdict_settled"
+
+# Only a structural inability to operate on the destination authorizes
+# discarding it. Busy, full, out-of-memory and I/O-resource failures leave a
+# perfectly good generation live and must return a retryable failure instead.
+# The structural tokens are `_cctally_db._SQLITE_CORRUPTION_MESSAGES` plus the
+# encrypted-file spelling of SQLITE_NOTADB.
+_STRUCTURAL = (
+    "database disk image is malformed",
+    "file is not a database",
+    "malformed database schema",
+    "encrypted or is not a database",
+)
+_RETRYABLE = (
+    "locked",
+    "busy",
+    "full",
+    "out of memory",
+    "i/o error",
+    "readonly",
+    "permission denied",
+)
+
+
+# --------------------------------------------------------------------------
+# Three-state stamp resolution (#496 S3 §5)
+# --------------------------------------------------------------------------
+#
+# An in-place publish attaches its scratch read-only and detaches it, so the
+# scratch survives commit and rollback identically and the publication marker's
+# `scratchPath` crash discriminator inverts. The publication writes its own
+# identity into `stats_publication_stamp` inside the publication transaction
+# instead, and the opener compares that against the marker's `recordPath`.
+#
+# The answer is three-valued, not boolean. "The stamp does not name this
+# record" proves a rollback only if the stamp was READ successfully; a missing
+# table, a corrupt page, an unreadable schema, a malformed or duplicated row,
+# or any query error proves nothing at all. Treating one of those as
+# never-committed would discard a verdict owed on bytes that are live.
+
+#: The stamp names this marker's rebuild record: the publication COMMITTED.
+STAMP_MATCH = "MATCH"
+#: The stamp was read and does not name this record: the publication never
+#: became live, and the live bytes are its untouched predecessor.
+STAMP_PROVEN_PREDECESSOR = "PROVEN_PREDECESSOR"
+#: The stamp could not be read, or read as something it may not be. Preserve
+#: the marker and fail closed; only PROVEN_PREDECESSOR may discard it.
+STAMP_INDETERMINATE = "INDETERMINATE"
+
+
+def resolve_stamp(stamp, marker_record_path) -> str:
+    """Resolve a pending publication against the stamp read from its destination.
+
+    ``stamp`` is whatever the read produced: the exception that prevented it,
+    ``None`` or an empty sequence when the table read cleanly and held no row,
+    a single row mapping, or the sequence of row mappings the table held. Each
+    mapping carries at least ``record_path``.
+
+    ``marker_record_path`` is the marker's ``recordPath`` as a string.
+
+    Returns one of `STAMP_MATCH`, `STAMP_PROVEN_PREDECESSOR` or
+    `STAMP_INDETERMINATE`. Anything the function cannot interpret resolves
+    INDETERMINATE, because that is the state that preserves evidence.
+    """
+    if isinstance(stamp, BaseException):
+        return STAMP_INDETERMINATE
+    if not isinstance(marker_record_path, str) or not marker_record_path:
+        return STAMP_INDETERMINATE
+    if stamp is None:
+        return STAMP_PROVEN_PREDECESSOR
+    if isinstance(stamp, (list, tuple)):
+        if not stamp:
+            return STAMP_PROVEN_PREDECESSOR
+        if len(stamp) != 1:
+            # The publication transaction deletes before it inserts, so more
+            # than one row is a state the protocol cannot produce.
+            return STAMP_INDETERMINATE
+        stamp = stamp[0]
+    if not isinstance(stamp, dict):
+        return STAMP_INDETERMINATE
+    recorded = stamp.get("record_path")
+    if not isinstance(recorded, str) or not recorded:
+        return STAMP_INDETERMINATE
+    if recorded == marker_record_path:
+        return STAMP_MATCH
+    return STAMP_PROVEN_PREDECESSOR
+
+
+def may_fall_back_to_replacement(exc) -> bool:
+    """Whether ``exc`` authorizes physically replacing the destination.
+
+    Fails closed: anything unrecognized returns False, leaving the old
+    generation live and the failure retryable. SQLite's numeric code is
+    preferred where the exception still carries one, mirroring
+    ``_cctally_db._is_sqlite_corruption_error``; the canonical messages are the
+    string-only boundary.
+    """
+    code = getattr(exc, "sqlite_errorcode", None)
+    if isinstance(code, int):
+        primary = code & 0xFF
+        if primary in {sqlite3.SQLITE_CORRUPT, sqlite3.SQLITE_NOTADB}:
+            return True
+        return False
+    text = str(exc).casefold()
+    if any(token in text for token in _RETRYABLE):
+        return False
+    return any(token in text for token in _STRUCTURAL)

package/bin/cctally

@@ -967,7 +967,9 @@ ProdMigrationRefused = _cctally_db.ProdMigrationRefused
 StatsDbCorruptError = _cctally_db.StatsDbCorruptError
 StatsDbMaintenanceError = _cctally_db.StatsDbMaintenanceError
 StatsEpochMismatchError = _cctally_db.StatsEpochMismatchError
+StatsRebuildDeferred = _cctally_db.StatsRebuildDeferred
 StatsEpochRebuildDeferred = _cctally_db.StatsEpochRebuildDeferred
+StatsHealDeferred = _cctally_db.StatsHealDeferred
 _is_sqlite_corruption_error = _cctally_db._is_sqlite_corruption_error
 _stats_corruption_guidance = _cctally_db._stats_corruption_guidance
 _STATS_MIGRATIONS = _cctally_db._STATS_MIGRATIONS
@@ -997,6 +999,9 @@ _cctally_store = _load_sibling("_cctally_store")
 cmd_stats_epoch_rebuild_internal = (
     _cctally_store.cmd_stats_epoch_rebuild_internal
 )
+cmd_stats_corruption_heal_internal = (
+    _cctally_store.cmd_stats_corruption_heal_internal
+)
 _print_migration_error_banner_if_needed = _cctally_db._print_migration_error_banner_if_needed
 cmd_db_status = _cctally_db.cmd_db_status
 _db_status_for = _cctally_db._db_status_for
@@ -3355,7 +3360,10 @@ def main(argv: list[str] | None = None) -> int:
         # generic DatabaseError so it never renders as a raw traceback.
         eprint(f"cctally: {exc}")
         return 3
-    except StatsEpochRebuildDeferred as exc:
+    except StatsRebuildDeferred as exc:
+        # One catch for both deferral classes (#496 S3 §6): the epoch worker
+        # and the corruption-heal worker both mean "a rebuild is running; do
+        # not print a partial report".
         eprint(f"cctally: {exc}")
         return 3
     except AccountAttributionUnavailable as exc:
@@ -3477,8 +3485,9 @@ def _post_command_update_hooks(command: str | None, args) -> None:
         # still cctally-dev at this point). Same rationale class as doctor.
         return
     if command in ("_update-check", "_telemetry-beat", "_codex-quota-verify",
-                   "_stats-epoch-rebuild", "_codex-replay-drain"):
-        # All five hidden workers are detached and have already done their one job
+                   "_stats-epoch-rebuild", "_stats-corruption-heal",
+                   "_codex-replay-drain"):
+        # All six hidden workers are detached and have already done their one job
         # in their own command handler; none must re-enter this hook.
         # Without the guard the ``_update-check`` worker would fall through to
         # the telemetry gate below and (throttle-bounded) spawn a