cctally 1.92.2 → 1.93.0

package/bin/_cctally_journal_repair.py

@@ -10,13 +10,31 @@ import pathlib
 import signal
 import sqlite3
 import sys
+import typing
 
 import _cctally_core
 import _cctally_journal as _journal
+import _lib_accounts
 import _lib_journal
+import _lib_journal_router
 from _lib_json_envelope import stamp_schema_version
 
 
+class _PrefixSnapshot(typing.NamedTuple):
+    """One pinned prefix, read exactly once (#496 S5 §4).
+
+    `audit_ends` maps each `journal_protocol_resolution` op id to the end
+    coordinate of its line. The already-resolved recovery branch needs those
+    coordinates and used to re-read the whole prefix through `_audit_high_water`
+    to find them, even though this pass had them in hand.
+    """
+
+    high_water: "tuple[str, int] | None"
+    prefix_hash: "str | None"
+    selection: object
+    audit_ends: dict
+
+
 def _read_only_high_water() -> "tuple[str, int] | None":
     """Capture the append-only prefix without creating a lock or sidecar."""
     segments = _journal.list_segments()
@@ -27,37 +45,99 @@ def _read_only_high_water() -> "tuple[str, int] | None":
 
 
 def _read_prefix(high_water):
+    """Stream the pinned prefix once, producing everything derived from it.
+
+    Returns `(records, evidence, prefix_hash, audit_ends)`. The prefix hash
+    comes from a `PrefixHashAccumulator` fed by the same bytes this pass reads,
+    the protocol evidence is captured from that accumulator rather than by
+    re-reading the prefix per resolution op, and the cutover account is captured
+    inline exactly as `rebuild_stats_index` captures it. Before this, those were
+    four separate whole-prefix traversals on top of this one (#496 S5 §4).
+
+    Every record stays decoded. Unlike the rebuild, the selector here feeds an
+    acknowledgement the repair command may then mint, and unlike the rebuild's
+    filtered retention there is no placeholder scheme to keep the `enumerate`
+    numbering identical — so the list is unfiltered, exactly as before.
+    """
     if high_water is None:
-        return [], ()
+        return [], (), None, {}
+    segments = _journal.list_segments()
+    if high_water[0] not in segments:
+        raise OSError(
+            f"journal high-water segment is unavailable: {high_water[0]}"
+        )
     records = []
     evidence = []
+    audit_ends: dict = {}
     malformed = 0
     prior_high_water = None
-    for segment, offset, raw in _journal._read_range(None, high_water):
+    cutover_captured = _journal._CUTOVER_UNSEEN
+    hasher = _lib_journal_router.PrefixHashAccumulator()
+    for segment, offset, raw in _journal._iter_range_with_segments(
+        None,
+        high_water,
+        segments,
+        on_segment=lambda name: hasher.begin_segment(name, prior_high_water),
+        on_bytes=hasher.extend,
+    ):
         record = _lib_journal.decode_line(raw)
         if record is None:
             malformed += 1
             prior_high_water = (segment, offset + len(raw) + 1)
             continue
-        _journal._capture_protocol_prefix_evidence(
-            record,
-            prior_high_water,
-            evidence,
-        )
+        record_end = (segment, offset + len(raw) + 1)
+        if record.get("t") == "op":
+            _journal._capture_protocol_prefix_evidence(
+                record,
+                prior_high_water,
+                evidence,
+                hasher=hasher,
+            )
+            payload = record.get("payload")
+            if (
+                isinstance(payload, dict)
+                and payload.get("kind")
+                == _lib_journal._PROTOCOL_RESOLUTION_KIND
+            ):
+                audit_ends[record.get("id")] = record_end
+        # First cutover op wins, exactly as `find_accounts_cutover_op` scans.
+        if (
+            cutover_captured is _journal._CUTOVER_UNSEEN
+            and record.get("id") == _journal.CUTOVER_OP_ID
+        ):
+            cutover_captured = _journal._cutover_value_of(record)
         records.append(record)
-        prior_high_water = (segment, offset + len(raw) + 1)
+        prior_high_water = record_end
+    prefix_hash = hasher.digest_at(high_water)
+    # The accumulator buffers the segment it is reading — 410 MB on the
+    # maintainer's journal — so it is dropped the moment its pass ends, before
+    # the normalization loop below, exactly as `rebuild_stats_index` drops it.
+    # It is dropped before the raise too, so a malformed prefix does not pin the
+    # buffer on the traceback while the exception unwinds.
+    hasher = None
     if malformed:
         raise _lib_journal.JournalProtocolError(
             f"journal prefix contains {malformed} malformed line(s)"
         )
-    cutover_claude = _journal.resolve_cutover_claude_account()
+    # No suffix fallback, deliberately. `_read_only_high_water` pins the
+    # canonically-last segment at its full size, so this prefix IS the whole
+    # journal and an op the prefix does not contain is not in the journal at
+    # all — which is exactly what `resolve_cutover_claude_account` used to
+    # answer by re-reading every segment. An op appended in the window between
+    # that pin and this loop is therefore NOT seen, where the whole-journal scan
+    # would have found it; that divergence is accepted, because the account then
+    # matches the prefix the fingerprints are computed over and `_apply`'s
+    # conflict check catches a preview that has gone stale. The rebuild's
+    # `_resolve_cutover_for_rebuild` cannot be reused here: its fallback calls
+    # `journal_high_water`, which takes the leaf lock and so CREATES
+    # `journal.lock`, and the preview must leave no sidecar behind.
+    if cutover_captured is _journal._CUTOVER_UNSEEN or cutover_captured is None:
+        cutover_claude = _lib_accounts.UNATTRIBUTED
+    else:
+        cutover_claude = cutover_captured
     for record in records:
         _journal._normalize_legacy_account_stamp(record, cutover_claude)
-    return records, tuple(evidence)
-
-
-def _prefix_hash(high_water) -> "str | None":
-    return _journal.journal_prefix_hash(high_water)
+    return records, tuple(evidence), prefix_hash, audit_ends
 
 
 def _high_water_dict(high_water):
@@ -66,18 +146,21 @@ def _high_water_dict(high_water):
     return {"segment": high_water[0], "offset": high_water[1]}
 
 
-def _selection_snapshot():
+def _selection_snapshot() -> _PrefixSnapshot:
     high_water = _read_only_high_water()
-    records, evidence = _read_prefix(high_water)
+    records, evidence, prefix_hash, audit_ends = _read_prefix(high_water)
     selection = _lib_journal.resolve_effective_events(
         records,
         protocol_prefix_evidence=evidence,
     )
-    return high_water, _prefix_hash(high_water), selection
+    return _PrefixSnapshot(high_water, prefix_hash, selection, audit_ends)
 
 
 def _preview_payload(requested=()):
-    high_water, prefix_hash, selection = _selection_snapshot()
+    snapshot = _selection_snapshot()
+    high_water = snapshot.high_water
+    prefix_hash = snapshot.prefix_hash
+    selection = snapshot.selection
     unacknowledged = {
         violation.fingerprint: violation
         for violation in selection.protocol_violations
@@ -115,7 +198,7 @@ def _preview_payload(requested=()):
         "rebuild": None,
         "errors": errors,
     }
-    return stamp_schema_version(body, version=1), selection
+    return stamp_schema_version(body, version=1), snapshot
 
 
 def _rebuild_dict(result):
@@ -230,7 +313,7 @@ def _repair_failure_guidance(exc: Exception) -> str:
 
 def _post_audit_failure(requested, audit_ids, exc):
     """Report durable acknowledgement truth when index publication declined."""
-    payload, _selection = _preview_payload(requested)
+    payload, _snapshot = _preview_payload(requested)
     payload["status"] = "failed"
     payload["errors"] = [_repair_failure_guidance(exc)]
     if len(audit_ids) == 1:
@@ -261,12 +344,19 @@ def _stats_has_acknowledgements(fingerprints) -> bool:
         return False
 
 
-def _audit_high_water(audit_ids, high_water):
-    found = {}
-    for segment, offset, raw in _journal._read_range(None, high_water):
-        record = _lib_journal.decode_line(raw)
-        if record is not None and record.get("id") in audit_ids:
-            found[record["id"]] = (segment, offset + len(raw) + 1)
+def _recovery_high_water(audit_ids, audit_ends):
+    """The last acknowledged audit record's end coordinate.
+
+    `audit_ends` is produced by the same streaming pass that produced the
+    selection, so an already-resolved recovery no longer re-reads the whole
+    prefix to find coordinates that pass already held (#496 S5 §4.1). The
+    ordering rule is the pre-change one: canonical segment order, then offset.
+    """
+    found = {
+        audit_id: audit_ends[audit_id]
+        for audit_id in audit_ids
+        if audit_id in audit_ends
+    }
     if set(found) != set(audit_ids):
         raise _journal.JournalError(
             "acknowledged protocol audit record is missing from the journal"
@@ -279,7 +369,8 @@ def _audit_high_water(audit_ids, high_water):
 def _apply(requested, initial_preview):
     _call_pause_hook("before-lock")
     with _repair_locks():
-        preview, selection = _preview_payload(requested)
+        preview, snapshot = _preview_payload(requested)
+        selection = snapshot.selection
         if preview["errors"]:
             preview["status"] = "conflict"
             return preview, 2
@@ -347,7 +438,7 @@ def _apply(requested, initial_preview):
                     {audit["id"]},
                     exc,
                 )
-            final_payload, _final_selection = _preview_payload(requested)
+            final_payload, _final_snapshot = _preview_payload(requested)
             final_payload["status"] = "applied"
             final_payload["selectedViolations"] = [
                 violation.to_dict() for violation in to_acknowledge
@@ -374,8 +465,8 @@ def _apply(requested, initial_preview):
         if len(audit_ids) == 1:
             preview["auditId"] = next(iter(audit_ids))
         if not _stats_has_acknowledgements(requested):
-            recovery_high_water = _audit_high_water(
-                audit_ids, reviewed_high_water
+            recovery_high_water = _recovery_high_water(
+                audit_ids, snapshot.audit_ends
             )
             try:
                 _call_rebuild_error_hook()
@@ -395,7 +486,7 @@ def _apply(requested, initial_preview):
                     audit_ids,
                     exc,
                 )
-            final_payload, _final_selection = _preview_payload(requested)
+            final_payload, _final_snapshot = _preview_payload(requested)
             final_payload["status"] = "recovered"
             if len(audit_ids) == 1:
                 final_payload["auditId"] = next(iter(audit_ids))
@@ -408,7 +499,7 @@ def cmd_db_journal_repair(args) -> int:
     """Preview structural violations without mutating the journal or indexes."""
     try:
         requested = list(getattr(args, "violation", ()) or ())
-        payload, _selection = _preview_payload(requested)
+        payload, _snapshot = _preview_payload(requested)
     except (OSError, _lib_journal.JournalProtocolError) as exc:
         if bool(getattr(args, "json", False)):
             try:

package/bin/_cctally_milestone_history.py

@@ -30,7 +30,7 @@ import sys
 from dataclasses import replace
 
 from _cctally_core import make_week_ref, parse_iso_datetime
-from _cctally_quota import codex_quota_breakdown
+from _cctally_quota import assert_projection_readable, codex_quota_breakdown
 from _lib_accounts import UNATTRIBUTED
 from _lib_codex_pools import is_model_scoped_codex_quota
 from _lib_dashboard_sources import dashboard_resource_key
@@ -677,6 +677,8 @@ def _load_codex_cycles(stats_conn, root_keys, *, include_orphaned=False,
     # `quota_window_blocks.account_key` is NOT NULL DEFAULT 'unattributed'.
     account_clause = "" if account_key is None else "AND account_key=? "
     account_params = () if account_key is None else (account_key,)
+    # BEFORE the SQL, per #496 S5b section 4.7.
+    assert_projection_readable(stats_conn)
     rows = stats_conn.execute(
         "SELECT source_root_key, logical_limit_key, observed_slot, "
         "       window_minutes, limit_id, limit_name, account_key, "
@@ -808,6 +810,7 @@ def _codex_five_hour_rows(stats_conn, cyc, *, include_orphaned=False) -> list:
     never-combine rule.
     """
     orphan_clause = "" if include_orphaned else "AND orphaned_at IS NULL "
+    assert_projection_readable(stats_conn)
     return stats_conn.execute(
         "SELECT source_root_key, logical_limit_key, observed_slot, "
         "       window_minutes, limit_id, limit_name, account_key, "

package/bin/_cctally_project.py

@@ -768,12 +768,14 @@ def cmd_project(args: argparse.Namespace) -> int:
     # Shareable-reports gate: --format short-circuits the JSON / table
     # dispatch via `_share_render_and_emit`. The mutex in
     # `_add_share_args` keeps `--format` and `--json` from coexisting.
-    # Privacy invariant (Section 8.4 / 5.3): the wrapper runs `_lib_share._scrub`
-    # before rendering, so default output anonymizes project labels to
-    # `project-1` / `project-2` / ...; `--reveal-projects` opts back in.
-    # The builder populates `ProjectCell.label` / `ChartPoint.project_label`
-    # / `ChartPoint.x_label` with REAL names; the wrapper-level scrubber is
-    # the single chokepoint that rewrites them.
+    # Privacy invariant (Section 8.4 / 5.3): `_lib_share.render()` prepares
+    # the RAW snapshot the wrapper hands it, so default output anonymizes
+    # project labels to `project-1` / `project-2` / ...; `--reveal-projects`
+    # opts back in. The builder populates `ProjectCell.label` /
+    # `ChartPoint.project_label` / `ChartPoint.x_label` with REAL names;
+    # `render()` is the chokepoint that rewrites them. (It is NOT `_scrub()`:
+    # that function is retained for backward compatibility and no production
+    # path calls it.)
     if getattr(args, "format", None):
         # Note: --breakdown is a no-op under --format (snapshot focuses on
         # the headline per-project usage table + HBar chart; per-model