cctally 1.113.0 → 1.115.0

package/bin/_cctally_refresh.py

@@ -609,19 +609,25 @@ def _refresh_usage_inproc(timeout_seconds: float = 5.0) -> _RefreshUsageResult:
             reason="test stub",
         )
 
-    token = c._resolve_oauth_token()
-    if not token:
-        return _RefreshUsageResult(status="no_oauth_token", reason="no token")
-
     # Force-refresh deliberately bypasses the pre-fetch backoff gate, but its
     # resulting 429/success transition must be ordered with the automatic
     # refresh writer.  Hold the canonical selected-state lock from the fetch
     # through authoritative publication and the matching backoff update.
     try:
         with c._selected_state_lock():
+            # #825/#832 spec 4.6: the selected credential and its binding are
+            # captured TOGETHER, before the fetch.
+            binding, token = _capture_selected_credential(c)
+            if binding is None:
+                return _RefreshUsageResult(
+                    status="record_failed",
+                    reason="the active Claude identity is unreadable or changing")
+            if not token:
+                return _RefreshUsageResult(status="no_oauth_token", reason="no token")
             return _refresh_usage_inproc_locked(
                 c,
                 token=token,
+                binding=binding,
                 timeout_seconds=timeout_seconds,
             )
     except OSError as exc:
@@ -632,12 +638,20 @@ def _refresh_usage_inproc_locked(
     c,
     *,
     token: str,
+    binding,
     timeout_seconds: float,
 ) -> _RefreshUsageResult:
-    """Force-refresh with the canonical selected-state lock already held."""
+    """Force-refresh with the canonical selected-state lock already held.
+
+    ``binding`` was captured together with ``token`` before the fetch. The
+    response is revalidated against it before ANY mutation (spec 4.6)."""
     try:
         api = c._fetch_oauth_usage(token=token, timeout_seconds=timeout_seconds)
     except RefreshUsageRateLimitError as exc:
+        if not _revalidate_selected_credential(c, binding):
+            return _RefreshUsageResult(
+                status="record_failed",
+                reason="the selected identity changed during the fetch")
         # Force-refresh bypasses the backfill/backoff GATE (user asked for
         # it), but a resulting 429 MUST advance the shared backoff deadline
         # so automatic polling still honors it (spec §4).
@@ -698,6 +712,14 @@ def _refresh_usage_inproc_locked(
                 f"ignoring unparseable five_hour fields: {exc}"
             )
 
+    # #825/#832 spec 4.6: publication proceeds only when the second stable
+    # binding equals the captured one AND the selected credential is still
+    # the one the response was fetched with. A->B, a torn read or a changed
+    # credential discards the response before any mutation.
+    if not _revalidate_selected_credential(c, binding):
+        return _RefreshUsageResult(
+            status="record_failed",
+            reason="the selected identity or credential changed during the fetch")
     record_args = argparse.Namespace(
         percent=seven_pct,
         resets_at=str(seven_resets_epoch),
@@ -716,6 +738,7 @@ def _refresh_usage_inproc_locked(
             *({"fiveHour"} if five_pct is not None else set()),
         },
         lock_held=True,
+        binding=binding,
         # #583 S2 §5.3: the dashboard calls this INSIDE its own `sync_lock`
         # while servicing a `refresh=1`, so an inner nudge would enqueue a
         # second rebuild for work already underway. `cmd_refresh_usage`
@@ -918,11 +941,46 @@ def cmd_refresh_usage(args: argparse.Namespace) -> int:
 # Hook-tick OAuth refresh path
 # =========================================================================
 
-def _hook_tick_oauth_skip_status(c, *, throttle_seconds: float) -> str | None:
-    """Return the automatic-refresh suppression reason under selected lock."""
+def _capture_selected_credential(c):
+    """``(binding, token)`` captured together under the selected-state lock
+    (#825/#832 spec 4.6), or ``(None, None)`` when the identity is torn or
+    changes across the credential read. The identity is read on both sides of
+    the credential read so a credential is never paired with an identity it
+    was not selected under; the binding then carries the credential's salted
+    digest (never the credential). ``(binding, None)``: stable, no token."""
+    import _cctally_statusline as _sl
+
+    first = _sl._capture_binding()
+    if first is None:
+        return None, None
+    token = c._resolve_oauth_token()
+    second = _sl._capture_binding()
+    if second is None or second != first:
+        return None, None
+    if not token:
+        return first, None
+    bound = _sl._capture_binding(credential=token)
+    if bound is None or bound.without_credential() != first:
+        return None, None
+    return bound, token
+
+
+def _revalidate_selected_credential(c, binding) -> bool:
+    """The second stable read after the fetch: the identity AND the selected
+    credential must still be exactly the captured ones. A->B->A is accepted
+    only when the credential is unchanged (the response was fetched with A's
+    captured credential)."""
+    current, token = _capture_selected_credential(c)
+    return current is not None and bool(token) and current == binding
+
+
+def _hook_tick_oauth_skip_status(c, *, throttle_seconds: float, binding) -> str | None:
+    """Return the automatic-refresh suppression reason under selected lock.
+    Repair need and selected freshness are the captured account's own."""
     now_epoch = int(time.time())
-    needs_repair = c._authoritative_repair_required(now_epoch=now_epoch)
-    obs_age = c._statusline_observe_age_seconds()
+    needs_repair = c._authoritative_repair_required(
+        now_epoch=now_epoch, binding=binding)
+    obs_age = c._statusline_observe_age_seconds(binding=binding)
     if (not needs_repair
             and obs_age < float(_cctally_core.OAUTH_BACKFILL_STALE_SECONDS)):
         return f"skipped(statusline-fresh:{int(obs_age)}s)"
@@ -995,9 +1053,6 @@ def _hook_tick_oauth_refresh(
     `five_hour`) on success, or None on any non-ok branch.
     """
     c = _cctally()
-    token = c._resolve_oauth_token()
-    if not token:
-        return "skipped-no-token", None
     if throttle_seconds is None:
         try:
             throttle_seconds = float(_get_oauth_usage_config(c.load_config())["throttle_seconds"])
@@ -1010,8 +1065,15 @@ def _hook_tick_oauth_refresh(
     # during network I/O.
     try:
         with c._selected_state_lock():
+            # #825/#832 spec 4.6: capture the selected credential and its
+            # binding together before the fetch.
+            binding, token = _capture_selected_credential(c)
+            if binding is None:
+                return "err(identity-unstable)", None
+            if not token:
+                return "skipped-no-token", None
             skipped = _hook_tick_oauth_skip_status(
-                c, throttle_seconds=throttle_seconds,
+                c, throttle_seconds=throttle_seconds, binding=binding,
             )
     except OSError:
         return "err(record-usage=exc)", None
@@ -1023,6 +1085,8 @@ def _hook_tick_oauth_refresh(
     except RefreshUsageRateLimitError as exc:
         try:
             with c._selected_state_lock():
+                if not _revalidate_selected_credential(c, binding):
+                    return "err(identity-changed)", None
                 c._oauth_backoff_register_429(
                     retry_after_deadline=getattr(
                         exc, "retry_after_deadline", None,
@@ -1048,9 +1112,15 @@ def _hook_tick_oauth_refresh(
     deferred = []
     try:
         with c._selected_state_lock():
+            # Revalidate under the reacquired lock BEFORE any mutation: the
+            # response is published only for the identity AND credential it
+            # was fetched with (spec 4.6).
+            if not _revalidate_selected_credential(c, binding):
+                return "err(identity-changed)", None
             skipped = _hook_tick_oauth_skip_status(
                 c,
                 throttle_seconds=throttle_seconds,
+                binding=binding,
             )
             if skipped is not None:
                 return skipped, None
@@ -1058,6 +1128,7 @@ def _hook_tick_oauth_refresh(
                 record_args,
                 axes,
                 lock_held=True,
+                binding=binding,
                 nudge_sink=lambda: deferred.append(1),
             )
             if authoritative.status != "ok":

package/bin/_cctally_source_analytics.py

@@ -11,7 +11,7 @@ import sqlite3
 import sys
 from copy import copy
 from collections import defaultdict
-from typing import Iterable
+from typing import Callable, Iterable
 
 from _cctally_core import (
     _command_as_of,
@@ -19,7 +19,7 @@ from _cctally_core import (
     get_week_start_name,
     parse_iso_datetime,
 )
-from _cctally_cache import _codex_provider_roots
+from _cctally_cache import CodexHistoryUnavailable, _codex_provider_roots
 import _lib_accounts
 from _lib_codex_metadata import (
     DecodedCodexProjectMetadata,
@@ -153,18 +153,45 @@ _QUALIFIED_CODEX_ENTRIES_SQL = """
 
 
 def _qualified_codex_path_entries_sql(identity_count: int) -> str:
-    """Return the qualified read constrained to physical cache identities."""
+    """Return the qualified read constrained to physical cache identities.
+
+    Unordered: ``_load_qualified_codex_rows`` orders its rows (#901
+    Amendment 19 T1). Seeking ``idx_codex_entries_root_path`` cannot also
+    deliver the window read's order, so the old ``ORDER BY`` sorted every
+    entry of the dirty rollouts through a temp b-tree, and one rollout file is
+    not a bound.
+    """
     predicates = " OR ".join(
         "(entries.source_root_key = ? AND entries.source_path = ?)"
         for _ in range(identity_count)
     )
-    return _QUALIFIED_CODEX_ENTRIES_SQL.replace(
+    head, order_by, _tail = _QUALIFIED_CODEX_ENTRIES_SQL.partition(
+        "     ORDER BY entries.timestamp_utc ASC")
+    if not order_by:
+        raise RuntimeError(
+            "_QUALIFIED_CODEX_ENTRIES_SQL no longer carries its ORDER BY")
+    return head.replace(
         "INDEXED BY idx_codex_entries_ts_root_conversation",
         "INDEXED BY idx_codex_entries_root_path",
-    ).replace(
-        "     ORDER BY entries.timestamp_utc ASC",
-        f"       AND ({predicates})\n     ORDER BY entries.timestamp_utc ASC",
-    )
+    ) + f"       AND ({predicates})\n"
+
+
+def _qualified_codex_row_order(row) -> tuple:
+    """The window read's ``ORDER BY entries.timestamp_utc,
+    entries.source_root_key, entries.conversation_key, entries.id`` as a Python
+    key: NULL first, text by UTF-8 bytes (SQLite's BINARY collation). ``id``
+    is unique, so the order is total."""
+    def collate(value):
+        if value is None:
+            return (0, b"")
+        if isinstance(value, str):
+            return (2, value.encode("utf-8"))
+        if isinstance(value, (bytes, bytearray, memoryview)):
+            return (3, bytes(value))
+        return (1, value)
+
+    return (collate(row[0]), collate(row[3]), collate(row[4]),
+            collate(row[16]))
 
 
 #: The thread leg of the decode inventory (§4.3 item 1). ``{threads}`` is the
@@ -1184,6 +1211,32 @@ def _require_joined_metadata(row: sqlite3.Row) -> tuple[str, str]:
     return root_key, conversation_key
 
 
+def _load_qualified_codex_rows(
+    conn: sqlite3.Connection, start: dt.datetime, end: dt.datetime,
+    identities: "tuple[tuple[str, str], ...] | None",
+) -> tuple:
+    """The qualified accounting rows of ``[start, end)``, in
+    ``(timestamp_utc, source_root_key, conversation_key, id)`` order: the whole
+    window, or only the given dirty ``(root, path)`` identities.
+
+    The window read streams in that order from
+    ``idx_codex_entries_ts_root_conversation``. The dirty-path read seeks
+    ``idx_codex_entries_root_path`` instead and is ordered here, in Python, over
+    the rows it returns whole anyway (#901 Amendment 19 T1); the rows keep the
+    connection's row factory.
+    """
+    params: tuple[object, ...] = (
+        start.astimezone(UTC).isoformat(), end.astimezone(UTC).isoformat(),
+        *(value for identity in (identities or ()) for value in identity),
+    )
+    if identities is None:
+        return tuple(conn.execute(_QUALIFIED_CODEX_ENTRIES_SQL, params))
+    rows = conn.execute(
+        _qualified_codex_path_entries_sql(len(identities)), params).fetchall()
+    rows.sort(key=_qualified_codex_row_order)
+    return tuple(rows)
+
+
 def load_qualified_codex_entries(
     start: dt.datetime,
     end: dt.datetime,
@@ -1194,6 +1247,7 @@ def load_qualified_codex_entries(
     cache_conn: sqlite3.Connection | None = None,
     source_identities: Iterable[tuple[str, str]] | None = None,
     inventory: "CodexThreadMetadataInventory | None" = None,
+    _on_sync: "Callable[[object], object] | None" = None,
 ) -> tuple[QualifiedCodexEntry, ...]:
     """Load exactly one bounded, root-qualified Codex accounting read.
 
@@ -1240,16 +1294,10 @@ def load_qualified_codex_entries(
             )
             if stats.lock_contended:
                 raise QualifiedMetadataUnavailable("Codex qualified project metadata is unavailable")
+            if _on_sync is not None:
+                _on_sync(stats)
         conn.row_factory = sqlite3.Row
-        sql = (
-            _QUALIFIED_CODEX_ENTRIES_SQL if identities is None
-            else _qualified_codex_path_entries_sql(len(identities))
-        )
-        params: tuple[object, ...] = (
-            start.astimezone(UTC).isoformat(), end.astimezone(UTC).isoformat(),
-            *(value for identity in (identities or ()) for value in identity),
-        )
-        rows = tuple(conn.execute(sql, params))
+        rows = _load_qualified_codex_rows(conn, start, end, identities)
         # §4.3: one inventory read per build. The caller supplies the build's
         # own inventory when it has one; its alias leg is unbounded, which is a
         # SUPERSET of the `paths=identities` leg this read would have loaded,
@@ -1903,6 +1951,8 @@ def _source_entries(
     if qualified:
         values = load_qualified_codex_entries(
             start, query_end, speed=speed, sync=sync, group=group,
+            _on_sync=_cctally()._load_sibling(
+                "_cctally_cache")._warn_compressed_unreadable_once,
         )
         return assign_collision_safe_project_labels(values)
     return load_codex_accounting_entries(
@@ -3228,6 +3278,8 @@ def cmd_source_diff(args: object) -> int:
         except ValueError as exc:
             print(f"diff: {exc}", file=sys.stderr)
             return 2
+        except CodexHistoryUnavailable:
+            raise
         except RuntimeError as exc:
             print(str(exc), file=sys.stderr)
             return 1
@@ -3261,12 +3313,16 @@ def cmd_source_diff(args: object) -> int:
     except QualifiedMetadataUnavailable:
         try:
             entries = _source_entries(entry_args, start, end, qualified=False)
+        except CodexHistoryUnavailable:
+            raise
         except RuntimeError:
             return _emit_source_result(
                 args, SourceResult("codex", "unavailable", None, (QUALIFIED_METADATA_WARNING,)),
                 diff=True, claude=claude,
             )
         metadata_available = False
+    except CodexHistoryUnavailable:
+        raise
     except RuntimeError:
         return _emit_source_result(
             args, SourceResult("codex", "unavailable", None, (QUALIFIED_METADATA_WARNING,)),
@@ -3316,6 +3372,8 @@ def cmd_source_range_cost(args: object) -> int:
             args, start, end, qualified=bool(getattr(args, "project", None)),
             inclusive_end=True,
         )
+    except CodexHistoryUnavailable:
+        raise
     except (QualifiedMetadataUnavailable, RuntimeError):
         return _emit_source_result(
             args, SourceResult("codex", "unavailable", None, (QUALIFIED_METADATA_WARNING,)),
@@ -3372,6 +3430,8 @@ def cmd_source_cache_report(args: object) -> int:
             )
         try:
             entries = _source_entries(entry_args, start, end, qualified=False)
+        except CodexHistoryUnavailable:
+            raise
         except RuntimeError:
             return _emit_source_result(
                 args, SourceResult("codex", "unavailable", None, (QUALIFIED_METADATA_WARNING,)),
@@ -3432,6 +3492,10 @@ def cmd_source_report(args: object) -> int:
             c.reconcile_codex_quota_projection(now=as_of)
             force_direct_accounting = bool(getattr(sync_stats, "lock_contended", False))
             already_synced = not force_direct_accounting
+            if already_synced:
+                cache_mod._warn_compressed_unreadable_once(sync_stats)
+        except CodexHistoryUnavailable:
+            raise
         except (OSError, sqlite3.Error, RuntimeError):
             return _emit_source_result(
                 args, build_codex_report_result((), (), as_of=as_of, quota_available=False),
@@ -3458,6 +3522,8 @@ def cmd_source_report(args: object) -> int:
                 args, start, end, qualified=False, sync=not already_synced,
                 force_direct=force_direct_accounting,
             )
+        except CodexHistoryUnavailable:
+            raise
         except RuntimeError:
             return _emit_source_result(
                 args, build_codex_report_result((), (), as_of=as_of, quota_available=False),