cctally 1.112.0 → 1.114.0

package/bin/_cctally_milestone_history.py

@@ -817,22 +817,75 @@ def _load_codex_cycles(stats_conn, root_keys, *, include_orphaned=False,
     if not roots:
         return []
     placeholders = ",".join("?" for _ in roots)
-    orphan_clause = "" if include_orphaned else "AND orphaned_at IS NULL "
-    # `quota_window_blocks.account_key` is NOT NULL DEFAULT 'unattributed'.
-    account_clause = "" if account_key is None else "AND account_key=? "
-    account_params = () if account_key is None else (account_key,)
     # BEFORE the SQL, per #496 S5b section 4.7.
     assert_projection_readable(stats_conn)
+    # #901 (I1/G1, Amendment 1 item 2, 1b and 1c): stream every weekly block
+    # in reset order through an epoch-1017 partial expression index. Unpinned,
+    # the planner seeks a root-leading index and sorts the whole weekly
+    # history through a temp b-tree (measured for every root/account/orphan
+    # variant).
+    #
+    # The ORDER BY names the tie order explicitly: `_canonicalize_codex_
+    # cluster` keeps the first maximal member, so rows tied on one reset
+    # second must arrive in the order the pre-#901 sorter received them, and
+    # that order came from the legacy plan, which differs by root count
+    # (Amendment 1c, measured on SQLite 3.53.4). With two or more roots the
+    # legacy plan scanned `idx_quota_blocks_active`, whose order includes
+    # orphaned_at; with exactly one it seeked the UNIQUE autoindex, which has
+    # no orphaned_at (its one exception, an account-scoped live-only read, has
+    # a constant orphaned_at, so the two orders agree there). The branch
+    # counts the NORMALIZED roots above, so duplicate or empty arguments take
+    # the single-root order, exactly as they took the single-root plan.
+    #
+    # Each branch pins its own index only when that index exists; on an
+    # older index the statement stays unpinned with ordinary predicates and
+    # the same explicit order. Under the pin, the root, account and orphan
+    # predicates carry a unary `+`: SQLite treats a column that a WHERE term
+    # fixes but the scan does not seek as satisfying its ORDER BY term and
+    # then cannot match that index column, so the later ORDER BY terms fell
+    # back to a tie-run sorter ("USE TEMP B-TREE FOR LAST n TERMS OF ORDER
+    # BY", measured). Limitation: a unary `+` keeps the operand's value but
+    # removes its column affinity. That is equivalent here ONLY because these
+    # comparisons are TEXT against TEXT: the root keys are filtered to `str`
+    # above, `account_key` is a TEXT key bound as a string, and `IS NULL`
+    # involves no affinity. It is not an equivalence for numeric parameters or
+    # any other affinity-dependent comparison; a caller binding a non-string
+    # account key would need the unpinned predicate.
+    if len(roots) == 1:
+        order_index = "idx_quota_blocks_weekly_single_root_order"
+        order_by = (
+            "ORDER BY unixepoch(resets_at_utc) DESC, source_root_key ASC, "
+            "account_key ASC, logical_limit_key ASC, observed_slot ASC, "
+            "resets_at_utc ASC")
+    else:
+        order_index = "idx_quota_blocks_weekly_reset_order"
+        order_by = (
+            "ORDER BY unixepoch(resets_at_utc) DESC, source_root_key ASC, "
+            "account_key ASC, orphaned_at ASC, logical_limit_key ASC, "
+            "observed_slot ASC, resets_at_utc ASC")
+    pinned = stats_conn.execute(
+        "SELECT 1 FROM sqlite_master WHERE type='index' AND name=?",
+        (order_index,),
+    ).fetchone() is not None
+    order_hint = f"INDEXED BY {order_index} " if pinned else ""
+    plus = "+" if pinned else ""
+    orphan_clause = (
+        "" if include_orphaned else f"AND {plus}orphaned_at IS NULL ")
+    # `quota_window_blocks.account_key` is NOT NULL DEFAULT 'unattributed'.
+    account_clause = (
+        "" if account_key is None else f"AND {plus}account_key=? ")
+    account_params = () if account_key is None else (account_key,)
     rows = stats_conn.execute(
         "SELECT source_root_key, logical_limit_key, observed_slot, "
         "       window_minutes, limit_id, limit_name, account_key, "
         "       resets_at_utc, nominal_start_at_utc, current_percent "
         "FROM quota_window_blocks "
+        f"{order_hint}"
         "WHERE source='codex' AND window_minutes=10080 "
         f"{orphan_clause}"
         f"{account_clause}"
-        f"AND source_root_key IN ({placeholders}) "
-        "ORDER BY unixepoch(resets_at_utc) DESC",
+        f"AND {plus}source_root_key IN ({placeholders}) "
+        f"{order_by}",
         (*account_params, *roots),
     ).fetchall()
 
@@ -952,10 +1005,20 @@ def _codex_five_hour_rows(stats_conn, cyc, *, include_orphaned=False) -> list:
     The widening is one-directional on purpose: an unattributed CYCLE never
     picks up another account's identified 5h rows, which would violate #341's
     never-combine rule.
+
+    #901 Amendment 19 T1 (spec I1): the rows are returned whole, so they are
+    ordered here, where ``ORDER BY unixepoch(nominal_start_at_utc)`` sorted
+    them through a temp b-tree. Each physical 5h block surfaces as many
+    jittered rows, so one weekly cycle is not a bound. The key is SQLite's own
+    ``unixepoch()`` of each distinct start spelling (a table-free scalar
+    query, memoized), so it is the old key exactly, and the rows keep the
+    caller's row factory. ``list.sort`` is stable over the same scan SQLite's
+    in-memory sorter consumed, so rows tied on one start second keep their
+    order. The WHERE clause admits no NULL start.
     """
     orphan_clause = "" if include_orphaned else "AND orphaned_at IS NULL "
     assert_projection_readable(stats_conn)
-    return stats_conn.execute(
+    rows = stats_conn.execute(
         "SELECT source_root_key, logical_limit_key, observed_slot, "
         "       window_minutes, limit_id, limit_name, account_key, "
         "       resets_at_utc, nominal_start_at_utc, current_percent "
@@ -964,12 +1027,22 @@ def _codex_five_hour_rows(stats_conn, cyc, *, include_orphaned=False) -> list:
         f"{orphan_clause}"
         "AND source_root_key=? AND account_key IN (?,?) "
         "AND unixepoch(nominal_start_at_utc) < unixepoch(?) "
-        "AND unixepoch(resets_at_utc) > unixepoch(?) "
-        "ORDER BY unixepoch(nominal_start_at_utc) ASC",
+        "AND unixepoch(resets_at_utc) > unixepoch(?)",
         (cyc.root, cyc.account_key, UNATTRIBUTED,
          cyc.end.astimezone(UTC).isoformat(),
          cyc.start.astimezone(UTC).isoformat()),
     ).fetchall()
+    epochs: dict = {}
+
+    def start_epoch(row):
+        text = row[8]
+        if text not in epochs:
+            epochs[text] = stats_conn.execute(
+                "SELECT unixepoch(?)", (text,)).fetchone()[0]
+        return epochs[text]
+
+    rows.sort(key=start_epoch)
+    return rows
 
 
 def _codex_five_hour_clusters(stats_conn, cyc, *, include_orphaned=False) -> list:

package/bin/_cctally_percent_breakdown.py

@@ -262,7 +262,8 @@ def observation_gap_marginal_cell(marginal_value: Any, *, withheld: bool) -> str
     table already says what the code means.
     """
     if marginal_value is not None:
-        return f"${float(marginal_value):.6f}"
+        c = _cctally()
+        return c._fmt_usd_accounting(float(marginal_value), 6)
     return OBSERVATION_GAP_CAUSE if withheld else "n/a"
 
 
@@ -315,7 +316,7 @@ def _render_percent_breakdown_terminal(
         rows.append([
             str(idx),
             f"{percent}%",
-            f"${cumulative:.6f}",
+            c._fmt_usd_accounting(cumulative, 6),
             c.observation_gap_marginal_cell(
                 marginal_value, withheld=(idx - 1) in withheld),
             f"{float(five_hour_value):.0f}%" if five_hour_value is not None else "n/a",

package/bin/_cctally_quota.py

@@ -378,7 +378,14 @@ PROJECTION_DYNAMIC_READ_SITES: "dict[str, int]" = {
     # so no `PROJECTION_DYNAMIC_READ_ACTIONS` classification applies.
     "_lib_codex_conversation_query.py": 1,
     "_lib_conversation_query.py": 1,
-    "_lib_conversation_retention.py": 2,
+    # Four, not two, since #901's paced retention (`f2077468a`, spec §5.4):
+    # `group_row_count` counts the rows a deletion's reservation charges with
+    # `SELECT COUNT(*) FROM {table}` in its two shapes (by identity key, and by
+    # source path for a NULL-identity group). `{table}` comes from
+    # `_GROUP_TABLES`, a hardcoded pair (`conversation_messages`,
+    # `codex_conversation_events`); neither is a quota projection table, so no
+    # `PROJECTION_DYNAMIC_READ_ACTIONS` classification applies.
+    "_lib_conversation_retention.py": 4,
     # Five: `_target_has_cursor_gap` selects one of the two accounting source
     # tables, `_conversation_source_paths` and `_conversation_target_risk`
     # select one of the two transcript source tables, and #769 S6's
@@ -1307,6 +1314,107 @@ def _codex_quota_required_text_present(values: Iterable[object]) -> bool:
         value is None or not str(value).strip() for value in values)
 
 
+# ── #901 W1: reading the maintained raw-partition maxima (spec §5.1, T1) ─────
+
+#: The objects the latest-only read and the general loader's plan pins depend
+#: on, probed together once per load.
+_CODEX_QUOTA_READ_OBJECTS = (
+    "codex_quota_partition_latest", "idx_qws_partition_capture",
+    "trg_qws_latest_ins", "trg_qws_latest_del", "trg_qws_latest_upd",
+    "idx_qws_codex_load_order", "idx_qws_physical_group",
+)
+_CODEX_QUOTA_SUMMARY_OBJECTS = frozenset(_CODEX_QUOTA_READ_OBJECTS[:5])
+
+#: The latest path's SELECT list over the summary join, aliased so every name
+#: the interpretation loop reads resolves exactly as it does on the window path.
+_CODEX_QUOTA_LATEST_SELECT = ", ".join(
+    f"q.{column} AS {column}" for column in (
+        "source", "source_root_key", "source_path", "line_offset",
+        "captured_at_utc", "observed_slot", "logical_limit_key", "limit_id",
+        "limit_name", "window_minutes", "used_percent", "resets_at_utc",
+        "plan_type", "individual_limit_json", "reached_type",
+        "observed_model", "account_key", "canonical_resets_at_utc",
+    ))
+
+
+def _codex_quota_read_objects(conn: sqlite3.Connection) -> "frozenset[str]":
+    """Which of the read-path objects this store carries (one probe)."""
+    placeholders = ",".join("?" for _ in _CODEX_QUOTA_READ_OBJECTS)
+    return frozenset(
+        str(row[0]) for row in conn.execute(
+            f"SELECT name FROM sqlite_master WHERE name IN ({placeholders})",
+            _CODEX_QUOTA_READ_OBJECTS)
+    )
+
+
+def _codex_quota_latest_summary_ready(read_objects: "frozenset[str]") -> bool:
+    """True when the summary and everything that maintains it are present.
+
+    The five objects are created and bootstrapped in one transaction
+    (``_cctally_db._apply_codex_quota_latest_summary``), and SQLite drops a
+    table's triggers and indexes only together with the table, so their joint
+    presence means the summary has been maintained since its bootstrap.
+    Anything less — a read-only open of a store migration 047 has not reached,
+    or a legacy shape — keeps the window query, which selects the same rows.
+    """
+    return _CODEX_QUOTA_SUMMARY_OBJECTS <= read_objects
+
+
+def _sqlite_order_value(value) -> tuple:
+    """Rank a stored value the way SQLite's BINARY ORDER BY ranks it.
+
+    NULL first, then numbers (integers and reals compared numerically), then
+    text in code-point order (which is UTF-8 byte order), then blobs.
+    """
+    if value is None:
+        return (0, 0)
+    if isinstance(value, (int, float)):
+        return (1, value)
+    if isinstance(value, str):
+        return (2, value)
+    return (3, bytes(value))
+
+
+def _codex_quota_latest_rows(
+    conn: sqlite3.Connection, requested: "set[str] | None",
+) -> "list[sqlite3.Row]":
+    """Every valid physical row tied at its raw partition's maximum second.
+
+    One scan of the summary (one row per retained raw partition) and one
+    ``idx_qws_partition_capture`` seek per partition. ``CROSS JOIN`` fixes the
+    summary as the outer loop and ``INDEXED BY`` the inner seek, so the plan
+    cannot drift into walking the snapshots. The window path's ORDER BY
+    (``source_root_key, captured_at_utc, resets_at_utc, source_path,
+    line_offset``) is reproduced here in Python over this bounded set, so the
+    rows reach the interpretation loop in exactly the order they did before.
+    """
+    import _cctally_db
+
+    on = " AND ".join(
+        f"{expr} = s.{key}" for key, expr in zip(
+            _cctally_db._CODEX_QUOTA_LATEST_KEY_COLUMNS,
+            _cctally_db._codex_quota_latest_key_exprs("q.")))
+    sql = (
+        f"SELECT {_CODEX_QUOTA_LATEST_SELECT}"
+        " FROM codex_quota_partition_latest AS s"
+        " CROSS JOIN quota_window_snapshots AS q"
+        " INDEXED BY idx_qws_partition_capture"
+        f" ON {on} AND unixepoch(q.captured_at_utc) = s.latest_capture_epoch"
+        f" WHERE {_cctally_db._codex_quota_latest_validity_sql('q.')}"
+    )
+    params: "list[object]" = []
+    if requested is not None:
+        sql += (" AND s.k_source_root_key IN ("
+                + ",".join("quote(?)" for _ in requested) + ")")
+        params.extend(sorted(requested))
+    rows = conn.execute(sql, params).fetchall()
+    rows.sort(key=lambda row: tuple(
+        _sqlite_order_value(row[name]) for name in (
+            "source_root_key", "captured_at_utc", "resets_at_utc",
+            "source_path", "line_offset")))
+    return rows
+
+
 # --------------------------------------------------------------------------
 # #500 §6.4 — the fold-time operator attribution overlay
 # --------------------------------------------------------------------------
@@ -1449,6 +1557,56 @@ def _codex_attribution_stored_limit_keys(logical_limit_key: str) -> "list[str]":
     )
 
 
+def _codex_shard_axis_filter(axes: tuple) -> "tuple[str, tuple]":
+    """The WHERE clause and parameters selecting one attribution axis shard.
+
+    Rows are STORED under whichever length and key spelling the provider sent,
+    so the filter enumerates every equivalent rather than snapping in SQL —
+    which it could not do and still seek the index.
+    """
+    root_key, limit_key, slot, window_minutes = axes
+    limit_keys = _codex_attribution_stored_limit_keys(limit_key)
+    minutes = sorted({
+        int(value)
+        for value in codex_snap_equivalent_window_minutes(window_minutes)
+    })
+    axis_clause = (
+        " WHERE source='codex' AND source_root_key = ?"
+        "   AND logical_limit_key IN ("
+        + ",".join("?" * len(limit_keys)) + ")"
+        "   AND observed_slot = ?"
+        "   AND window_minutes IN (" + ",".join("?" * len(minutes)) + ")"
+    )
+    return axis_clause, (root_key, *limit_keys, slot, *minutes)
+
+
+def _codex_witness_anchors(
+    conn: sqlite3.Connection, anchor_expr: str, axis_clause: str,
+    axis_params: tuple, witnesses: "Sequence[str]",
+) -> "set[str]":
+    """Pass 1: the canonical anchors of the groups a witness reset reaches.
+
+    #901 Amendment 19 T1 (spec I1): the set is built in Python from the
+    streamed rows. ``SELECT DISTINCT`` sorted the anchors of every capture
+    carrying a witness reset through a temp b-tree — a window group's whole
+    activity — and one attribution axis shard is not a bound. The result is
+    the same set; the Python set holds one entry per distinct anchor.
+    """
+    pass_one = (
+        f"SELECT {anchor_expr} AS anchor"
+        "  FROM quota_window_snapshots"
+        + axis_clause
+        + "   AND unixepoch(resets_at_utc) IN ("
+        + ",".join("unixepoch(?)" for _ in witnesses) + ")"
+    )
+    anchors: "set[str]" = set()
+    for row in conn.execute(pass_one, (*axis_params, *witnesses)):
+        if row[0] is None or not str(row[0]).strip():
+            continue
+        anchors.add(str(row[0]))
+    return anchors
+
+
 def _load_codex_window_group_evidence(
     conn: sqlite3.Connection, assertions: "Sequence[Mapping[str, object]]",
 ) -> "tuple[_wa.WindowGroup, ...]":
@@ -1557,36 +1715,10 @@ def _load_codex_window_group_evidence(
     witness_texts: "dict[str, str]" = {}
     model_scoped: "dict[tuple, bool]" = {}
     for axes, shard_witnesses in sorted(shards.items()):
-        root_key, limit_key, slot, window_minutes = axes
-        # Rows are STORED under whichever length and key spelling the provider
-        # sent, so the filter enumerates every equivalent rather than snapping
-        # in SQL — which it could not do and still seek the index.
-        limit_keys = _codex_attribution_stored_limit_keys(limit_key)
-        minutes = sorted({
-            int(value)
-            for value in codex_snap_equivalent_window_minutes(window_minutes)
-        })
-        witnesses = sorted(shard_witnesses)
-        axis_clause = (
-            " WHERE source='codex' AND source_root_key = ?"
-            "   AND logical_limit_key IN ("
-            + ",".join("?" * len(limit_keys)) + ")"
-            "   AND observed_slot = ?"
-            "   AND window_minutes IN (" + ",".join("?" * len(minutes)) + ")"
-        )
-        axis_params = (root_key, *limit_keys, slot, *minutes)
-        anchors: "set[str]" = set()
-        pass_one = (
-            f"SELECT DISTINCT {anchor_expr} AS anchor"
-            "  FROM quota_window_snapshots"
-            + axis_clause
-            + "   AND unixepoch(resets_at_utc) IN ("
-            + ",".join("unixepoch(?)" for _ in witnesses) + ")"
-        )
-        for row in conn.execute(pass_one, (*axis_params, *witnesses)):
-            if row[0] is None or not str(row[0]).strip():
-                continue
-            anchors.add(str(row[0]))
+        axis_clause, axis_params = _codex_shard_axis_filter(axes)
+        anchors = _codex_witness_anchors(
+            conn, anchor_expr, axis_clause, axis_params,
+            sorted(shard_witnesses))
         if not anchors:
             continue
         # ONE SHARD PER GROUP, for the reason the loader states at its own group
@@ -1983,6 +2115,16 @@ def load_codex_quota_observations(
     and therefore the same verdict as the full load, on a store where the full
     load meant interpreting 266,337 rows to answer a question about 608 of them.
 
+    #901 W1 (spec §5.1, T1): on a store carrying cache migration 047's objects
+    the candidate rows come from ``codex_quota_partition_latest``, a
+    trigger-maintained table holding every raw partition's maximum whole-second
+    capture, joined back to every physical row tied at that second through
+    ``idx_qws_partition_capture``. The window query described below stays as the
+    fallback for a store without those objects (a read-only open before the
+    migration ran, or a legacy shape); both select exactly the same rows, and
+    tests/test_901_partition_latest_maintenance.py holds the summary to the
+    frozen pre-#901 window query after every writer class.
+
     SQL narrows to the rows that can possibly win, and Python still decides. The
     partition is the RAW spelling of every column the interpretation reads
     (root, limit key, slot, window minutes, limit id and name, observed model,
@@ -2106,6 +2248,8 @@ def load_codex_quota_observations(
             "canonical_resets_at_utc" if has_anchor
             else "NULL AS canonical_resets_at_utc"
         )
+        # #901: one probe for the summary and the two plan-pin indexes.
+        read_objects = _codex_quota_read_objects(conn)
         # Public #5: the model is read from THIS table alone. There used to be a
         # COALESCE onto the nearest preceding `codex_session_entries.model` at
         # or before the snapshot's byte offset, for rows written before the
@@ -2129,9 +2273,26 @@ def load_codex_quota_observations(
             model_expr=model_expr, account_expr=account_expr,
             anchor_expr=anchor_expr,
         )
+        # #901 W3 (spec §5.1, T1/T3): pin the plan. Unpinned, the planner sorts
+        # an unbounded or multi-root load through a temp b-tree via the
+        # `source` autoindex, and the load-order index hijacks the projector's
+        # per-group shards into a whole-root walk (both measured). A bounded
+        # `max_rows` read keeps its plan: its sorter holds at most `max_rows`
+        # rows. Each pin is applied only when its index exists, so a store
+        # without them keeps today's SQL.
+        if group_filter is not None:
+            table_hint = (
+                " INDEXED BY idx_qws_physical_group"
+                if "idx_qws_physical_group" in read_objects else "")
+        elif physical_signatures is None and max_rows is not None:
+            table_hint = ""
+        else:
+            table_hint = (
+                " INDEXED BY idx_qws_codex_load_order"
+                if "idx_qws_codex_load_order" in read_objects else "")
         sql = f"""
             SELECT {select_list}
-              FROM quota_window_snapshots
+              FROM quota_window_snapshots{table_hint}
              WHERE source='codex' AND source_root_key IS NOT NULL
         """
         params: list[object] = []
@@ -2271,9 +2432,16 @@ def load_codex_quota_observations(
             shard_sql = f"{head}{clause}{order_by}"
             for group in group_filter:
                 shards.append((shard_sql, (*params, *group)))
+        if latest_per_identity and _codex_quota_latest_summary_ready(
+                read_objects):
+            # #901 W1 (spec §5.1, T1): the maintained summary yields exactly
+            # the rows the window query above selects, without sorting history.
+            row_source = _codex_quota_latest_rows(conn, requested)
+        else:
+            row_source = _iter_shard_rows(conn, shards)
         result: list[QuotaObservation] = []
         signature_tuples: dict[str, dict[str, list[tuple[object, ...]]]] = {}
-        for row in _iter_shard_rows(conn, shards):
+        for row in row_source:
             if not _codex_quota_required_text_present(
                 row[name] for name in _CODEX_QUOTA_REQUIRED_TEXT
             ):

package/bin/_cctally_record.py

@@ -6596,6 +6596,16 @@ def _record_dashboard_activity(provider: str, transcript_path: str) -> None:
         pass
 
 
+def _finalize_wal_checkpoints() -> None:
+    """The W9 finalizer before a detached worker's ``os._exit`` (#901 Q14)."""
+    try:
+        import _lib_wal_checkpoint
+
+        _lib_wal_checkpoint.finalize()
+    except Exception:  # noqa: BLE001 — a hook must exit 0 whatever happens
+        pass
+
+
 def cmd_hook_tick(args: argparse.Namespace) -> int:
     """Per-fire hook runtime (Section 3 of onboarding spec).
 
@@ -6831,6 +6841,10 @@ def cmd_hook_tick(args: argparse.Namespace) -> int:
         # Forked child: skip Python's atexit / argparse / cleanup paths
         # (they may try to flush already-redirected stdio handles).
         if forked:
+            # #901 W9 (Q14): `os._exit` skips `atexit`, so the checkpoint
+            # policy's finalizer runs here: one PASSIVE per armed store, then
+            # the keepers close (the last close drains the WAL).
+            _finalize_wal_checkpoints()
             os._exit(0)
         return 0
     # --explain mapping (Section 3 of spec)

package/bin/_cctally_rederive.py

@@ -90,6 +90,13 @@ def _fingerprint(value) -> str:
     return "sha256:" + hashlib.sha256(_canonical_bytes(value)).hexdigest()
 
 
+#: #874: the only tables the cache contract names. Inspecting any other table
+#: is not the planner's business, and on a virtual table whose module this
+#: SQLite build lacks (a legacy FTS5 index on a build without FTS5) even
+#: `PRAGMA table_info` raises before the contract could type the gap.
+_CONTRACT_TABLES = ("session_entries", "session_files", "codex_session_entries")
+
+
 def _cache_contract(cache_conn: sqlite3.Connection) -> dict[str, set[str]]:
     names = {
         row[0] for row in cache_conn.execute(
@@ -99,12 +106,18 @@ def _cache_contract(cache_conn: sqlite3.Connection) -> dict[str, set[str]]:
         name: {
             row[1] for row in cache_conn.execute(f"PRAGMA table_info({name})")
         }
-        for name in names
+        for name in _CONTRACT_TABLES
+        if name in names
     }
 
 
 def _cache_fingerprint(cache_conn: sqlite3.Connection, *, include_codex=False) -> str:
-    """Hash every cost-bearing Claude cache row and its project metadata."""
+    """Hash every cost-bearing Claude cache row and its project metadata.
+
+    The caller must have validated the cache contract first, including
+    ``validate_codex_cache_contract`` when ``include_codex`` is set (#874):
+    a missing column here is a raw sqlite error, not a typed data gap.
+    """
     entry_columns = (
         "source_path", "line_offset", "timestamp_utc", "model", "input_tokens",
         "output_tokens", "cache_create_tokens", "cache_read_tokens",
@@ -123,10 +136,6 @@ def _cache_fingerprint(cache_conn: sqlite3.Connection, *, include_codex=False) -
     ]
     basis = {"sessionEntries": entries, "sessionFiles": files}
     if include_codex:
-        if "codex_session_entries" not in _cache_contract(cache_conn):
-            raise _lib_rederive.RederiveDataGap(
-                "missing cache.db table codex_session_entries"
-            )
         basis["codexEntries"] = [
             list(row) for row in cache_conn.execute(
                 "SELECT source_path,line_offset,timestamp_utc,session_id,model,"
@@ -1088,15 +1097,20 @@ def plan_claude_usage(
             + ",".join(report.unclassified_evt_kinds)
             + " op=" + ",".join(report.unclassified_op_kinds)
         )
-    _lib_rederive.validate_claude_cache_contract(_cache_contract(cache_conn))
-    raw_records = _rederivable_raw_records(records)
-    _validate_cache_rows(cache_conn, raw_records)
+    tables = _cache_contract(cache_conn)
+    _lib_rederive.validate_claude_cache_contract(tables)
     codex_pricing_records = any(
         (record.get("payload") or {}).get("vendor") == "codex"
         and (record.get("payload") or {}).get("kind") == "budget"
         and (record.get("payload") or {}).get("_pricing")
         for record in records
     )
+    if codex_pricing_records:
+        # #874: the Codex fingerprint and re-pricing read the Codex cache only
+        # here, so its contract is required only here, and before any reader.
+        _lib_rederive.validate_codex_cache_contract(tables)
+    raw_records = _rederivable_raw_records(records)
+    _validate_cache_rows(cache_conn, raw_records)
     cache_fingerprint = _cache_fingerprint(
         cache_conn, include_codex=codex_pricing_records,
     )

package/bin/_cctally_source_analytics.py

@@ -153,18 +153,45 @@ _QUALIFIED_CODEX_ENTRIES_SQL = """
 
 
 def _qualified_codex_path_entries_sql(identity_count: int) -> str:
-    """Return the qualified read constrained to physical cache identities."""
+    """Return the qualified read constrained to physical cache identities.
+
+    Unordered: ``_load_qualified_codex_rows`` orders its rows (#901
+    Amendment 19 T1). Seeking ``idx_codex_entries_root_path`` cannot also
+    deliver the window read's order, so the old ``ORDER BY`` sorted every
+    entry of the dirty rollouts through a temp b-tree, and one rollout file is
+    not a bound.
+    """
     predicates = " OR ".join(
         "(entries.source_root_key = ? AND entries.source_path = ?)"
         for _ in range(identity_count)
     )
-    return _QUALIFIED_CODEX_ENTRIES_SQL.replace(
+    head, order_by, _tail = _QUALIFIED_CODEX_ENTRIES_SQL.partition(
+        "     ORDER BY entries.timestamp_utc ASC")
+    if not order_by:
+        raise RuntimeError(
+            "_QUALIFIED_CODEX_ENTRIES_SQL no longer carries its ORDER BY")
+    return head.replace(
         "INDEXED BY idx_codex_entries_ts_root_conversation",
         "INDEXED BY idx_codex_entries_root_path",
-    ).replace(
-        "     ORDER BY entries.timestamp_utc ASC",
-        f"       AND ({predicates})\n     ORDER BY entries.timestamp_utc ASC",
-    )
+    ) + f"       AND ({predicates})\n"
+
+
+def _qualified_codex_row_order(row) -> tuple:
+    """The window read's ``ORDER BY entries.timestamp_utc,
+    entries.source_root_key, entries.conversation_key, entries.id`` as a Python
+    key: NULL first, text by UTF-8 bytes (SQLite's BINARY collation). ``id``
+    is unique, so the order is total."""
+    def collate(value):
+        if value is None:
+            return (0, b"")
+        if isinstance(value, str):
+            return (2, value.encode("utf-8"))
+        if isinstance(value, (bytes, bytearray, memoryview)):
+            return (3, bytes(value))
+        return (1, value)
+
+    return (collate(row[0]), collate(row[3]), collate(row[4]),
+            collate(row[16]))
 
 
 #: The thread leg of the decode inventory (§4.3 item 1). ``{threads}`` is the
@@ -1184,6 +1211,32 @@ def _require_joined_metadata(row: sqlite3.Row) -> tuple[str, str]:
     return root_key, conversation_key
 
 
+def _load_qualified_codex_rows(
+    conn: sqlite3.Connection, start: dt.datetime, end: dt.datetime,
+    identities: "tuple[tuple[str, str], ...] | None",
+) -> tuple:
+    """The qualified accounting rows of ``[start, end)``, in
+    ``(timestamp_utc, source_root_key, conversation_key, id)`` order: the whole
+    window, or only the given dirty ``(root, path)`` identities.
+
+    The window read streams in that order from
+    ``idx_codex_entries_ts_root_conversation``. The dirty-path read seeks
+    ``idx_codex_entries_root_path`` instead and is ordered here, in Python, over
+    the rows it returns whole anyway (#901 Amendment 19 T1); the rows keep the
+    connection's row factory.
+    """
+    params: tuple[object, ...] = (
+        start.astimezone(UTC).isoformat(), end.astimezone(UTC).isoformat(),
+        *(value for identity in (identities or ()) for value in identity),
+    )
+    if identities is None:
+        return tuple(conn.execute(_QUALIFIED_CODEX_ENTRIES_SQL, params))
+    rows = conn.execute(
+        _qualified_codex_path_entries_sql(len(identities)), params).fetchall()
+    rows.sort(key=_qualified_codex_row_order)
+    return tuple(rows)
+
+
 def load_qualified_codex_entries(
     start: dt.datetime,
     end: dt.datetime,
@@ -1241,15 +1294,7 @@ def load_qualified_codex_entries(
             if stats.lock_contended:
                 raise QualifiedMetadataUnavailable("Codex qualified project metadata is unavailable")
         conn.row_factory = sqlite3.Row
-        sql = (
-            _QUALIFIED_CODEX_ENTRIES_SQL if identities is None
-            else _qualified_codex_path_entries_sql(len(identities))
-        )
-        params: tuple[object, ...] = (
-            start.astimezone(UTC).isoformat(), end.astimezone(UTC).isoformat(),
-            *(value for identity in (identities or ()) for value in identity),
-        )
-        rows = tuple(conn.execute(sql, params))
+        rows = _load_qualified_codex_rows(conn, start, end, identities)
         # §4.3: one inventory read per build. The caller supplies the build's
         # own inventory when it has one; its alias leg is unbounded, which is a
         # SUPERSET of the `paths=identities` leg this read would have loaded,