cctally 1.112.0 → 1.114.0

package/bin/_cctally_statusline.py

@@ -1314,6 +1314,21 @@ def _preliminary_decision() -> "_candidates.ReductionDecision | None":
     )
 
 
+def _finalize_wal_checkpoints() -> None:
+    """The W9 finalizer before a forked child's ``os._exit`` (#901 Q14).
+
+    `os._exit` skips `atexit`. The persist child records usage and the OAuth
+    child may too, and either can reach a cache sync that arms a store; the
+    finalizer then attempts one PASSIVE per armed store and closes the
+    keepers. With nothing armed it does nothing."""
+    try:
+        import _lib_wal_checkpoint
+
+        _lib_wal_checkpoint.finalize()
+    except BaseException:  # noqa: BLE001 — the child exits regardless
+        pass
+
+
 def _fork_persist(parent_lock_fd: int) -> None:
     """Run the revalidated spool reducer in a detached serialized child."""
     try:
@@ -1375,6 +1390,7 @@ def _fork_persist(parent_lock_fd: int) -> None:
     except BaseException:
         pass
     finally:
+        _finalize_wal_checkpoints()
         os._exit(0)
 
 
@@ -1487,6 +1503,7 @@ def _fork_statusline_oauth_refresh(lock_fd: int) -> bool:
         pass
     finally:
         _release_statusline_oauth_lock(lock_fd)
+        _finalize_wal_checkpoints()
         os._exit(0)
 
 

package/bin/_cctally_store.py

@@ -168,25 +168,100 @@ def open_index(store: str) -> sqlite3.Connection:
     return conn
 
 
-def apply_policy(conn: sqlite3.Connection, store: str) -> None:
+#: `PRAGMA auto_vacuum`'s numeric report for each policy value.
+_AUTO_VACUUM_MODES = {"NONE": 0, "FULL": 1, "INCREMENTAL": 2}
+
+
+def apply_policy(
+    conn: sqlite3.Connection, store: str, *, memory_temp_store: bool = True,
+) -> None:
     """Apply the §6.1 PRAGMA policy for ``store`` to an open connection.
 
     ``auto_vacuum`` (when set) is emitted first: it only takes effect before the
     first page is written, so it must precede ``journal_mode`` / any DDL.
+    ``memory_temp_store`` is forwarded to ``apply_connection_policy``.
+
+    #901 W9 (Q14): the pragma is emitted only when the database does not
+    already report the policy's mode. Setting INCREMENTAL or FULL rewrites
+    page 1 (a write transaction) even when the value is unchanged, so a
+    current-schema conversations.db open wrote a WAL frame on every pass.
+    Creation (an empty file reports 0) and a database in another mode still
+    receive it, the latter so a later ``VACUUM`` on that connection converts.
     """
     policy = STORE_POLICY[store]
     if policy.auto_vacuum is not None:
-        conn.execute(f"PRAGMA auto_vacuum={policy.auto_vacuum}")
+        wanted = _AUTO_VACUUM_MODES[policy.auto_vacuum.upper()]
+        current = conn.execute("PRAGMA auto_vacuum").fetchone()[0]
+        if int(current) != wanted:
+            conn.execute(f"PRAGMA auto_vacuum={policy.auto_vacuum}")
     conn.execute(f"PRAGMA journal_mode={policy.journal_mode}")
-    apply_connection_policy(conn, store)
+    apply_connection_policy(conn, store, memory_temp_store=memory_temp_store)
 
 
-def apply_connection_policy(conn: sqlite3.Connection, store: str) -> None:
-    """Apply non-schema connection settings without changing journal mode."""
+def apply_connection_policy(
+    conn: sqlite3.Connection, store: str, *, memory_temp_store: bool = True,
+) -> None:
+    """Apply non-schema connection settings without changing journal mode.
+
+    Every ordinary writable opener calls this once at open, before any
+    transaction, so it also carries the #901 writer temp-store policy
+    (``apply_writer_temp_store``). ``memory_temp_store=False`` is for a
+    writer §4.7 does not enumerate, which keeps SQLite's default.
+    """
     policy = STORE_POLICY[store]
     conn.execute(f"PRAGMA synchronous={policy.synchronous}")
     conn.execute(f"PRAGMA busy_timeout={policy.busy_timeout}")
     conn.execute(f"PRAGMA journal_size_limit={policy.journal_size_limit}")
+    if memory_temp_store:
+        apply_writer_temp_store(conn)
+
+
+# #901 §5.3a (Q11, design answer dc7 D1): the sync and ingest writers keep
+# their statement journals in memory. A statement that dirties more than 64 KiB
+# of pages that existed when it started (an FTS5 segment write behind the
+# conversation-message trigger, a quota insert maintaining eight indexes and
+# two trigger tables) otherwise spills its journal to an `etilqs_*` file, which
+# SQLite unlinks at creation, so nothing in the data directory shows it.
+# Measured on the revision-5 candidate: 3.7-10 MB of such files per dashboard
+# window and per Codex hook.
+#
+# Set once per connection at open, never toggled per pass. The bound is per
+# statement (the original images of the pages that one statement dirties), not
+# a fixed cap; the cache size is unchanged. It is NOT a licence for sorts:
+# G1 (tests/test_901_sql_plan_guard.py) judges every recurring plan regardless
+# of this setting, so moving a history-sized sort into memory never satisfies
+# I1. Openers outside spec §4.7 keep SQLite's default: read-only, state,
+# backup, checkpoint and vacuum connections (VACUUM builds its copy in the temp
+# store, so MEMORY there would hold the whole database in memory), and the
+# stats publication connection. The deletion connection keeps its own Q5
+# MEMORY/FILE selection and reclaim its own settings
+# (`_lib_conversation_retention`).
+#
+#: The temp store every ordinary writable opener selects. A test sets "FILE"
+#: or "DEFAULT" for the file-temp control (tests/test_901_connection_policy.py).
+WRITER_TEMP_STORE = "MEMORY"
+
+
+def apply_writer_temp_store(conn: sqlite3.Connection) -> None:
+    """Select ``WRITER_TEMP_STORE`` on a writable connection, outside any
+    transaction (SQLite ignores the change inside one)."""
+    conn.execute(f"PRAGMA temp_store = {WRITER_TEMP_STORE}")
+
+
+#: PR-3: the temp store an opener selects for the duration of PENDING schema
+#: and migration work (a first open, an upgrade, a cutover), then restores
+#: ``WRITER_TEMP_STORE``. A one-time index build or a migration over a
+#: history-sized table sorts and journals in the temp store, which under MEMORY
+#: is a RAM spike as large as that table; before #901 no temp store was
+#: selected, so this is the base behaviour for that work. A current-schema open
+#: never selects it, so the steady state is unchanged.
+SCHEMA_WORK_TEMP_STORE = "FILE"
+
+
+def apply_schema_work_temp_store(conn: sqlite3.Connection) -> None:
+    """Select ``SCHEMA_WORK_TEMP_STORE`` before pending schema work, outside
+    any transaction; the opener selects the writer temp store again after."""
+    conn.execute(f"PRAGMA temp_store = {SCHEMA_WORK_TEMP_STORE}")
 
 
 # --------------------------------------------------------------------------

package/bin/_cctally_tui.py

@@ -223,7 +223,7 @@ from _lib_display_tz import (
     _compute_display_block,
 )
 from _lib_aggregators import _aggregate_monthly, codex_path_scope
-from _lib_fmt import stable_sum
+from _lib_fmt import _fmt_usd_accounting, stable_sum
 # Opt-in backend phase-instrumentation collector (issue #276, Session A). Pure
 # stdlib leaf; near-noop when CCTALLY_PERF_TRACE is unset (phase() returns a
 # shared no-op singleton), so the _tui_build_snapshot seam wraps below cost
@@ -8401,8 +8401,9 @@ def _tui_modal_current_week(snap, runtime, width):
             ms.crossed_at, runtime.display_tz,
             fmt="%b %d %H:%M:%S", suffix=True,
         )
-        cumul_str = f"${ms.cumulative_cost_usd:.2f}".ljust(8)
-        marg_str = (f"${ms.marginal_cost_usd:.2f}" if ms.marginal_cost_usd is not None else "—").ljust(10)
+        cumul_str = _fmt_usd_accounting(ms.cumulative_cost_usd, 2).ljust(8)
+        marg_str = (_fmt_usd_accounting(ms.marginal_cost_usd, 2)
+                    if ms.marginal_cost_usd is not None else "—").ljust(10)
         line = f"   {{b}}{ms.percent:>3}{{/}} {{bright}}{ts_str:<22}{{/}} {{b}}{cumul_str}{{/}} {{b}}{marg_str}{{/}}"
         if show_5h:
             five_str = (f"{int(ms.five_hour_pct_at_crossing)}%"

package/bin/_cctally_weekrefs.py

@@ -88,6 +88,131 @@ def _get_canonical_boundary_for_date(
     return None, None
 
 
+#: #901 (I1/G1, Amendment 1 item 8 and 1b): one leg per snapshot table of the
+#: recent-week read. Each groups ONE table's weeks newest first, streamed off a
+#: week-date-leading index, so neither leg sorts anything; `_recent_week_rows`
+#: merges the two. The usage leg reads the covering epoch-1017
+#: `idx_usage_week_date_group` (no table lookups), falling back to
+#: `idx_usage_week_time` on an index without it; the cost leg reads
+#: `idx_cost_week_time`.
+_RECENT_USAGE_WEEKS_SQL = (
+    "SELECT week_start_date, MAX(week_end_date) AS week_end_date "
+    "FROM weekly_usage_snapshots{hint}{account} "
+    "GROUP BY week_start_date ORDER BY week_start_date DESC"
+)
+_RECENT_COST_WEEKS_SQL = (
+    "SELECT week_start_date, MAX(week_end_date) AS week_end_date "
+    "FROM weekly_cost_snapshots{hint}{account} "
+    "GROUP BY week_start_date ORDER BY week_start_date DESC"
+)
+
+
+def _sqlite_sort_key(value) -> tuple:
+    """Rank a stored value the way SQLite's BINARY comparison ranks it.
+
+    NULL first, then numbers (integers and reals compared numerically), then
+    text in code-point order (which is UTF-8 byte order), then blobs.
+    """
+    if value is None:
+        return (0, 0)
+    if isinstance(value, (int, float)):
+        return (1, value)
+    if isinstance(value, str):
+        return (2, value)
+    return (3, bytes(value))
+
+
+def _sqlite_max(left, right):
+    """SQLite's MAX() over two group maxima: NULL never wins."""
+    if left is None:
+        return right
+    if right is None:
+        return left
+    return left if _sqlite_sort_key(left) >= _sqlite_sort_key(right) else right
+
+
+def _recent_week_rows(
+    conn: sqlite3.Connection, limit: "int | None", *,
+    account_key: "str | None" = None,
+) -> "list[tuple[object, object]]":
+    """The newest ``limit`` week dates across both snapshot tables (#901).
+
+    Returns ``(week_start_date, week_end_date)`` pairs, newest date first, where
+    ``week_end_date`` is the latest end date EITHER table records for that
+    date: exactly the rows of the pre-#901 statement, which grouped the
+    ``UNION ALL`` of both tables through a temp b-tree over every snapshot ever
+    recorded and applied ``LIMIT`` only afterwards (Amendment 1 item 8,
+    ``c901-design-2``). Each table now streams its own groups in descending
+    date order; this merges the two cursors, combining equal dates with MAX,
+    holds one pending group per cursor, and stops at ``limit`` merged distinct
+    dates. ``None`` or a negative ``limit`` is unbounded and ``0`` returns
+    nothing, as SQL ``LIMIT`` did. The limit applies before any boundary
+    conversion or invalid-ref filtering, as it did.
+
+    Both cursors read one snapshot: inside the caller's transaction when there
+    is one (left open), otherwise inside a read transaction opened and closed
+    here. Each leg is pinned to its index only when the index exists; the
+    usage leg prefers the covering ``idx_usage_week_date_group`` (Amendment
+    1b) and keeps ``idx_usage_week_time`` as its compatibility fallback.
+    """
+    limit_n = -1 if limit is None else int(limit)
+    if limit_n == 0:
+        return []
+    present = {
+        str(row[0]) for row in conn.execute(
+            "SELECT name FROM sqlite_master WHERE type='index' AND name IN "
+            "('idx_usage_week_date_group', 'idx_usage_week_time', "
+            "'idx_cost_week_time')")
+    }
+    account = "" if account_key is None else " WHERE account_key = ?"
+    params: tuple = () if account_key is None else (account_key,)
+    usage_index = next(
+        (name for name in ("idx_usage_week_date_group", "idx_usage_week_time")
+         if name in present), None)
+    usage_sql = _RECENT_USAGE_WEEKS_SQL.format(
+        hint=f" INDEXED BY {usage_index}" if usage_index else "",
+        account=account)
+    cost_sql = _RECENT_COST_WEEKS_SQL.format(
+        hint=(" INDEXED BY idx_cost_week_time"
+              if "idx_cost_week_time" in present else ""),
+        account=account)
+    own_transaction = not conn.in_transaction
+    if own_transaction:
+        conn.execute("BEGIN")
+    try:
+        usage = conn.execute(usage_sql, params)
+        cost = conn.execute(cost_sql, params)
+        try:
+            rows: "list[tuple[object, object]]" = []
+            pending_usage = usage.fetchone()
+            pending_cost = cost.fetchone()
+            while (pending_usage is not None or pending_cost is not None) and (
+                    limit_n < 0 or len(rows) < limit_n):
+                if pending_cost is None or (
+                        pending_usage is not None
+                        and _sqlite_sort_key(pending_usage[0])
+                        > _sqlite_sort_key(pending_cost[0])):
+                    rows.append((pending_usage[0], pending_usage[1]))
+                    pending_usage = usage.fetchone()
+                elif pending_usage is None or (
+                        _sqlite_sort_key(pending_cost[0])
+                        > _sqlite_sort_key(pending_usage[0])):
+                    rows.append((pending_cost[0], pending_cost[1]))
+                    pending_cost = cost.fetchone()
+                else:
+                    rows.append((pending_usage[0], _sqlite_max(
+                        pending_usage[1], pending_cost[1])))
+                    pending_usage = usage.fetchone()
+                    pending_cost = cost.fetchone()
+        finally:
+            usage.close()
+            cost.close()
+    finally:
+        if own_transaction:
+            conn.execute("COMMIT")
+    return rows
+
+
 def get_recent_weeks(
     conn: sqlite3.Connection, limit: "int | None", *,
     account_key: "str | None" = None,
@@ -100,33 +225,15 @@ def get_recent_weeks(
     # (today's byte-identical behavior); a real key / ``unattributed`` scopes both
     # snapshot legs to that account's weeks (the ``--account`` render consumers —
     # ``report`` — pass it explicitly).
-    limit_sql = -1 if limit is None else int(limit)
-    acct_pred = "" if account_key is None else " WHERE account_key = ?"
-    acct_p: tuple = () if account_key is None else (account_key,)
-    rows = conn.execute(
-        f"""
-        SELECT week_start_date, MAX(week_end_date) AS week_end_date
-        FROM (
-          SELECT week_start_date, week_end_date FROM weekly_usage_snapshots{acct_pred}
-          UNION ALL
-          SELECT week_start_date, week_end_date FROM weekly_cost_snapshots{acct_pred}
-        )
-        GROUP BY week_start_date
-        ORDER BY week_start_date DESC
-        LIMIT ?
-        """,
-        acct_p + acct_p + (limit_sql,),
-    ).fetchall()
-
     refs: list[WeekRef] = []
-    for row in rows:
-        date_str = row["week_start_date"]
+    for date_str, week_end_date in _recent_week_rows(
+            conn, limit, account_key=account_key):
         canon_start, canon_end = _get_canonical_boundary_for_date(
             conn, date_str, account_key=account_key)
         try:
             ref = make_week_ref(
                 week_start_date=date_str,
-                week_end_date=row["week_end_date"],
+                week_end_date=week_end_date,
                 week_start_at=canon_start,
                 week_end_at=canon_end,
             )

package/bin/_lib_cache_report.py

@@ -59,9 +59,13 @@ stable_sum = _import_stable_sum()
 
 
 def _import_pricing_kernel():
-    """Resolve the ``_lib_pricing`` symbols this kernel needs (#195/#413):
-    ``CACHE_WRITE_1H_MULTIPLIER`` (the derived 1-hour cache-write rate) and
-    ``claude_usage_dict`` (the single cost-feeding usage-dict constructor).
+    """Resolve the ``_lib_pricing`` symbols this kernel needs, returned in
+    this order: ``CACHE_WRITE_1H_MULTIPLIER`` (the derived 1-hour cache-write
+    rate, #195), ``_claude_fast_multiplier`` (the Fast-mode multiplier for a
+    model), ``claude_usage_dict`` (the single cost-feeding usage-dict
+    constructor, #413), ``TIERED_THRESHOLD`` (the legacy 200K marginal-tier
+    boundary) and ``_select_claude_request_card`` (the whole-request card
+    selector, #929).
 
     Same shape and same justification as ``_import_stable_sum`` above:
     ``_lib_pricing`` is a PURE stdlib leaf with no sibling imports, so binding
@@ -79,6 +83,7 @@ def _import_pricing_kernel():
             m._claude_fast_multiplier,
             m.claude_usage_dict,
             m.TIERED_THRESHOLD,
+            m._select_claude_request_card,
         )
     from pathlib import Path
     import importlib.util
@@ -99,11 +104,13 @@ def _import_pricing_kernel():
         m._claude_fast_multiplier,
         m.claude_usage_dict,
         m.TIERED_THRESHOLD,
+        m._select_claude_request_card,
     )
 
 
 (CACHE_WRITE_1H_MULTIPLIER, _claude_fast_multiplier,
- claude_usage_dict, DEFAULT_TIERED_THRESHOLD) = _import_pricing_kernel()
+ claude_usage_dict, DEFAULT_TIERED_THRESHOLD,
+ _select_claude_request_card) = _import_pricing_kernel()
 
 
 # ``DEFAULT_TIERED_THRESHOLD`` above is Anthropic's per-call >200K-tokens tier,
@@ -353,6 +360,7 @@ def _compute_entry_cache_dollars(
     cache_creation_tokens: int,
     cache_read_tokens: int,
     *,
+    input_tokens: int,
     pricing: dict,
     tiered_threshold: "int | None" = None,
     cache_1h_tokens: int | None = None,
@@ -384,12 +392,27 @@ def _compute_entry_cache_dollars(
     than in the signature. A default argument is evaluated once, at
     function-definition time, so a change to the module global would never
     reach the calls.
+
+    ``input_tokens`` (REQUIRED keyword, #929) is the entry's uncached input.
+    A whole-request card (Claude Haiku 5.5) selects ONE card per request from
+    its full prompt — input + cache creation + cache read — exactly as the
+    cost kernel does, so the card is selected once here, before any rate is
+    read. Selecting from the cache tokens alone would price a long request at
+    the base card. A legacy card is returned unchanged and every expression
+    below runs exactly as before.
     """
     if tiered_threshold is None:
         tiered_threshold = DEFAULT_TIERED_THRESHOLD
     p = _lookup_pricing(model, pricing) or {}
     if not p:
         return (0.0, 0.0, 0.0)
+    # The selected card carries no `_above_200k_tokens` key, so
+    # `_tiered_rate` returns its flat rates, and the wasted expression below
+    # is already the direct TTL form on the selected input rate.
+    p = _select_claude_request_card(
+        p, input_tokens=input_tokens,
+        cache_creation_tokens=cache_creation_tokens,
+        cache_read_tokens=cache_read_tokens)
 
     def _tiered_rate(tokens: int, base_key: str, tiered_key: str) -> float:
         """Blended $/token rate for a single-call token count under tiered pricing."""
@@ -527,7 +550,9 @@ def _aggregate_cache_by_day(
             )
         else:
             saved, wasted, net = _compute_entry_cache_dollars(
-                entry.model, create_tok, read_tok, pricing=pricing,
+                entry.model, create_tok, read_tok,
+                input_tokens=entry.usage.get("input_tokens", 0),   # #929
+                pricing=pricing,
                 # #195: the normalized flat key the ingest chokepoint writes;
                 # absent on a pre-split row -> None -> unchanged pricing.
                 cache_1h_tokens=entry.usage.get("cache_creation_1h_input_tokens"),
@@ -703,6 +728,7 @@ def _aggregate_cache_by_session(
                 entry.model,
                 entry.cache_creation_tokens,
                 entry.cache_read_tokens,
+                input_tokens=entry.input_tokens,   # #929
                 pricing=pricing,
                 # #195: getattr, not attribute access. This is a PURE kernel with
                 # a duck-typed entry contract — callers pass `_JoinedClaudeEntry`
@@ -1156,6 +1182,7 @@ def _aggregate_cache_breakdown(
                 getattr(e, "model", ""),
                 getattr(e, "cache_creation_tokens", 0),
                 getattr(e, "cache_read_tokens", 0),
+                input_tokens=getattr(e, "input_tokens", 0),   # #929
                 pricing=pricing,
                 # #195: getattr default None == split unknown, so an entry type
                 # that does not carry the split prices exactly as before.
@@ -1270,7 +1297,9 @@ def aggregate_by_day_project(
         create_tok = getattr(e, "cache_creation_tokens", 0)
         read_tok = getattr(e, "cache_read_tokens", 0)
         _s, _w, net = _compute_entry_cache_dollars(
-            model, create_tok, read_tok, pricing=pricing,
+            model, create_tok, read_tok,
+            input_tokens=getattr(e, "input_tokens", 0),   # #929
+            pricing=pricing,
             cache_1h_tokens=getattr(e, "cache_1h_tokens", None),   # #195
             speed=getattr(e, "speed", None),
         )

package/bin/_lib_codex_conversation.py

@@ -2167,6 +2167,38 @@ def _is_late_turn_anchor(
     )
 
 
+def in_physical_order(rows: Iterable[Any], *, timestamp, position) -> list:
+    """Rows read in ``(timestamp_utc, id)`` index order, returned in physical
+    order: ``(timestamp_utc, source_path, line_offset)``.
+
+    #901 Amendment 19 T1 (spec I1). ``idx_codex_conv_msgs_conversation`` is
+    ``(conversation_key, timestamp_utc, id)``, so a read ordered by the
+    physical position past the timestamp left SQLite a "LAST 2 TERMS" sorter
+    over every run of rows tied on one timestamp, and one conversation's tie
+    run (a whole conversation, when its rows share or lack a timestamp) is not
+    a bound. The caller now reads in index order and this orders each run in
+    Python: the same rows the caller already holds, no temp structure.
+
+    Exact: equal timestamps (NULL included) are contiguous in index order and
+    compare equal here as they do in an ORDER BY, and ``UNIQUE(source_path,
+    line_offset)`` makes ``position`` a total order inside a run.
+    """
+    ordered: list = []
+    run: list = []
+    run_timestamp = None
+    for row in rows:
+        stamp = timestamp(row)
+        if run and stamp != run_timestamp:
+            run.sort(key=position)
+            ordered.extend(run)
+            run = []
+        run_timestamp = stamp
+        run.append(row)
+    run.sort(key=position)
+    ordered.extend(run)
+    return ordered
+
+
 def codex_event_is_late_turn_anchor(event: Any) -> bool:
     """Public predicate shared by delta persistence and turn inference."""
     try: