cctally 1.75.1 → 1.76.0

package/CHANGELOG.md

@@ -5,6 +5,17 @@ based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 
 ## [Unreleased]
 
+## [1.76.0] - 2026-07-20
+
+### Added
+- `cctally doctor` now reports `db.reclaimable` when at least 25% of `cache.db` pages are free, with a direct `cctally db vacuum --db cache` remediation. (#315)
+
+### Fixed
+- Codex dashboard Weekly cycles and `$ / 1%` accounting now keep the separately-metered GPT-5.3-Codex-Spark quota pool distinct from the standard seven-day Codex pool, preventing brief Spark sessions from creating phantom standard weeks.
+- Dashboard Codex Weekly cost deltas now use the shared fractional-ratio contract instead of rendering 100× too large, and the Codex **$/1% Trend** table once again aligns Used%, $/1%, and vs-prior values under their matching columns.
+- Claude dashboard quota now keeps updating automatically while Claude Code uses a bracketed context variant such as `opus[1m]`; context-window metadata no longer suppresses the valid account-wide 5-hour and 7-day observations supplied to the status line.
+- Conversation-retention pruning now commits each whole Claude session or Codex conversation separately while retaining its maintenance locks, bounding first-prune WAL growth without exposing partially deleted conversations; a 9.37 GB production-shaped benchmark reduced peak WAL from 889.7 MiB to 28.8 MiB with identical deleted-row counts and FTS integrity. (#315)
+
 ## [1.75.1] - 2026-07-20
 
 ### Fixed

package/bin/_cctally_dashboard_sources.py

@@ -45,7 +45,7 @@ from _lib_quota import (
     quota_freshness,
     select_baseline,
 )
-from _lib_jsonl import CodexEntry
+from _lib_jsonl import CodexEntry, codex_model_scoped_quota_pool
 from _lib_fmt import stable_sum
 from _lib_aggregators import _aggregate_codex_buckets
 from _lib_five_hour import _FIVE_HOUR_JITTER_FLOOR_SECONDS
@@ -103,6 +103,21 @@ class CodexWeeklyPeriod:
     used_percent: float | None = None
 
 
+def _is_model_scoped_codex_quota(logical_limit_key: object) -> bool:
+    """Whether an interpreted native identity belongs outside standard quota."""
+    if not isinstance(logical_limit_key, str):
+        return False
+    try:
+        payload = json.loads(logical_limit_key)
+    except (json.JSONDecodeError, TypeError):
+        return False
+    return (
+        isinstance(payload, dict)
+        and isinstance(payload.get("modelPool"), str)
+        and bool(payload["modelPool"].strip())
+    )
+
+
 def _resolve_codex_weekly_cycle(
     observations: Iterable[object],
     now_utc: dt.datetime,
@@ -113,6 +128,8 @@ def _resolve_codex_weekly_cycle(
     for history in build_history(tuple(observations)):
         if history.identity.window_minutes != 10_080:
             continue
+        if _is_model_scoped_codex_quota(history.identity.logical_limit_key):
+            continue
         baseline = select_baseline(history.observations, now_utc)
         if baseline is None or baseline.resets_at <= now_utc:
             continue
@@ -161,7 +178,8 @@ def _codex_weekly_periods(
     placeholders = ",".join("?" for _ in roots)
     try:
         rows = stats_conn.execute(
-            "SELECT source_root_key, resets_at_utc, nominal_start_at_utc, current_percent "
+            "SELECT source_root_key, logical_limit_key, resets_at_utc, "
+            "nominal_start_at_utc, current_percent "
             "FROM quota_window_blocks "
             "WHERE source='codex' AND window_minutes=10080 "
             f"AND source_root_key IN ({placeholders}) AND orphaned_at IS NULL "
@@ -174,7 +192,9 @@ def _codex_weekly_periods(
 
     raw_boundaries: list[tuple[dt.datetime, dt.datetime, set[str], list[float]]] = []
 
-    for root_key, resets_at_raw, start_at_raw, current_percent in rows:
+    for root_key, logical_limit_key, resets_at_raw, start_at_raw, current_percent in rows:
+        if _is_model_scoped_codex_quota(logical_limit_key):
+            continue
         try:
             start_at = dt.datetime.fromisoformat(str(start_at_raw).replace("Z", "+00:00"))
             resets_at = dt.datetime.fromisoformat(str(resets_at_raw).replace("Z", "+00:00"))
@@ -1833,6 +1853,8 @@ def _build_codex_native_weekly_view(
     labels: dict[str, str] = {}
     periods_by_bucket: dict[str, CodexWeeklyPeriod] = {}
     for entry in entries:
+        if codex_model_scoped_quota_pool(getattr(entry, "model", None)) is not None:
+            continue
         timestamp = getattr(entry, "timestamp").astimezone(UTC)
         root_key = str(getattr(entry, "source_root_key", "") or "")
         period = next((

package/bin/_cctally_doctor.py

@@ -385,10 +385,21 @@ def doctor_gather_state(
 
     cache_entries_count = None
     cache_last_entry_at = None
+    cache_db_page_count = None
+    cache_db_freelist_count = None
     try:
         if _cctally_core.CACHE_DB_PATH.exists():
             conn = sqlite3.connect(str(_cctally_core.CACHE_DB_PATH))
             try:
+                try:
+                    row = conn.execute("PRAGMA page_count").fetchone()
+                    if row and row[0] is not None:
+                        cache_db_page_count = int(row[0])
+                    row = conn.execute("PRAGMA freelist_count").fetchone()
+                    if row and row[0] is not None:
+                        cache_db_freelist_count = int(row[0])
+                except sqlite3.Error:
+                    pass
                 row = conn.execute(
                     "SELECT COUNT(*), MAX(timestamp_utc) FROM session_entries"
                 ).fetchone()
@@ -863,6 +874,9 @@ def doctor_gather_state(
         locks_held=locks_held,
         # #297: cache.db WAL size backstop (gathered outside the deep branch).
         cache_db_wal_bytes=cache_db_wal_bytes,
+        # #315: read-only cache free-page evidence for the reclaim hint.
+        cache_db_page_count=cache_db_page_count,
+        cache_db_freelist_count=cache_db_freelist_count,
         codex_quota_windows=codex_quota_windows,
         codex_hook_roots=codex_hook_roots,
         codex_lifecycle_activity_24h=codex_lifecycle_activity_24h,

package/bin/_cctally_quota.py

@@ -41,10 +41,12 @@ from _lib_quota import (
     source_path_key,
 )
 from _lib_json_envelope import stamp_schema_version
+from _lib_jsonl import _codex_logical_limit_key, codex_model_scoped_quota_pool
 
 
 UTC = dt.timezone.utc
 _DASHBOARD_PROJECTION_CERTIFICATE_KEY = "codex_quota_projection_certificate"
+_CODEX_QUOTA_INTERPRETATION_VERSION = 2
 
 
 @dataclass(frozen=True)
@@ -134,6 +136,11 @@ def load_codex_quota_projection_certificate(
         if row is None:
             return None
         payload = json.loads(str(row[0]))
+        if (
+            int(payload["interpretationVersion"])
+            != _CODEX_QUOTA_INTERPRETATION_VERSION
+        ):
+            return None
         sequence = int(payload["sequence"])
         signatures = {
             str(root_key): str(signature)
@@ -169,6 +176,7 @@ def _store_codex_quota_projection_certificate(
                 conn.rollback()
                 return
             payload = json.dumps({
+                "interpretationVersion": _CODEX_QUOTA_INTERPRETATION_VERSION,
                 "sequence": sequence,
                 "signatures": dict(sorted(signatures.items())),
             }, sort_keys=True, separators=(",", ":"))
@@ -262,14 +270,51 @@ def load_codex_quota_observations(
     previous_row_factory = conn.row_factory
     try:
         conn.row_factory = sqlite3.Row
+
+        def has_columns(table: str, required: set[str]) -> bool:
+            columns = {
+                str(row[1]) for row in conn.execute(
+                    f"PRAGMA table_info({table})"
+                )
+            }
+            return required <= columns
+
+        has_conversation_events = has_columns(
+            "codex_conversation_events",
+            {"source_path", "line_offset", "record_type", "payload_json"},
+        )
+        has_session_entries = has_columns(
+            "codex_session_entries", {"source_path", "line_offset", "model"},
+        )
+        if has_conversation_events:
+            model_expr = """
+                (SELECT json_extract(events.payload_json, '$.payload.model')
+                   FROM codex_conversation_events AS events
+                  WHERE events.source_path=quota_window_snapshots.source_path
+                    AND events.line_offset<=quota_window_snapshots.line_offset
+                    AND events.record_type IN ('turn_context','session_meta')
+                    AND json_type(events.payload_json, '$.payload.model')='text'
+                  ORDER BY events.line_offset DESC LIMIT 1) AS observed_model
+            """
+        elif has_session_entries:
+            model_expr = """
+                (SELECT entries.model
+                   FROM codex_session_entries AS entries
+                  WHERE entries.source_path=quota_window_snapshots.source_path
+                    AND entries.line_offset<=quota_window_snapshots.line_offset
+                  ORDER BY entries.line_offset DESC LIMIT 1) AS observed_model
+            """
+        else:
+            model_expr = "NULL AS observed_model"
         sql = """
             SELECT source, source_root_key, source_path, line_offset,
                    captured_at_utc, observed_slot, logical_limit_key, limit_id,
                    limit_name, window_minutes, used_percent, resets_at_utc,
-                   plan_type, individual_limit_json, reached_type
+                   plan_type, individual_limit_json, reached_type,
+                   {model_expr}
               FROM quota_window_snapshots
              WHERE source='codex' AND source_root_key IS NOT NULL
-        """
+        """.format(model_expr=model_expr)
         params: list[object] = []
         if requested is not None:
             if not requested:
@@ -317,10 +362,17 @@ def load_codex_quota_observations(
             if any(row[name] is None or not str(row[name]).strip() for name in required_text):
                 continue
             try:
+                logical_limit_key = str(row["logical_limit_key"])
+                if codex_model_scoped_quota_pool(row["observed_model"]) is not None:
+                    logical_limit_key = _codex_logical_limit_key(
+                        str(row["source_root_key"]), row["limit_id"],
+                        str(row["observed_slot"]), int(row["window_minutes"]),
+                        str(row["observed_model"]),
+                    )
                 identity = QuotaWindowIdentity(
                     source=str(row["source"]),
                     source_root_key=str(row["source_root_key"]),
-                    logical_limit_key=str(row["logical_limit_key"]),
+                    logical_limit_key=logical_limit_key,
                     observed_slot=str(row["observed_slot"]),
                     window_minutes=int(row["window_minutes"]),
                     limit_id=row["limit_id"],

package/bin/_cctally_statusline.py

@@ -1205,8 +1205,6 @@ def _fork_persist(parent_lock_fd: int) -> None:
 
 def _statusline_persist(parsed, *, sync_for_test: bool = False) -> None:
     """Spool an eligible session candidate then reduce it opportunistically."""
-    if _lib_statusline.is_alternate_pool_model_id(parsed.model_id):
-        return
     candidate = _candidate_from_input(parsed, received_at=int(time.time()))
     if candidate is None:
         return

package/bin/_lib_conversation_retention.py

@@ -25,14 +25,16 @@ The FTS5 indexes over ``conversation_messages`` (``conversation_fts``) and
 ``conversation_ai_titles`` (``conversation_title_fts``) are external-content and
 maintained by AFTER-DELETE triggers, which are logically correct on subset
 deletes. Whole groups are deleted so those triggers keep the index consistent.
-The kernel runs inside the caller's open transaction (the orchestrator owns the
-flocks, ``BEGIN IMMEDIATE``, and commit).
+The kernel normally runs inside the caller's open transaction. The orchestrator
+supplies a post-group boundary that commits each whole conversation separately
+while retaining every flock for the full pass (#315), bounding WAL growth.
 """
 from __future__ import annotations
 
 import datetime as dt
 import fcntl
 import sqlite3
+from collections.abc import Callable
 from dataclasses import dataclass
 
 import _cctally_core
@@ -71,12 +73,25 @@ def _cutoff_iso(cutoff_utc: dt.datetime) -> str:
 
 
 def prune_conversation_transcripts(
-    conn: sqlite3.Connection, *, cutoff_utc: dt.datetime
+    conn: sqlite3.Connection,
+    *,
+    cutoff_utc: dt.datetime,
+    after_group: "Callable[[], None] | None" = None,
 ) -> PruneStats:
-    """Prune every transcript group whose latest activity is before ``cutoff_utc``."""
+    """Prune every transcript group whose latest activity is before ``cutoff_utc``.
+
+    ``after_group`` runs only after every table and FTS posting owned by one
+    session/conversation has been deleted. Direct kernel callers leave it unset
+    and retain their caller-managed transaction; the orchestrator uses it for
+    #315's whole-conversation intermediate commits.
+    """
     cutoff = _cutoff_iso(cutoff_utc)
-    claude_sessions, claude_messages = _prune_claude(conn, cutoff)
-    codex_conversations, codex_events = _prune_codex(conn, cutoff)
+    claude_sessions, claude_messages = _prune_claude(
+        conn, cutoff, after_group=after_group
+    )
+    codex_conversations, codex_events = _prune_codex(
+        conn, cutoff, after_group=after_group
+    )
     return PruneStats(
         claude_sessions=claude_sessions,
         claude_messages=claude_messages,
@@ -116,7 +131,12 @@ def _prunable_null_identity_paths(
     return [row[0] for row in conn.execute(sql, (cutoff,))]
 
 
-def _prune_claude(conn: sqlite3.Connection, cutoff: str) -> tuple[int, int]:
+def _prune_claude(
+    conn: sqlite3.Connection,
+    cutoff: str,
+    *,
+    after_group: "Callable[[], None] | None" = None,
+) -> tuple[int, int]:
     sessions = 0
     messages = 0
     for session_id in _prunable_groups(
@@ -140,6 +160,8 @@ def _prune_claude(conn: sqlite3.Connection, cutoff: str) -> tuple[int, int]:
             (session_id,),
         )
         sessions += 1
+        if after_group is not None:
+            after_group()
     for source_path in _prunable_null_identity_paths(
         conn, "conversation_messages", "session_id", cutoff
     ):
@@ -166,6 +188,8 @@ def _prune_claude(conn: sqlite3.Connection, cutoff: str) -> tuple[int, int]:
         )
         messages += cur.rowcount if cur.rowcount and cur.rowcount > 0 else 0
         sessions += 1
+        if after_group is not None:
+            after_group()
     return sessions, messages
 
 
@@ -235,9 +259,11 @@ def _maybe_prune_conversation_retention(
     two provider flocks are taken in a FIXED order (Claude then Codex),
     non-blocking, so a rebuild/reingest mid-flight makes the prune skip this
     cycle rather than race between candidate selection and deletion. The prune of
-    both providers and the throttle stamp run in ONE ``BEGIN IMMEDIATE``
-    transaction, so the stamp is written only after both provider phases succeed;
-    any failure rolls back the whole thing (no stamp → retried next cycle).
+    each whole session/conversation runs in its own ``BEGIN IMMEDIATE``
+    transaction (#315), while every flock remains held for the full pass. The
+    throttle stamp is committed only after both provider phases succeed. A
+    failure rolls back the active group but preserves completed groups, writes no
+    stamp, and therefore retries the remainder next cycle.
 
     ``conn`` must hold no provider flock and no open transaction (the caller
     guarantees this — the dashboard opens a dedicated cache connection; the
@@ -274,7 +300,15 @@ def _maybe_prune_conversation_retention(
                 cutoff = now_utc - dt.timedelta(days=int(retention_days))
                 conn.execute("BEGIN IMMEDIATE")
                 try:
-                    stats = prune_conversation_transcripts(conn, cutoff_utc=cutoff)
+                    def commit_group() -> None:
+                        conn.commit()
+                        conn.execute("BEGIN IMMEDIATE")
+
+                    stats = prune_conversation_transcripts(
+                        conn,
+                        cutoff_utc=cutoff,
+                        after_group=commit_group,
+                    )
                     _stamp_retention_prune(conn, now_utc)
                     conn.commit()
                 except Exception:
@@ -354,7 +388,12 @@ def _delete_codex_conversation_derived(
     )
 
 
-def _prune_codex(conn: sqlite3.Connection, cutoff: str) -> tuple[int, int]:
+def _prune_codex(
+    conn: sqlite3.Connection,
+    cutoff: str,
+    *,
+    after_group: "Callable[[], None] | None" = None,
+) -> tuple[int, int]:
     conversations = 0
     events = 0
     for conversation_key in _prunable_groups(
@@ -369,6 +408,8 @@ def _prune_codex(conn: sqlite3.Connection, cutoff: str) -> tuple[int, int]:
         )
         events += cur.rowcount if cur.rowcount and cur.rowcount > 0 else 0
         conversations += 1
+        if after_group is not None:
+            after_group()
     for source_path in _prunable_null_identity_paths(
         conn, "codex_conversation_events", "conversation_key", cutoff
     ):
@@ -381,4 +422,6 @@ def _prune_codex(conn: sqlite3.Connection, cutoff: str) -> tuple[int, int]:
         )
         events += cur.rowcount if cur.rowcount and cur.rowcount > 0 else 0
         conversations += 1
+        if after_group is not None:
+            after_group()
     return conversations, events

package/bin/_lib_doctor.py

@@ -200,6 +200,12 @@ class DoctorState:
     # DOCTOR_WAL_WARN_BYTES (2x the WAL cap) — only when the journal_size_limit
     # + forced-checkpoint machinery has genuinely failed to contain the WAL.
     cache_db_wal_bytes: Optional[int] = None
+    # #315: read-only PRAGMA page_count/freelist_count evidence. The pure
+    # db.reclaimable check warns when free pages reach 25% of cache.db and
+    # points at the already-guarded explicit vacuum command. None means the
+    # cache was absent or unreadable; the check degrades to OK.
+    cache_db_page_count: Optional[int] = None
+    cache_db_freelist_count: Optional[int] = None
     # #294 S2: root-qualified physical Codex quota freshness, per-root native
     # hook state, and lifecycle activity are gathered by _cctally_doctor.
     codex_quota_windows: Optional[list[dict]] = None
@@ -1725,6 +1731,48 @@ def _check_db_wal_size(s: DoctorState) -> CheckResult:
     )
 
 
+# #315: conservative advisory threshold. A quarter of cache.db being free is
+# large enough to make an explicit, guarded VACUUM useful without nagging for
+# ordinary page churn. This is a ratio, so it remains page-size independent.
+DOCTOR_RECLAIMABLE_WARN_RATIO = 0.25
+
+
+def _check_db_reclaimable(s: DoctorState) -> CheckResult:
+    """Surface cache free pages without mutating or auto-vacuuming the DB."""
+    page_count = s.cache_db_page_count
+    freelist_count = s.cache_db_freelist_count
+    ratio = None
+    if (
+        isinstance(page_count, int)
+        and not isinstance(page_count, bool)
+        and page_count > 0
+        and isinstance(freelist_count, int)
+        and not isinstance(freelist_count, bool)
+        and 0 <= freelist_count <= page_count
+    ):
+        ratio = freelist_count / page_count
+    details = {
+        "cache_db_page_count": page_count,
+        "cache_db_freelist_count": freelist_count,
+        "cache_db_free_ratio": ratio,
+        "warn_ratio": DOCTOR_RECLAIMABLE_WARN_RATIO,
+    }
+    if ratio is not None and ratio >= DOCTOR_RECLAIMABLE_WARN_RATIO:
+        return CheckResult(
+            id="db.reclaimable", title="Reclaimable cache space",
+            severity="warn",
+            summary=f"high — {ratio * 100:.1f}% of cache.db pages are free",
+            remediation=(
+                "Run `cctally db vacuum --db cache` to reclaim disk space."
+            ),
+            details=details,
+        )
+    return CheckResult(
+        id="db.reclaimable", title="Reclaimable cache space", severity="ok",
+        summary="below threshold", remediation=None, details=details,
+    )
+
+
 # Each entry is (category_id, category_title, ((check_id, evaluator_fn_name), ...)).
 # The dotted check_id is the stable JSON-contract ID (spec §5.2) AND the
 # fingerprint identity-slice key (spec §5.5). When an evaluator raises,
@@ -1759,6 +1807,7 @@ _CATEGORY_DEFINITIONS: tuple[tuple[str, str, tuple[tuple[str, str], ...]], ...]
         ("db.migrations.pending", "_check_db_migrations_pending"),
         ("db.lock_state", "_check_db_lock_state"),
         ("db.wal_size", "_check_db_wal_size"),
+        ("db.reclaimable", "_check_db_reclaimable"),
     )),
     ("data", "Data", (
         ("data.latest_snapshot_age", "_check_data_latest_snapshot_age"),

package/bin/_lib_jsonl.py

@@ -584,20 +584,37 @@ def _first_valid_reset_at(
 
 def _codex_logical_limit_key(
     source_root_key: str | None, limit_id: str | None, observed_slot: str,
-    window_minutes: int,
+    window_minutes: int, model: str | None = None,
 ) -> str:
-    return _codex_canonical_json({
+    payload = {
         "limitId": limit_id,
         "observedSlot": observed_slot,
         "source": "codex",
         "sourceRootKey": source_root_key,
         "windowMinutes": window_minutes,
-    })
+    }
+    if (model_pool := codex_model_scoped_quota_pool(model)) is not None:
+        payload["modelPool"] = model_pool
+    return _codex_canonical_json(payload)
+
+
+def codex_model_scoped_quota_pool(model: object) -> str | None:
+    """Return the native model pool when Codex documents it as separate.
+
+    GPT Codex Spark runs against its own allowance and does not consume the
+    standard Codex quota.  Native payloads currently reuse ``limit_id=codex``
+    and the same slot/duration as the standard pool, so the sticky rollout
+    model is the only retained discriminator.
+    """
+    if not isinstance(model, str):
+        return None
+    normalized = model.strip().lower()
+    return normalized if "-codex-spark" in normalized else None
 
 
 def _codex_quota_observations(
     obj: dict[str, Any], payload: dict[str, Any], path_str: str, line_offset: int,
-    source_root_key: str | None,
+    source_root_key: str | None, model: str | None,
 ) -> tuple[CodexQuotaObservation, ...]:
     captured_at = _parse_codex_timestamp(obj.get("timestamp"))
     if captured_at is None:
@@ -638,7 +655,7 @@ def _codex_quota_observations(
             captured_at_utc=_format_codex_timestamp(captured_at),
             observed_slot=slot,
             logical_limit_key=_codex_logical_limit_key(
-                source_root_key, limit_id, slot, window_minutes
+                source_root_key, limit_id, slot, window_minutes, model
             ),
             limit_id=limit_id,
             limit_name=_first_valid_string(
@@ -867,7 +884,7 @@ def _iter_codex_fused_records_with_offsets(
             filename_session_id_warned,
         )
         quotas = _codex_quota_observations(
-            obj, payload, path_str, line_offset, source_root_key
+            obj, payload, path_str, line_offset, source_root_key, state.model
         )
         yield CodexFusedEmission(
             line_offset=line_offset,

package/bin/_lib_statusline.py

@@ -191,32 +191,6 @@ def parse_statusline_stdin(raw: "bytes | str") -> "StatuslineInput | ParseError"
     )
 
 
-# ---- Pool-identity guard (persist-only; spec 2026-07-17 #311 D1) ----------
-
-
-_ALTERNATE_POOL_MODEL_ID_RE = re.compile(r"\[[^\]]+\]$")
-
-
-def is_alternate_pool_model_id(model_id) -> bool:
-    """True iff ``model_id`` is a bracketed variant id (e.g.
-    ``claude-opus-4-8[1m]``) that reports a SEPARATE rate-limit pool.
-
-    Such a session's stdin ``rate_limits`` describes a DIFFERENT usage pool
-    on the same account; persisting it poisons the default-pool DB (the 7d
-    HWM clamp latches the foreign high value and dedup then freezes tracking
-    — see the #311 spec). The persist feeder skips it.
-
-    Matches ANY trailing bracket suffix, not just the literal ``[1m]``: a
-    future variant that turns out to share the default pool would merely lose
-    one redundant writer (fail-safe toward data purity), whereas matching
-    only ``[1m]`` would let the next variant poison the DB again. Missing /
-    ``None`` / non-string / no-suffix ids return ``False`` (persist proceeds —
-    only a positive variant match skips). Never raises."""
-    if not isinstance(model_id, str) or not model_id:
-        return False
-    return _ALTERNATE_POOL_MODEL_ID_RE.search(model_id) is not None
-
-
 # ---- Segment 1: model -----------------------------------------------------