cctally 1.75.1 → 1.77.0

package/bin/_lib_conversation_retention.py

@@ -25,14 +25,16 @@ The FTS5 indexes over ``conversation_messages`` (``conversation_fts``) and
 ``conversation_ai_titles`` (``conversation_title_fts``) are external-content and
 maintained by AFTER-DELETE triggers, which are logically correct on subset
 deletes. Whole groups are deleted so those triggers keep the index consistent.
-The kernel runs inside the caller's open transaction (the orchestrator owns the
-flocks, ``BEGIN IMMEDIATE``, and commit).
+The kernel normally runs inside the caller's open transaction. The orchestrator
+supplies a post-group boundary that commits each whole conversation separately
+while retaining every flock for the full pass (#315), bounding WAL growth.
 """
 from __future__ import annotations
 
 import datetime as dt
 import fcntl
 import sqlite3
+from collections.abc import Callable
 from dataclasses import dataclass
 
 import _cctally_core
@@ -71,12 +73,25 @@ def _cutoff_iso(cutoff_utc: dt.datetime) -> str:
 
 
 def prune_conversation_transcripts(
-    conn: sqlite3.Connection, *, cutoff_utc: dt.datetime
+    conn: sqlite3.Connection,
+    *,
+    cutoff_utc: dt.datetime,
+    after_group: "Callable[[], None] | None" = None,
 ) -> PruneStats:
-    """Prune every transcript group whose latest activity is before ``cutoff_utc``."""
+    """Prune every transcript group whose latest activity is before ``cutoff_utc``.
+
+    ``after_group`` runs only after every table and FTS posting owned by one
+    session/conversation has been deleted. Direct kernel callers leave it unset
+    and retain their caller-managed transaction; the orchestrator uses it for
+    #315's whole-conversation intermediate commits.
+    """
     cutoff = _cutoff_iso(cutoff_utc)
-    claude_sessions, claude_messages = _prune_claude(conn, cutoff)
-    codex_conversations, codex_events = _prune_codex(conn, cutoff)
+    claude_sessions, claude_messages = _prune_claude(
+        conn, cutoff, after_group=after_group
+    )
+    codex_conversations, codex_events = _prune_codex(
+        conn, cutoff, after_group=after_group
+    )
     return PruneStats(
         claude_sessions=claude_sessions,
         claude_messages=claude_messages,
@@ -116,7 +131,12 @@ def _prunable_null_identity_paths(
     return [row[0] for row in conn.execute(sql, (cutoff,))]
 
 
-def _prune_claude(conn: sqlite3.Connection, cutoff: str) -> tuple[int, int]:
+def _prune_claude(
+    conn: sqlite3.Connection,
+    cutoff: str,
+    *,
+    after_group: "Callable[[], None] | None" = None,
+) -> tuple[int, int]:
     sessions = 0
     messages = 0
     for session_id in _prunable_groups(
@@ -140,6 +160,8 @@ def _prune_claude(conn: sqlite3.Connection, cutoff: str) -> tuple[int, int]:
             (session_id,),
         )
         sessions += 1
+        if after_group is not None:
+            after_group()
     for source_path in _prunable_null_identity_paths(
         conn, "conversation_messages", "session_id", cutoff
     ):
@@ -166,6 +188,8 @@ def _prune_claude(conn: sqlite3.Connection, cutoff: str) -> tuple[int, int]:
         )
         messages += cur.rowcount if cur.rowcount and cur.rowcount > 0 else 0
         sessions += 1
+        if after_group is not None:
+            after_group()
     return sessions, messages
 
 
@@ -235,9 +259,11 @@ def _maybe_prune_conversation_retention(
     two provider flocks are taken in a FIXED order (Claude then Codex),
     non-blocking, so a rebuild/reingest mid-flight makes the prune skip this
     cycle rather than race between candidate selection and deletion. The prune of
-    both providers and the throttle stamp run in ONE ``BEGIN IMMEDIATE``
-    transaction, so the stamp is written only after both provider phases succeed;
-    any failure rolls back the whole thing (no stamp → retried next cycle).
+    each whole session/conversation runs in its own ``BEGIN IMMEDIATE``
+    transaction (#315), while every flock remains held for the full pass. The
+    throttle stamp is committed only after both provider phases succeed. A
+    failure rolls back the active group but preserves completed groups, writes no
+    stamp, and therefore retries the remainder next cycle.
 
     ``conn`` must hold no provider flock and no open transaction (the caller
     guarantees this — the dashboard opens a dedicated cache connection; the
@@ -251,7 +277,7 @@ def _maybe_prune_conversation_retention(
         core.APP_DIR.mkdir(parents=True, exist_ok=True)
     except OSError:
         return None
-    maint_fh = open(core.CACHE_LOCK_MAINTENANCE_PATH, "w")
+    maint_fh = open(core.CONVERSATIONS_LOCK_MAINTENANCE_PATH, "w")
     try:
         try:
             fcntl.flock(maint_fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
@@ -259,13 +285,13 @@ def _maybe_prune_conversation_retention(
             return None  # another prune holds it; skip cleanly, do NOT stamp
         if not force and not _retention_due(conn, now_utc):
             return None
-        claude_fh = open(core.CACHE_LOCK_PATH, "w")
+        claude_fh = open(core.CONVERSATIONS_LOCK_PATH, "w")
         try:
             try:
                 fcntl.flock(claude_fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
             except (BlockingIOError, OSError):
                 return None  # a Claude sync is mid-flight; retry next cycle
-            codex_fh = open(core.CACHE_LOCK_CODEX_PATH, "w")
+            codex_fh = open(core.CONVERSATIONS_LOCK_CODEX_PATH, "w")
             try:
                 try:
                     fcntl.flock(codex_fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
@@ -274,7 +300,15 @@ def _maybe_prune_conversation_retention(
                 cutoff = now_utc - dt.timedelta(days=int(retention_days))
                 conn.execute("BEGIN IMMEDIATE")
                 try:
-                    stats = prune_conversation_transcripts(conn, cutoff_utc=cutoff)
+                    def commit_group() -> None:
+                        conn.commit()
+                        conn.execute("BEGIN IMMEDIATE")
+
+                    stats = prune_conversation_transcripts(
+                        conn,
+                        cutoff_utc=cutoff,
+                        after_group=commit_group,
+                    )
                     _stamp_retention_prune(conn, now_utc)
                     conn.commit()
                 except Exception:
@@ -354,7 +388,12 @@ def _delete_codex_conversation_derived(
     )
 
 
-def _prune_codex(conn: sqlite3.Connection, cutoff: str) -> tuple[int, int]:
+def _prune_codex(
+    conn: sqlite3.Connection,
+    cutoff: str,
+    *,
+    after_group: "Callable[[], None] | None" = None,
+) -> tuple[int, int]:
     conversations = 0
     events = 0
     for conversation_key in _prunable_groups(
@@ -369,6 +408,8 @@ def _prune_codex(conn: sqlite3.Connection, cutoff: str) -> tuple[int, int]:
         )
         events += cur.rowcount if cur.rowcount and cur.rowcount > 0 else 0
         conversations += 1
+        if after_group is not None:
+            after_group()
     for source_path in _prunable_null_identity_paths(
         conn, "codex_conversation_events", "conversation_key", cutoff
     ):
@@ -381,4 +422,6 @@ def _prune_codex(conn: sqlite3.Connection, cutoff: str) -> tuple[int, int]:
         )
         events += cur.rowcount if cur.rowcount and cur.rowcount > 0 else 0
         conversations += 1
+        if after_group is not None:
+            after_group()
     return conversations, events

package/bin/_lib_doctor.py

@@ -200,6 +200,16 @@ class DoctorState:
     # DOCTOR_WAL_WARN_BYTES (2x the WAL cap) — only when the journal_size_limit
     # + forced-checkpoint machinery has genuinely failed to contain the WAL.
     cache_db_wal_bytes: Optional[int] = None
+    # #315: read-only PRAGMA page_count/freelist_count evidence. The pure
+    # db.reclaimable check warns when free pages reach 25% of cache.db and
+    # points at the already-guarded explicit vacuum command. None means the
+    # cache was absent or unreadable; the check degrades to OK.
+    cache_db_page_count: Optional[int] = None
+    cache_db_freelist_count: Optional[int] = None
+    # #320: the independently large transcript store needs the same reclaim
+    # visibility, with remediation targeted at its own VACUUM surface.
+    conversations_db_page_count: Optional[int] = None
+    conversations_db_freelist_count: Optional[int] = None
     # #294 S2: root-qualified physical Codex quota freshness, per-root native
     # hook state, and lifecycle activity are gathered by _cctally_doctor.
     codex_quota_windows: Optional[list[dict]] = None
@@ -1229,21 +1239,22 @@ def _check_data_conversation_sessions_rollup(s: DoctorState) -> CheckResult:
     row per distinct ``conversation_messages.session_id`` (#217 S1 / U9).
 
     OK when the two counts are equal, when either is ``None`` (pre-rollup or an
-    unreadable cache.db — consistent with the kernel's graceful-degrade posture),
+    unreadable conversations.db — consistent with the kernel's graceful-degrade
+    posture),
     OR when a sync/reingest/backfill is in progress. WARN ONLY on a mismatch
     observed in a QUIESCENT cache.
 
-    False-WARN avoidance (Codex P2): ``sync_cache`` commits
+    False-WARN avoidance (Codex P2): ``sync_claude_conversations`` commits
     ``conversation_messages`` per file *before* the ``conversation_sessions``
     recompute, and resumable reingest commits per file before its rollup
     completes — so an unsynchronized read can transiently mismatch. The
     in-progress signal (``conv_rollup_sync_in_progress``) is set by
-    ``doctor_gather_state`` from a NON-BLOCKING ``cache.db.lock`` flock probe
+    ``doctor_gather_state`` from a NON-BLOCKING ``conversations.db.lock`` flock probe
     (lock held ⇒ a writer is mid-flight) AND the presence of any pending
     ``cache_meta`` reingest/split/backfill flag; if either says in-progress, this
     stays OK. Doctor remains read-only and never blocks on the lock.
 
-    Informational only (no remediation): the next full ``sync_cache`` re-derives
+    Informational only (no remediation): the next conversation sync re-derives
     the rollup via its incremental DELETE+INSERT.
     """
     rollup = s.conv_sessions_rollup_count
@@ -1725,6 +1736,91 @@ def _check_db_wal_size(s: DoctorState) -> CheckResult:
     )
 
 
+# #315: conservative advisory threshold. A quarter of cache.db being free is
+# large enough to make an explicit, guarded VACUUM useful without nagging for
+# ordinary page churn. This is a ratio, so it remains page-size independent.
+DOCTOR_RECLAIMABLE_WARN_RATIO = 0.25
+
+
+def _check_db_reclaimable(s: DoctorState) -> CheckResult:
+    """Surface cache free pages without mutating or auto-vacuuming the DB."""
+    page_count = s.cache_db_page_count
+    freelist_count = s.cache_db_freelist_count
+    ratio = None
+    if (
+        isinstance(page_count, int)
+        and not isinstance(page_count, bool)
+        and page_count > 0
+        and isinstance(freelist_count, int)
+        and not isinstance(freelist_count, bool)
+        and 0 <= freelist_count <= page_count
+    ):
+        ratio = freelist_count / page_count
+    details = {
+        "cache_db_page_count": page_count,
+        "cache_db_freelist_count": freelist_count,
+        "cache_db_free_ratio": ratio,
+        "warn_ratio": DOCTOR_RECLAIMABLE_WARN_RATIO,
+    }
+    if ratio is not None and ratio >= DOCTOR_RECLAIMABLE_WARN_RATIO:
+        return CheckResult(
+            id="db.reclaimable", title="Reclaimable cache space",
+            severity="warn",
+            summary=f"high — {ratio * 100:.1f}% of cache.db pages are free",
+            remediation=(
+                "Run `cctally db vacuum --db cache` to reclaim disk space."
+            ),
+            details=details,
+        )
+    return CheckResult(
+        id="db.reclaimable", title="Reclaimable cache space", severity="ok",
+        summary="below threshold", remediation=None, details=details,
+    )
+
+
+def _check_db_conversations_reclaimable(s: DoctorState) -> CheckResult:
+    """Surface transcript-store free pages without mutating the DB (#320)."""
+    page_count = s.conversations_db_page_count
+    freelist_count = s.conversations_db_freelist_count
+    ratio = None
+    if (
+        isinstance(page_count, int)
+        and not isinstance(page_count, bool)
+        and page_count > 0
+        and isinstance(freelist_count, int)
+        and not isinstance(freelist_count, bool)
+        and 0 <= freelist_count <= page_count
+    ):
+        ratio = freelist_count / page_count
+    details = {
+        "conversations_db_page_count": page_count,
+        "conversations_db_freelist_count": freelist_count,
+        "conversations_db_free_ratio": ratio,
+        "warn_ratio": DOCTOR_RECLAIMABLE_WARN_RATIO,
+    }
+    if ratio is not None and ratio >= DOCTOR_RECLAIMABLE_WARN_RATIO:
+        return CheckResult(
+            id="db.conversations_reclaimable",
+            title="Reclaimable transcript space",
+            severity="warn",
+            summary=(
+                f"high — {ratio * 100:.1f}% of conversations.db pages are free"
+            ),
+            remediation=(
+                "Run `cctally db vacuum --db conversations` to reclaim disk space."
+            ),
+            details=details,
+        )
+    return CheckResult(
+        id="db.conversations_reclaimable",
+        title="Reclaimable transcript space",
+        severity="ok",
+        summary="below threshold",
+        remediation=None,
+        details=details,
+    )
+
+
 # Each entry is (category_id, category_title, ((check_id, evaluator_fn_name), ...)).
 # The dotted check_id is the stable JSON-contract ID (spec §5.2) AND the
 # fingerprint identity-slice key (spec §5.5). When an evaluator raises,
@@ -1759,6 +1855,8 @@ _CATEGORY_DEFINITIONS: tuple[tuple[str, str, tuple[tuple[str, str], ...]], ...]
         ("db.migrations.pending", "_check_db_migrations_pending"),
         ("db.lock_state", "_check_db_lock_state"),
         ("db.wal_size", "_check_db_wal_size"),
+        ("db.reclaimable", "_check_db_reclaimable"),
+        ("db.conversations_reclaimable", "_check_db_conversations_reclaimable"),
     )),
     ("data", "Data", (
         ("data.latest_snapshot_age", "_check_data_latest_snapshot_age"),

package/bin/_lib_jsonl.py

@@ -584,20 +584,37 @@ def _first_valid_reset_at(
 
 def _codex_logical_limit_key(
     source_root_key: str | None, limit_id: str | None, observed_slot: str,
-    window_minutes: int,
+    window_minutes: int, model: str | None = None,
 ) -> str:
-    return _codex_canonical_json({
+    payload = {
         "limitId": limit_id,
         "observedSlot": observed_slot,
         "source": "codex",
         "sourceRootKey": source_root_key,
         "windowMinutes": window_minutes,
-    })
+    }
+    if (model_pool := codex_model_scoped_quota_pool(model)) is not None:
+        payload["modelPool"] = model_pool
+    return _codex_canonical_json(payload)
+
+
+def codex_model_scoped_quota_pool(model: object) -> str | None:
+    """Return the native model pool when Codex documents it as separate.
+
+    GPT Codex Spark runs against its own allowance and does not consume the
+    standard Codex quota.  Native payloads currently reuse ``limit_id=codex``
+    and the same slot/duration as the standard pool, so the sticky rollout
+    model is the only retained discriminator.
+    """
+    if not isinstance(model, str):
+        return None
+    normalized = model.strip().lower()
+    return normalized if "-codex-spark" in normalized else None
 
 
 def _codex_quota_observations(
     obj: dict[str, Any], payload: dict[str, Any], path_str: str, line_offset: int,
-    source_root_key: str | None,
+    source_root_key: str | None, model: str | None,
 ) -> tuple[CodexQuotaObservation, ...]:
     captured_at = _parse_codex_timestamp(obj.get("timestamp"))
     if captured_at is None:
@@ -638,7 +655,7 @@ def _codex_quota_observations(
             captured_at_utc=_format_codex_timestamp(captured_at),
             observed_slot=slot,
             logical_limit_key=_codex_logical_limit_key(
-                source_root_key, limit_id, slot, window_minutes
+                source_root_key, limit_id, slot, window_minutes, model
             ),
             limit_id=limit_id,
             limit_name=_first_valid_string(
@@ -867,7 +884,7 @@ def _iter_codex_fused_records_with_offsets(
             filename_session_id_warned,
         )
         quotas = _codex_quota_observations(
-            obj, payload, path_str, line_offset, source_root_key
+            obj, payload, path_str, line_offset, source_root_key, state.model
         )
         yield CodexFusedEmission(
             line_offset=line_offset,

package/bin/_lib_statusline.py

@@ -191,32 +191,6 @@ def parse_statusline_stdin(raw: "bytes | str") -> "StatuslineInput | ParseError"
     )
 
 
-# ---- Pool-identity guard (persist-only; spec 2026-07-17 #311 D1) ----------
-
-
-_ALTERNATE_POOL_MODEL_ID_RE = re.compile(r"\[[^\]]+\]$")
-
-
-def is_alternate_pool_model_id(model_id) -> bool:
-    """True iff ``model_id`` is a bracketed variant id (e.g.
-    ``claude-opus-4-8[1m]``) that reports a SEPARATE rate-limit pool.
-
-    Such a session's stdin ``rate_limits`` describes a DIFFERENT usage pool
-    on the same account; persisting it poisons the default-pool DB (the 7d
-    HWM clamp latches the foreign high value and dedup then freezes tracking
-    — see the #311 spec). The persist feeder skips it.
-
-    Matches ANY trailing bracket suffix, not just the literal ``[1m]``: a
-    future variant that turns out to share the default pool would merely lose
-    one redundant writer (fail-safe toward data purity), whereas matching
-    only ``[1m]`` would let the next variant poison the DB again. Missing /
-    ``None`` / non-string / no-suffix ids return ``False`` (persist proceeds —
-    only a positive variant match skips). Never raises."""
-    if not isinstance(model_id, str) or not model_id:
-        return False
-    return _ALTERNATE_POOL_MODEL_ID_RE.search(model_id) is not None
-
-
 # ---- Segment 1: model -----------------------------------------------------
 
 

package/bin/cctally

@@ -1008,6 +1008,9 @@ get_codex_entries = _cctally_cache.get_codex_entries
 _sum_codex_cost_for_range = _cctally_cache._sum_codex_cost_for_range
 get_entries = _cctally_cache.get_entries
 open_cache_db = _cctally_cache.open_cache_db
+open_conversations_db = _cctally_cache.open_conversations_db
+sync_claude_conversations = _cctally_cache.sync_claude_conversations
+sync_codex_conversations = _cctally_cache.sync_codex_conversations
 _reset_orphan_warning_throttle = _cctally_cache._reset_orphan_warning_throttle
 cmd_cache_sync = _cctally_cache.cmd_cache_sync