cctally 1.76.0 → 1.77.0

package/bin/_cctally_doctor.py

@@ -429,16 +429,35 @@ def doctor_gather_state(
     # Conversation-sessions rollup consistency (#217 S1 / U9). Two cheap COUNTs
     # (graceful None on a missing table / unreadable DB) + an in-progress signal
     # so a transient mid-sync mismatch never WARNs. The in-progress signal is a
-    # NON-BLOCKING cache.db.lock flock probe (a writer mid-walk holds it) OR the
+    # NON-BLOCKING conversations flock probe (a writer mid-walk holds it) OR the
     # presence of any pending reingest/split/backfill cache_meta flag — doctor
     # stays read-only and never blocks on the lock.
     conv_sessions_rollup_count = None
     conv_messages_distinct_sessions = None
     conv_rollup_sync_in_progress = False
-    try:
-        if _cctally_core.CACHE_DB_PATH.exists():
-            conn = sqlite3.connect(str(_cctally_core.CACHE_DB_PATH))
+    conversations_db_page_count = None
+    conversations_db_freelist_count = None
+    try:
+        if _cctally_core.CONVERSATIONS_DB_PATH.exists():
+            # This gather also runs inside dashboard snapshot precompute. A
+            # transcript writer may hold an exclusive SQLite lock, so use a
+            # read-only zero-timeout probe: conversation health can degrade,
+            # but it must never delay core snapshot freshness (#320).
+            conv_uri = (
+                _cctally_core.CONVERSATIONS_DB_PATH.resolve().as_uri()
+                + "?mode=ro"
+            )
+            conn = sqlite3.connect(conv_uri, uri=True, timeout=0.0)
             try:
+                try:
+                    row = conn.execute("PRAGMA page_count").fetchone()
+                    if row and row[0] is not None:
+                        conversations_db_page_count = int(row[0])
+                    row = conn.execute("PRAGMA freelist_count").fetchone()
+                    if row and row[0] is not None:
+                        conversations_db_freelist_count = int(row[0])
+                except sqlite3.Error:
+                    pass
                 try:
                     row = conn.execute(
                         "SELECT COUNT(*) FROM conversation_sessions"
@@ -472,12 +491,12 @@ def doctor_gather_state(
                     pass
             finally:
                 conn.close()
-        # Non-blocking flock probe: if a writer (sync_cache / a reingest) holds
-        # the cache.db.lock, the rollup may be mid-recompute → in progress. We
+        # Non-blocking flock probe: if a transcript writer/reingest holds the
+        # conversations.db lock, the rollup may be mid-recompute → in progress. We
         # acquire LOCK_EX|LOCK_NB and immediately release; failure (held) is the
         # signal. Never blocks (LOCK_NB), so doctor stays read-only + prompt.
         if not conv_rollup_sync_in_progress:
-            lock_path = _cctally_core.CACHE_LOCK_PATH
+            lock_path = _cctally_core.CONVERSATIONS_LOCK_PATH
             if lock_path is not None and pathlib.Path(lock_path).exists():
                 import fcntl as _fcntl
                 lock_fh = open(str(lock_path), "w")
@@ -669,6 +688,15 @@ def doctor_gather_state(
         for _name, _lp in (
             ("cache.db.lock", _cctally_core.CACHE_LOCK_PATH),
             ("cache.db.codex.lock", _cctally_core.CACHE_LOCK_CODEX_PATH),
+            ("conversations.db.lock", _cctally_core.CONVERSATIONS_LOCK_PATH),
+            (
+                "conversations.db.codex.lock",
+                _cctally_core.CONVERSATIONS_LOCK_CODEX_PATH,
+            ),
+            (
+                "conversations.db.maintenance.lock",
+                _cctally_core.CONVERSATIONS_LOCK_MAINTENANCE_PATH,
+            ),
         ):
             if not _lp.exists():
                 locks_held[_name] = False
@@ -877,6 +905,8 @@ def doctor_gather_state(
         # #315: read-only cache free-page evidence for the reclaim hint.
         cache_db_page_count=cache_db_page_count,
         cache_db_freelist_count=cache_db_freelist_count,
+        conversations_db_page_count=conversations_db_page_count,
+        conversations_db_freelist_count=conversations_db_freelist_count,
         codex_quota_windows=codex_quota_windows,
         codex_hook_roots=codex_hook_roots,
         codex_lifecycle_activity_24h=codex_lifecycle_activity_24h,

package/bin/_cctally_parser.py

@@ -2924,7 +2924,7 @@ def _build_db_parser(subparsers, name, *, help_text, xref=None):
     )
     db_vacuum.add_argument(
         "--db",
-        choices=("cache", "stats", "all"),
+        choices=("cache", "conversations", "stats", "all"),
         default="cache",
         help="Which DB to VACUUM (default: cache)",
     )

package/bin/_cctally_quota.py

@@ -279,33 +279,32 @@ def load_codex_quota_observations(
             }
             return required <= columns
 
-        has_conversation_events = has_columns(
-            "codex_conversation_events",
-            {"source_path", "line_offset", "record_type", "payload_json"},
-        )
         has_session_entries = has_columns(
             "codex_session_entries", {"source_path", "line_offset", "model"},
         )
-        if has_conversation_events:
-            model_expr = """
-                (SELECT json_extract(events.payload_json, '$.payload.model')
-                   FROM codex_conversation_events AS events
-                  WHERE events.source_path=quota_window_snapshots.source_path
-                    AND events.line_offset<=quota_window_snapshots.line_offset
-                    AND events.record_type IN ('turn_context','session_meta')
-                    AND json_type(events.payload_json, '$.payload.model')='text'
-                  ORDER BY events.line_offset DESC LIMIT 1) AS observed_model
-            """
-        elif has_session_entries:
-            model_expr = """
-                (SELECT entries.model
-                   FROM codex_session_entries AS entries
-                  WHERE entries.source_path=quota_window_snapshots.source_path
-                    AND entries.line_offset<=quota_window_snapshots.line_offset
-                  ORDER BY entries.line_offset DESC LIMIT 1) AS observed_model
-            """
-        else:
-            model_expr = "NULL AS observed_model"
+        has_observed_model = has_columns(
+            "quota_window_snapshots", {"observed_model"},
+        )
+        entry_lookup = (
+            "(SELECT entries.model FROM codex_session_entries AS entries "
+            "WHERE entries.source_path=quota_window_snapshots.source_path "
+            "AND entries.line_offset<=quota_window_snapshots.line_offset "
+            "ORDER BY entries.line_offset DESC LIMIT 1)"
+        )
+        # A file's terminal model is not evidence for an earlier quota sample:
+        # the model may change later in the rollout. Prefer the compact stamp,
+        # then only a nearest-prior accounting context for legacy rows. If
+        # neither exists, leave the pool unscoped rather than fabricating it.
+        fallback_model = entry_lookup if has_session_entries else "NULL"
+        selected_model = (
+            f"COALESCE(quota_window_snapshots.observed_model,{fallback_model})"
+            if has_observed_model and fallback_model != "NULL"
+            else (
+                "quota_window_snapshots.observed_model"
+                if has_observed_model else fallback_model
+            )
+        )
+        model_expr = f"{selected_model} AS observed_model"
         sql = """
             SELECT source, source_root_key, source_path, line_offset,
                    captured_at_utc, observed_slot, logical_limit_key, limit_id,

package/bin/_cctally_transcript.py

@@ -86,7 +86,7 @@ def _cmd_transcript_export(args) -> int:
         eprint(_SPEED_ONLY_CODEX_MSG)
         return 2
 
-    conn = c.open_cache_db()
+    conn = c.open_conversations_db()
     try:
         cq = c._load_sibling("_lib_conversation_query")
         md = cq.get_conversation_export(conn, session_id, scope)
@@ -118,7 +118,7 @@ def _cmd_transcript_export_qualified(
         return 2
     speed = c._resolve_codex_speed(speed_arg or "auto")
 
-    conn = c.open_cache_db()
+    conn = c.open_conversations_db()
     try:
         env = disp.neutral_export(
             conn, session_id, scope=scope, effective_speed=speed)
@@ -189,7 +189,7 @@ def _cmd_transcript_search_claude(args) -> int:
             eprint(f"transcript: {exc}")
             return 2
 
-    conn = c.open_cache_db()
+    conn = c.open_conversations_db()
     try:
         cq = c._load_sibling("_lib_conversation_query")
         try:
@@ -264,7 +264,7 @@ def _cmd_transcript_search_codex(args) -> int:
             eprint("transcript: invalid --cursor")
             return 2
 
-    conn = c.open_cache_db()
+    conn = c.open_conversations_db()
     try:
         result = disp.neutral_search(
             conn, query, source="codex", kind=kind,

package/bin/_cctally_tui.py

@@ -1772,40 +1772,12 @@ def _tui_build_sessions(
             (), now_utc=now_utc, limit=limit, display_tz=None,
             aggregated_override=aggregated_override,
         )
-    rows = list(view.rows)
-    # Attach human-readable titles for the dashboard/TUI Sessions surface.
-    # Reuses the conversation viewer's title derivation (AI title wins, else
-    # first non-marker human line), which reads the CACHE db
-    # (conversation_messages / conversation_ai_titles). Fail-soft: any cache
-    # failure leaves titles ``None`` -> the client renders an em-dash. One
-    # bounded indexed query (<= `limit` session ids, rides idx_conv_session_ts)
-    # per snapshot build. Titles are stashed unconditionally on this
-    # server-internal row; the privacy gate is applied later, at envelope
-    # serialization (``snapshot_to_envelope(transcripts_visible=...)``).
-    if rows:
-        session_ids = [r.session_id for r in rows if r.session_id]
-        try:
-            conn = c.open_cache_db()
-        except (sqlite3.DatabaseError, OSError):
-            conn = None
-        if conn is not None:
-            try:
-                titles = c._load_sibling(
-                    "_lib_conversation_query"
-                )._session_titles_map(conn, session_ids)
-                rows = [
-                    dataclasses.replace(r, title=titles.get(r.session_id))
-                    if r.session_id in titles else r
-                    for r in rows
-                ]
-            except (sqlite3.DatabaseError, OSError):
-                pass  # fail-soft: leave titles None
-            finally:
-                try:
-                    conn.close()
-                except sqlite3.Error:
-                    pass
-    return rows
+    # #320: transcript-derived titles are optional decoration. The core
+    # dashboard/TUI snapshot must never open conversations.db, because even a
+    # fail-soft read pays SQLite's lock timeout before it can fail. Conversation
+    # routes retain title derivation; the accounting Sessions panel renders its
+    # existing em-dash fallback when the independent store is unavailable.
+    return list(view.rows)
 
 
 def _tui_sessions_cached(

package/bin/_lib_codex_conversation_query.py

@@ -71,9 +71,25 @@ _SEARCH_BADGE = {
 
 
 def codex_normalization_authoritative(conn: sqlite3.Connection) -> bool:
-    """True iff migration 025 is stamped applied — i.e. the normalized corpus is
-    authoritative (§3.5). A missing ``schema_migrations`` table (bare
-    ``_apply_cache_schema`` conn, or a pre-migration cache) reads as pending."""
+    """True iff the normalized Codex corpus is authoritative (§3.5).
+
+    Split stores use their provider-local rebuild marker: current schema alone
+    is not authority while migration 028's byte-zero replay is pending. Legacy
+    monolithic/bare connections retain the migration-025 stamp contract.
+    """
+    try:
+        split = conn.execute(
+            "SELECT 1 FROM cache_meta "
+            "WHERE key='conversation_schema_version'"
+        ).fetchone() is not None
+        if split:
+            pending = conn.execute(
+                "SELECT 1 FROM cache_meta "
+                "WHERE key='conversation_rebuild_codex_pending'"
+            ).fetchone() is not None
+            return not pending
+    except sqlite3.OperationalError:
+        pass
     try:
         row = conn.execute(
             "SELECT 1 FROM schema_migrations WHERE name = ?",

package/bin/_lib_conversation_query.py

@@ -2048,7 +2048,9 @@ def _assemble_memo_clear() -> None:
 def _assemble_memo_key(conn, session_id):
     """Return (session_id, msg_count, last_activity_utc, entry_mutation_seq), or
     None to signal 'bypass the memo' — a missing rollup row or an unreadable
-    cache_meta (bare/path-less conn). The caller has already confirmed the
+    accounting ``cache_meta``. Split-store connections read that watermark
+    from the read-only ``cache_db`` attachment; legacy/bare test connections
+    retain the main-schema fallback. The caller has already confirmed the
     rollup is authoritative."""
     try:
         row = conn.execute(
@@ -2059,9 +2061,12 @@ def _assemble_memo_key(conn, session_id):
     if row is None:
         return None
     try:
+        schemas = {row[1] for row in conn.execute("PRAGMA database_list")}
+        meta_schema = "cache_db" if "cache_db" in schemas else "main"
         seq_row = conn.execute(
-            "SELECT value FROM cache_meta "
-            "WHERE key='session_entries_mutation_seq'").fetchone()
+            f"SELECT value FROM {meta_schema}.cache_meta "
+            "WHERE key='session_entries_mutation_seq'"
+        ).fetchone()
     except sqlite3.OperationalError:
         return None
     seq = int(seq_row[0]) if seq_row and seq_row[0] is not None else 0

package/bin/_lib_conversation_retention.py

@@ -277,7 +277,7 @@ def _maybe_prune_conversation_retention(
         core.APP_DIR.mkdir(parents=True, exist_ok=True)
     except OSError:
         return None
-    maint_fh = open(core.CACHE_LOCK_MAINTENANCE_PATH, "w")
+    maint_fh = open(core.CONVERSATIONS_LOCK_MAINTENANCE_PATH, "w")
     try:
         try:
             fcntl.flock(maint_fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
@@ -285,13 +285,13 @@ def _maybe_prune_conversation_retention(
             return None  # another prune holds it; skip cleanly, do NOT stamp
         if not force and not _retention_due(conn, now_utc):
             return None
-        claude_fh = open(core.CACHE_LOCK_PATH, "w")
+        claude_fh = open(core.CONVERSATIONS_LOCK_PATH, "w")
         try:
             try:
                 fcntl.flock(claude_fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
             except (BlockingIOError, OSError):
                 return None  # a Claude sync is mid-flight; retry next cycle
-            codex_fh = open(core.CACHE_LOCK_CODEX_PATH, "w")
+            codex_fh = open(core.CONVERSATIONS_LOCK_CODEX_PATH, "w")
             try:
                 try:
                     fcntl.flock(codex_fh, fcntl.LOCK_EX | fcntl.LOCK_NB)

package/bin/_lib_doctor.py

@@ -206,6 +206,10 @@ class DoctorState:
     # cache was absent or unreadable; the check degrades to OK.
     cache_db_page_count: Optional[int] = None
     cache_db_freelist_count: Optional[int] = None
+    # #320: the independently large transcript store needs the same reclaim
+    # visibility, with remediation targeted at its own VACUUM surface.
+    conversations_db_page_count: Optional[int] = None
+    conversations_db_freelist_count: Optional[int] = None
     # #294 S2: root-qualified physical Codex quota freshness, per-root native
     # hook state, and lifecycle activity are gathered by _cctally_doctor.
     codex_quota_windows: Optional[list[dict]] = None
@@ -1235,21 +1239,22 @@ def _check_data_conversation_sessions_rollup(s: DoctorState) -> CheckResult:
     row per distinct ``conversation_messages.session_id`` (#217 S1 / U9).
 
     OK when the two counts are equal, when either is ``None`` (pre-rollup or an
-    unreadable cache.db — consistent with the kernel's graceful-degrade posture),
+    unreadable conversations.db — consistent with the kernel's graceful-degrade
+    posture),
     OR when a sync/reingest/backfill is in progress. WARN ONLY on a mismatch
     observed in a QUIESCENT cache.
 
-    False-WARN avoidance (Codex P2): ``sync_cache`` commits
+    False-WARN avoidance (Codex P2): ``sync_claude_conversations`` commits
     ``conversation_messages`` per file *before* the ``conversation_sessions``
     recompute, and resumable reingest commits per file before its rollup
     completes — so an unsynchronized read can transiently mismatch. The
     in-progress signal (``conv_rollup_sync_in_progress``) is set by
-    ``doctor_gather_state`` from a NON-BLOCKING ``cache.db.lock`` flock probe
+    ``doctor_gather_state`` from a NON-BLOCKING ``conversations.db.lock`` flock probe
     (lock held ⇒ a writer is mid-flight) AND the presence of any pending
     ``cache_meta`` reingest/split/backfill flag; if either says in-progress, this
     stays OK. Doctor remains read-only and never blocks on the lock.
 
-    Informational only (no remediation): the next full ``sync_cache`` re-derives
+    Informational only (no remediation): the next conversation sync re-derives
     the rollup via its incremental DELETE+INSERT.
     """
     rollup = s.conv_sessions_rollup_count
@@ -1773,6 +1778,49 @@ def _check_db_reclaimable(s: DoctorState) -> CheckResult:
     )
 
 
+def _check_db_conversations_reclaimable(s: DoctorState) -> CheckResult:
+    """Surface transcript-store free pages without mutating the DB (#320)."""
+    page_count = s.conversations_db_page_count
+    freelist_count = s.conversations_db_freelist_count
+    ratio = None
+    if (
+        isinstance(page_count, int)
+        and not isinstance(page_count, bool)
+        and page_count > 0
+        and isinstance(freelist_count, int)
+        and not isinstance(freelist_count, bool)
+        and 0 <= freelist_count <= page_count
+    ):
+        ratio = freelist_count / page_count
+    details = {
+        "conversations_db_page_count": page_count,
+        "conversations_db_freelist_count": freelist_count,
+        "conversations_db_free_ratio": ratio,
+        "warn_ratio": DOCTOR_RECLAIMABLE_WARN_RATIO,
+    }
+    if ratio is not None and ratio >= DOCTOR_RECLAIMABLE_WARN_RATIO:
+        return CheckResult(
+            id="db.conversations_reclaimable",
+            title="Reclaimable transcript space",
+            severity="warn",
+            summary=(
+                f"high — {ratio * 100:.1f}% of conversations.db pages are free"
+            ),
+            remediation=(
+                "Run `cctally db vacuum --db conversations` to reclaim disk space."
+            ),
+            details=details,
+        )
+    return CheckResult(
+        id="db.conversations_reclaimable",
+        title="Reclaimable transcript space",
+        severity="ok",
+        summary="below threshold",
+        remediation=None,
+        details=details,
+    )
+
+
 # Each entry is (category_id, category_title, ((check_id, evaluator_fn_name), ...)).
 # The dotted check_id is the stable JSON-contract ID (spec §5.2) AND the
 # fingerprint identity-slice key (spec §5.5). When an evaluator raises,
@@ -1808,6 +1856,7 @@ _CATEGORY_DEFINITIONS: tuple[tuple[str, str, tuple[tuple[str, str], ...]], ...]
         ("db.lock_state", "_check_db_lock_state"),
         ("db.wal_size", "_check_db_wal_size"),
         ("db.reclaimable", "_check_db_reclaimable"),
+        ("db.conversations_reclaimable", "_check_db_conversations_reclaimable"),
     )),
     ("data", "Data", (
         ("data.latest_snapshot_age", "_check_data_latest_snapshot_age"),

package/bin/cctally

@@ -1008,6 +1008,9 @@ get_codex_entries = _cctally_cache.get_codex_entries
 _sum_codex_cost_for_range = _cctally_cache._sum_codex_cost_for_range
 get_entries = _cctally_cache.get_entries
 open_cache_db = _cctally_cache.open_cache_db
+open_conversations_db = _cctally_cache.open_conversations_db
+sync_claude_conversations = _cctally_cache.sync_claude_conversations
+sync_codex_conversations = _cctally_cache.sync_codex_conversations
 _reset_orphan_warning_throttle = _cctally_cache._reset_orphan_warning_throttle
 cmd_cache_sync = _cctally_cache.cmd_cache_sync