cctally 1.75.1 → 1.77.0

package/bin/_cctally_core.py

@@ -60,8 +60,10 @@ def _init_paths_from_env() -> None:
     `import _cctally_core`).
     """
     global APP_DIR, LEGACY_APP_DIR, LOG_DIR, DEV_MODE
-    global DB_PATH, CACHE_DB_PATH
+    global DB_PATH, CACHE_DB_PATH, CONVERSATIONS_DB_PATH
     global CACHE_LOCK_PATH, CACHE_LOCK_CODEX_PATH, CACHE_LOCK_MAINTENANCE_PATH
+    global CONVERSATIONS_LOCK_PATH, CONVERSATIONS_LOCK_CODEX_PATH
+    global CONVERSATIONS_LOCK_MAINTENANCE_PATH
     global CONFIG_LOCK_PATH
     global CONFIG_PATH, MIGRATION_ERROR_LOG_PATH, CHANGELOG_PATH
     global HOOK_TICK_LOG_DIR, HOOK_TICK_LOG_PATH, HOOK_TICK_LOG_ROTATED_PATH
@@ -101,6 +103,7 @@ def _init_paths_from_env() -> None:
 
     DB_PATH = APP_DIR / "stats.db"
     CACHE_DB_PATH = APP_DIR / "cache.db"
+    CONVERSATIONS_DB_PATH = APP_DIR / "conversations.db"
 
     CACHE_LOCK_PATH = APP_DIR / "cache.db.lock"
     CACHE_LOCK_CODEX_PATH = APP_DIR / "cache.db.codex.lock"
@@ -108,6 +111,14 @@ def _init_paths_from_env() -> None:
     # retention prune across processes, held ABOVE the two provider flocks so a
     # rebuild/reingest cannot land between candidate selection and deletion.
     CACHE_LOCK_MAINTENANCE_PATH = APP_DIR / "cache.db.maintenance.lock"
+    # #320: transcript ingest and maintenance are physically independent from
+    # the latency-sensitive token/quota cache.  Never reuse the cache.db flock
+    # namespace for conversations.db writes.
+    CONVERSATIONS_LOCK_PATH = APP_DIR / "conversations.db.lock"
+    CONVERSATIONS_LOCK_CODEX_PATH = APP_DIR / "conversations.db.codex.lock"
+    CONVERSATIONS_LOCK_MAINTENANCE_PATH = (
+        APP_DIR / "conversations.db.maintenance.lock"
+    )
     CONFIG_LOCK_PATH = APP_DIR / "config.json.lock"
 
     CONFIG_PATH = APP_DIR / "config.json"

package/bin/_cctally_dashboard.py

@@ -359,7 +359,9 @@ import _lib_log
 from _cctally_config import save_config, _load_config_unlocked
 from _cctally_db import _render_migration_error_banner
 from _cctally_cache import (
-    get_entries, iter_entries, iter_entries_with_id, open_cache_db, sync_cache,
+    get_entries, iter_entries, iter_entries_with_id, open_cache_db,
+    open_conversations_db, sync_cache, sync_claude_conversations,
+    sync_codex_conversations,
     _prune_orphaned_cache_entries,
 )
 from _lib_snapshot_cache import (
@@ -1175,8 +1177,7 @@ def _dashboard_sync_loop(
 
 def _dashboard_maybe_prune_retention() -> None:
     """#313 P3 (F7): throttled transcript retention prune driven from the
-    dashboard sync thread. Runs once per iteration INSIDE the measured cooldown
-    work (Task 6), so its cost paces the deadline. Opens a dedicated cache
+    independent conversation sync thread. Opens a dedicated transcript
     connection (no provider flock, no open transaction) for the orchestrator.
     No-op when retention is disabled; never raises (a prune failure must not
     crash the sync thread)."""
@@ -1187,7 +1188,7 @@ def _dashboard_maybe_prune_retention() -> None:
         retention_days = resolve_retention_days(c.load_config())
         if retention_days <= 0:
             return
-        conn = c.open_cache_db()
+        conn = c.open_conversations_db(attach_cache=False)
         try:
             retention._maybe_prune_conversation_retention(
                 conn,
@@ -4100,10 +4101,6 @@ def _qs_str(q: dict, key: str, default: str | None) -> str | None:
 _DEBUG_CACHE_TABLES = (
     "session_entries",
     "session_files",
-    "conversation_messages",
-    "conversation_sessions",
-    "conversation_ai_titles",
-    "conversation_file_touches",
     "codex_session_entries",
     "codex_session_files",
 )
@@ -6769,11 +6766,6 @@ def cmd_dashboard(args: argparse.Namespace) -> int:
                         and _time.monotonic() - last_heal[0] >= 60.0):
                     last_heal[0] = _time.monotonic()
                     _dashboard_self_heal_orphans(skip_sync=self._skip_sync)
-                # #313 P3 (F7): throttled transcript retention prune, inside the
-                # measured iteration so its cost is in the cooldown work. Gated
-                # off under --no-sync (frozen mode makes no cache writes).
-                if not self._skip_sync:
-                    _dashboard_maybe_prune_retention()
 
             # Work-proportional cooldown (F10): sleep to t0 + max(interval, work)
             # so a slow rebuild cannot peg a full core. The manual POST /api/sync
@@ -6797,6 +6789,47 @@ def cmd_dashboard(args: argparse.Namespace) -> int:
     if sync_thread is not None:
         sync_thread.start()
 
+    # #320: transcript/search ingestion runs on its own thread and SQLite file.
+    # A multi-GB first rebuild or a contended conversations.db therefore cannot
+    # delay `_run_sync_now`, its `last_sync_at` stamp, or core SSE publication.
+    conversation_sync_stop = threading.Event()
+
+    def _conversation_sync_loop() -> None:
+        interval = max(5.0, float(args.sync_interval))
+        while not conversation_sync_stop.is_set():
+            conn = None
+            try:
+                conn = open_conversations_db()
+                sync_claude_conversations(conn)
+                sync_codex_conversations(conn)
+                _dashboard_maybe_prune_retention()
+            except (OSError, sqlite3.DatabaseError) as exc:
+                eprint(f"[conversations] background sync unavailable: {exc}")
+            except Exception as exc:  # noqa: BLE001
+                # Transcript parsing/normalization is deliberately outside the
+                # core freshness loop. Keep this worker alive so a later clean
+                # tick can self-heal instead of permanently stopping after one
+                # malformed provider record.
+                eprint(
+                    "[conversations] background sync failed: "
+                    f"{type(exc).__name__}: {exc}"
+                )
+            finally:
+                if conn is not None:
+                    conn.close()
+            conversation_sync_stop.wait(interval)
+
+    conversation_sync_thread = (
+        None if args.no_sync
+        else threading.Thread(
+            target=_conversation_sync_loop,
+            daemon=True,
+            name="dashboard-conversations-sync",
+        )
+    )
+    if conversation_sync_thread is not None:
+        conversation_sync_thread.start()
+
     # Spec §3.5 (codex review fix #5): update-check thread runs even
     # under --no-sync (frozen-data sessions still surface new versions).
     # Dedicated stop event so we can join cleanly on shutdown without
@@ -6879,6 +6912,9 @@ def cmd_dashboard(args: argparse.Namespace) -> int:
     finally:
         if sync_thread is not None:
             sync_thread.stop()
+        conversation_sync_stop.set()
+        if conversation_sync_thread is not None:
+            conversation_sync_thread.join(timeout=2)
         update_check_stop.set()
         srv.shutdown()
         http_thread.join(timeout=2)

package/bin/_cctally_dashboard_conversation.py

@@ -20,9 +20,9 @@ What lives here (spec §6):
   (``_require_transcripts_allowed`` / ``_transcript_gate`` / …) STAY on the
   class and are reached via the ``handler`` parameter (spec §6).
 
-Late-binding (spec §6): the two bare ``open_cache_db`` calls (in
+Late-binding (spec §6): the conversation DB opener calls (in
 ``_run_conversation_query_impl`` + the events handler) reach
-``sys.modules["_cctally_dashboard"].open_cache_db(...)`` — patched on the
+``sys.modules["_cctally_dashboard"].open_conversations_db(...)`` — patched on the
 dashboard module object at ``tests/test_conversation_endpoints.py:674``.
 
 Cross-module reaches (spec §2.1 "fully-qualify cross-module refs"): the
@@ -44,7 +44,13 @@ import socket
 import sqlite3
 import sys
 
-from _cctally_cache import sync_cache, sync_codex_cache, _codex_provider_roots
+from _cctally_cache import (
+    open_cache_db,
+    sync_codex_cache,
+    sync_claude_conversations,
+    sync_codex_conversations,
+    _codex_provider_roots,
+)
 
 # Live-tail watch-loop tuning — used ONLY by _handle_get_conversation_events_impl
 # below, so moved here with the events handler (spec §4.1 / §6).
@@ -91,7 +97,7 @@ class _BadConversationFilter(Exception):
 
 
 def _cached_file_sigs(conn, paths):
-    """{path: size_bytes} from session_files for the given paths — the cache's
+    """{path: size_bytes} from conversation_source_files for the given paths — the transcript cache's
     own view of how far each file is ingested. Size-only by design, matching the
     watch kernel's size-only signature (`file_sig`) and sync_cache's size-only
     delta signal: mtime is NOT consulted, because a size-unchanged ingest does
@@ -105,7 +111,7 @@ def _cached_file_sigs(conn, paths):
     placeholders = ",".join("?" for _ in paths)
     try:
         rows = conn.execute(
-            f"SELECT path, size_bytes FROM session_files "
+            f"SELECT path, size_bytes FROM conversation_source_files "
             f"WHERE path IN ({placeholders})", list(paths)).fetchall()
     except sqlite3.OperationalError:
         return out
@@ -115,7 +121,7 @@ def _cached_file_sigs(conn, paths):
 
 
 def _codex_cached_file_sigs(conn, paths):
-    """{path: size_bytes} from ``codex_session_files`` for the given paths — the
+    """{path: size_bytes} from ``codex_conversation_source_files`` for the given paths — the
     Codex analogue of ``_cached_file_sigs`` (spec §5.2). Size-only, matching the
     watch kernel's size-only signature and ``sync_codex_cache``'s size-only
     delta. Paths with no row are absent → treated as changed. Baselines the
@@ -129,7 +135,7 @@ def _codex_cached_file_sigs(conn, paths):
     placeholders = ",".join("?" for _ in paths)
     try:
         rows = conn.execute(
-            f"SELECT path, size_bytes FROM codex_session_files "
+            f"SELECT path, size_bytes FROM codex_conversation_source_files "
             f"WHERE path IN ({placeholders})", list(paths)).fetchall()
     except sqlite3.OperationalError:
         return out
@@ -144,7 +150,7 @@ def _codex_all_committed_sizes(conn):
     baseline. A plain SELECT (no lock), so it never widens the SSE lock scope."""
     try:
         rows = conn.execute(
-            "SELECT path, size_bytes FROM codex_session_files").fetchall()
+            "SELECT path, size_bytes FROM codex_conversation_source_files").fetchall()
     except sqlite3.OperationalError:
         return {}
     return {p: size for (p, size) in rows}
@@ -162,7 +168,7 @@ def _codex_classified_paths(conn, paths):
     placeholders = ",".join("?" for _ in paths)
     try:
         rows = conn.execute(
-            f"SELECT path FROM codex_session_files "
+            f"SELECT path FROM codex_conversation_source_files "
             f"WHERE path IN ({placeholders}) AND last_conversation_key IS NOT NULL",
             list(paths)).fetchall()
     except sqlite3.OperationalError:
@@ -444,12 +450,12 @@ def _run_conversation_query_impl(handler, kernel_call, log_label):
     has ALREADY been sent and the caller must just ``return``; ``ok=True``
     carries the kernel result (which may itself be ``None`` — the reader's
     404 sentinel — so the explicit flag, not ``body is None``, signals
-    failure). An ``open_cache_db`` failure is a ``cache unavailable:`` 500;
+    failure). An ``open_conversations_db`` failure is a ``cache unavailable:`` 500;
     a kernel exception is logged as ``<log_label> failed: %r`` and returned
     as a ``{type}: {msg}`` 500 — byte-identical to the inlined handlers.
     """
     try:
-        conn = sys.modules["_cctally_dashboard"].open_cache_db()  # late-binding: patched at test_conversation_endpoints.py:674
+        conn = sys.modules["_cctally_dashboard"].open_conversations_db()
     except (sqlite3.DatabaseError, OSError) as exc:
         handler._respond_json(500, {"error": f"cache unavailable: {exc}"})
         return False, None
@@ -785,7 +791,7 @@ def _bare_conversation_events(handler, session_id: str) -> None:
     _send_sse_headers(handler)
     passive = bool(type(handler).no_sync)
     try:
-        conn = sys.modules["_cctally_dashboard"].open_cache_db()  # late-binding: patched at test_conversation_endpoints.py:674
+        conn = sys.modules["_cctally_dashboard"].open_conversations_db()
     except (sqlite3.DatabaseError, OSError):
         # Cache unavailable — degrade to keep-alive only; client backstop
         # tick still surfaces turns. (Headers already sent; can't 500.)
@@ -796,7 +802,7 @@ def _bare_conversation_events(handler, session_id: str) -> None:
         return cq.session_source_paths(conn, session_id) if conn else []
 
     def _ingest(changed):
-        return sync_cache(conn, only_paths=set(changed))
+        return sync_claude_conversations(conn, only_paths=set(changed))
 
     try:
         _run_conversation_events_stream(
@@ -840,7 +846,24 @@ def _make_codex_discovery_step(handler, conn, conversation_key, cq_codex):
         if not to_ingest:
             return files, False
         try:
-            sync_codex_cache(conn, only_paths=set(to_ingest))
+            # A newly discovered child has no compact thread row yet. Advance
+            # the independent core cursor first so transcript normalization can
+            # link the child through the read-only attached cache, then commit
+            # the transcript cursor. Neither lock is held across the other.
+            core = open_cache_db()
+            try:
+                core_stats = sync_codex_cache(
+                    core, only_paths=set(to_ingest)
+                )
+            finally:
+                core.close()
+            if not core_stats.targeted_clean:
+                return files, False
+            transcript_stats = sync_codex_conversations(
+                conn, only_paths=set(to_ingest)
+            )
+            if not transcript_stats.targeted_clean:
+                return files, False
         except sqlite3.DatabaseError:
             return files, False
         # Reap the now-classified candidates (child → already widened; non-child
@@ -866,7 +889,7 @@ def _qualified_conversation_events(handler, key: str) -> None:
     targeted ingest + the directory-frontier child discovery."""
     disp = _conversation_dispatch_impl()
     try:
-        conn = sys.modules["_cctally_dashboard"].open_cache_db()  # late-binding: patched at test_conversation_endpoints.py:674
+        conn = sys.modules["_cctally_dashboard"].open_conversations_db()
     except (sqlite3.DatabaseError, OSError):
         conn = None
 
@@ -921,7 +944,7 @@ def _qualified_conversation_events(handler, key: str) -> None:
             return cq_codex.codex_conversation_source_paths(conn, key)
 
         def _ingest(changed):
-            return sync_codex_cache(conn, only_paths=set(changed))
+            return sync_codex_conversations(conn, only_paths=set(changed))
 
         cached = lambda paths: _codex_cached_file_sigs(conn, paths)
         discovery = _make_codex_discovery_step(handler, conn, key, cq_codex)
@@ -932,7 +955,7 @@ def _qualified_conversation_events(handler, key: str) -> None:
             return cq.session_source_paths(conn, native)
 
         def _ingest(changed):
-            return sync_cache(conn, only_paths=set(changed))
+            return sync_claude_conversations(conn, only_paths=set(changed))
 
         cached = lambda paths: _cached_file_sigs(conn, paths)
         discovery = None

package/bin/_cctally_dashboard_sources.py

@@ -45,7 +45,7 @@ from _lib_quota import (
     quota_freshness,
     select_baseline,
 )
-from _lib_jsonl import CodexEntry
+from _lib_jsonl import CodexEntry, codex_model_scoped_quota_pool
 from _lib_fmt import stable_sum
 from _lib_aggregators import _aggregate_codex_buckets
 from _lib_five_hour import _FIVE_HOUR_JITTER_FLOOR_SECONDS
@@ -103,6 +103,21 @@ class CodexWeeklyPeriod:
     used_percent: float | None = None
 
 
+def _is_model_scoped_codex_quota(logical_limit_key: object) -> bool:
+    """Whether an interpreted native identity belongs outside standard quota."""
+    if not isinstance(logical_limit_key, str):
+        return False
+    try:
+        payload = json.loads(logical_limit_key)
+    except (json.JSONDecodeError, TypeError):
+        return False
+    return (
+        isinstance(payload, dict)
+        and isinstance(payload.get("modelPool"), str)
+        and bool(payload["modelPool"].strip())
+    )
+
+
 def _resolve_codex_weekly_cycle(
     observations: Iterable[object],
     now_utc: dt.datetime,
@@ -113,6 +128,8 @@ def _resolve_codex_weekly_cycle(
     for history in build_history(tuple(observations)):
         if history.identity.window_minutes != 10_080:
             continue
+        if _is_model_scoped_codex_quota(history.identity.logical_limit_key):
+            continue
         baseline = select_baseline(history.observations, now_utc)
         if baseline is None or baseline.resets_at <= now_utc:
             continue
@@ -161,7 +178,8 @@ def _codex_weekly_periods(
     placeholders = ",".join("?" for _ in roots)
     try:
         rows = stats_conn.execute(
-            "SELECT source_root_key, resets_at_utc, nominal_start_at_utc, current_percent "
+            "SELECT source_root_key, logical_limit_key, resets_at_utc, "
+            "nominal_start_at_utc, current_percent "
             "FROM quota_window_blocks "
             "WHERE source='codex' AND window_minutes=10080 "
             f"AND source_root_key IN ({placeholders}) AND orphaned_at IS NULL "
@@ -174,7 +192,9 @@ def _codex_weekly_periods(
 
     raw_boundaries: list[tuple[dt.datetime, dt.datetime, set[str], list[float]]] = []
 
-    for root_key, resets_at_raw, start_at_raw, current_percent in rows:
+    for root_key, logical_limit_key, resets_at_raw, start_at_raw, current_percent in rows:
+        if _is_model_scoped_codex_quota(logical_limit_key):
+            continue
         try:
             start_at = dt.datetime.fromisoformat(str(start_at_raw).replace("Z", "+00:00"))
             resets_at = dt.datetime.fromisoformat(str(resets_at_raw).replace("Z", "+00:00"))
@@ -713,22 +733,33 @@ def _codex_conversation_metadata(
 
     ``state_5.sqlite.threads.title`` is Codex's persisted user-facing task name.
     Conversation rollup titles are derived from prompt text and therefore must
-    never be substituted for that name on the dashboard. Accounting remains
-    authoritative for totals; conversation rollups only decorate projects.
+    never be substituted for that name on the dashboard. Project attribution
+    is derived from the compact thread ``cwd``/``git_json`` retained in
+    ``cache.db`` so non-conversation panels never open ``conversations.db``.
     """
     metadata: dict[tuple[str, str], dict[str, object]] = {}
     try:
-        rows = tuple(cache_conn.execute(
+        core_rows = tuple(cache_conn.execute(
             "SELECT t.source_root_key, t.source_path, t.native_thread_id, "
             "(SELECT e.session_id FROM codex_session_entries AS e "
             " WHERE e.source_root_key=t.source_root_key AND e.source_path=t.source_path "
             " ORDER BY e.id LIMIT 1) AS accounting_session_id, "
-            "r.project_key, r.project_label, r.started_utc "
+            "t.cwd, t.git_json, t.first_seen_utc, t.last_seen_utc "
             "FROM codex_conversation_threads AS t "
-            "LEFT JOIN codex_conversation_rollups AS r "
-            "ON r.conversation_key=t.conversation_key "
-            "ORDER BY r.last_activity_utc DESC, t.conversation_key DESC"
+            "ORDER BY t.last_seen_utc DESC, t.conversation_key DESC"
         ))
+        from _cctally_cache import _codex_conversation_project_attribution
+        rows = tuple(
+            (
+                root_key, source_path, native_thread_id, accounting_session_id,
+                *_codex_conversation_project_attribution(root_key, cwd, git_json),
+                first_seen_at,
+            )
+            for (
+                root_key, source_path, native_thread_id, accounting_session_id,
+                cwd, git_json, first_seen_at, _last_seen_at,
+            ) in core_rows
+        )
         file_aliases = tuple(cache_conn.execute(
             "SELECT source_root_key, path, last_native_thread_id, last_session_id "
             "FROM codex_session_files "
@@ -1833,6 +1864,8 @@ def _build_codex_native_weekly_view(
     labels: dict[str, str] = {}
     periods_by_bucket: dict[str, CodexWeeklyPeriod] = {}
     for entry in entries:
+        if codex_model_scoped_quota_pool(getattr(entry, "model", None)) is not None:
+            continue
         timestamp = getattr(entry, "timestamp").astimezone(UTC)
         root_key = str(getattr(entry, "source_root_key", "") or "")
         period = next((