cctally 1.43.3 → 1.44.0

package/CHANGELOG.md

@@ -5,6 +5,11 @@ based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 
 ## [Unreleased]
 
+## [1.44.0] - 2026-06-14
+
+### Changed
+- Conversation viewer: the dashboard conversation browse-rail is now served from a `conversation_sessions` rollup (a small per-session COUNT/MIN/MAX table maintained inside `sync_cache` under the cache lock, indexed for the recent ordering) instead of a full-table `GROUP BY` over every message on each page load, so the rail stays fast as your history grows and when a backgrounded tab refocuses after queuing SSE ticks; the rollup is re-derivable like the rest of `cache.db` (a one-time backfill arms automatically on the next sync via cache migration `013`, with a live `GROUP BY` fallback until it lands, so the rail's output is byte-identical either way), and the rail now pauses its per-tick page-1 refetch while the tab is hidden, firing exactly one refetch on the hidden→visible transition so a freshly-revealed reader is current without re-querying every five seconds while idle.
+
 ## [1.43.3] - 2026-06-14
 
 ### Added

package/bin/_cctally_cache.py

@@ -714,6 +714,15 @@ def sync_cache(
                 "DELETE FROM cache_meta WHERE key IN "
                 "('conversation_promote_command_args_pending',"
                 " 'conversation_promote_command_args_cursor')")
+            # Browse-rail rollup: a rebuild re-derives conversation_messages from
+            # offset 0, so wipe the rollup here (in the same destructive txn,
+            # alongside clear_conversation_messages, so a crash-recovery read
+            # can't surface ghost rows) and arm the durable backfill flag. The
+            # post-walk recompute (after the per-file loop, still under the
+            # flock) consumes the flag and rebuilds the rollup from the freshly
+            # re-ingested messages, then drops it last (crash-safe).
+            conn.execute("DELETE FROM conversation_sessions")
+            _set_cache_meta(conn, "conversation_sessions_backfill_pending", "1")
             conn.commit()
             eprint("[cache-sync] rebuild: cleared Claude cached entries")
 
@@ -747,6 +756,12 @@ def sync_cache(
                     "DELETE FROM cache_meta "
                     "WHERE key='conversation_backfill_pending'"
                 )
+                # Browse-rail rollup: a #139 offset-0 backfill bulk-inserts
+                # history into conversation_messages, so arm the durable
+                # recompute flag (idempotent; covers a partial-migration state
+                # where the rollup is empty but messages just landed). The
+                # post-walk recompute rebuilds it and drops the flag last.
+                _set_cache_meta(conn, "conversation_sessions_backfill_pending", "1")
                 conn.commit()
 
             # #193: consume the deferred ai-title backfill. Cache migration 012 is
@@ -825,6 +840,13 @@ def sync_cache(
                 # three flags + cursor + gen atomically on completion. Never on the rebuild
                 # path (which already wipes + repopulates id-aware via the normal walk).
                 _resumable_reingest_conversation_messages(conn)
+                # Browse-rail rollup: a #179 reingest DELETEs + re-inserts every
+                # file's conversation_messages rows (bumping autoincrement ids and
+                # potentially MIN/MAX), so arm the durable recompute flag
+                # (idempotent; covers a partial-migration state). The post-walk
+                # recompute rebuilds the rollup and drops the flag last.
+                _set_cache_meta(conn, "conversation_sessions_backfill_pending", "1")
+                conn.commit()
 
             # #177 S6: consume the migration-010 search-column split under the
             # SAME held flock, AFTER the reingest so any just-re-ingested rows
@@ -975,6 +997,13 @@ def sync_cache(
             conn.execute(
                 "UPDATE session_files SET size_bytes = 0, last_byte_offset = 0"
             )
+            # Browse-rail rollup: truncation escalates to a full offset-0
+            # re-ingest of conversation_messages, so wipe the rollup here (in the
+            # same destructive txn, alongside clear_conversation_messages) and
+            # arm the durable backfill flag. The post-walk recompute rebuilds it
+            # from the re-ingested messages and drops the flag last (crash-safe).
+            conn.execute("DELETE FROM conversation_sessions")
+            _set_cache_meta(conn, "conversation_sessions_backfill_pending", "1")
             conn.commit()
             stats.files_reset_truncated += len(truncated_paths)
             # Force every file to re-ingest from offset 0: clearing the
@@ -984,6 +1013,14 @@ def sync_cache(
             # avoids a redundant per-file DELETE that would be a no-op).
             existing = {}
 
+        # Browse-rail rollup: accumulate the session_ids whose
+        # conversation_messages this walk touched, so the post-walk recompute can
+        # scope its DELETE+INSERT re-derive to just those sessions (steady
+        # state). Pure Python —
+        # updated only AFTER each per-file conn.commit() below, never inside the
+        # zero-write-lock read/parse region, so it adds no DML there.
+        touched_sessions: set = set()
+
         for jp in paths:
             path_str = str(jp)
             # Backfill session_id/project_path for A2 `session` subcommand.
@@ -1193,6 +1230,12 @@ def sync_cache(
                 )
                 conn.commit()
                 stats.files_processed += 1
+                # Browse-rail rollup: record the session_ids this file just
+                # committed so the post-walk recompute can scope its DELETE+INSERT
+                # re-derive to them. cr[0] is session_id per _conv_row_tuple's
+                # column order. Lands
+                # AFTER the commit (pure Python; no DML, no extra write lock).
+                touched_sessions.update(cr[0] for cr in conv_rows if cr[0] is not None)
             except sqlite3.DatabaseError as exc:
                 eprint(f"[cache] db error on {jp}: {exc}")
                 conn.rollback()
@@ -1205,6 +1248,31 @@ def sync_cache(
         if progress is not None:
             progress(stats)
 
+        # Browse-rail rollup maintenance (single post-walk recompute, under the
+        # still-held flock, after every per-file commit and before the
+        # walk-complete marker). Keyed on the DURABLE flag, not an in-memory
+        # bool: a crash between a destructive path's commit (rebuild /
+        # truncation / #139 backfill / #179 reingest, each of which armed the
+        # flag in its own committed txn) and this recompute leaves the flag set,
+        # so the next sync full-recomputes — never strands stale rollup rows
+        # (Codex gate BLOCKER 1). Flag set -> full GROUP BY over all sessions
+        # (rare, ~90ms), then drop the flag LAST (drop-it-last contract). Else ->
+        # scoped re-derive (DELETE+INSERT, not a SQL UPSERT) over just the
+        # sessions this walk touched (steady state,
+        # ~1 session/tick). Both recomputes derive COUNT/MIN/MAX from the same
+        # rows the rail's old live aggregate read, so the rollup stays
+        # byte-identical to that aggregate.
+        if _conversation_sessions_backfill_pending(conn):
+            _recompute_conversation_sessions(conn)
+            conn.execute(
+                "DELETE FROM cache_meta "
+                "WHERE key='conversation_sessions_backfill_pending'"
+            )
+            conn.commit()
+        elif touched_sessions:
+            _recompute_conversation_sessions(conn, touched_sessions)
+            conn.commit()
+
         # Walk-complete sentinel write (cctally-dev#93, D5a). Still inside the
         # held fcntl lock, before the finally-unlock. Only when the entire walk
         # was clean AND cache 001 was already applied at the start of this run
@@ -1416,6 +1484,88 @@ def _resumable_reingest_conversation_messages(conn):
     conn.commit()
 
 
+# === Browse-rail rollup (conversation_sessions) maintenance =================
+# Keeps conversation_sessions — the four structural aggregates the old live
+# GROUP BY produced — in lockstep with conversation_messages so
+# GET /api/conversations renders a page without scanning the whole message
+# table. Maintained entirely inside sync_cache under the cache.db.lock flock:
+# the steady-state per-file loop is insert-only, so a scoped re-derive
+# (DELETE+INSERT, not a SQL UPSERT) over the touched sessions suffices; the
+# rare heavy/destructive paths (rebuild,
+# truncation-escalation, #139 backfill, #179 reingest) set the durable
+# ``conversation_sessions_backfill_pending`` cache_meta flag (migration 013
+# arms it too) which forces one full recompute, crash-safe across a
+# destructive-commit/recompute crash window. The CALLER owns the commit.
+
+# All non-null sessions, recomputed from conversation_messages. Shared by the
+# full and scoped recompute so both paths derive byte-identical aggregates.
+_CONV_SESSIONS_SELECT = (
+    "SELECT session_id, COUNT(*), MIN(timestamp_utc), MAX(timestamp_utc) "
+    "FROM conversation_messages WHERE session_id IS NOT NULL"
+)
+
+
+def _conversation_sessions_backfill_pending(conn) -> bool:
+    """True while the durable ``conversation_sessions_backfill_pending`` flag is
+    set — the signal that the rollup needs one full GROUP BY recompute (armed by
+    migration 013 and by every heavy/destructive conversation_messages path).
+    Tolerates a missing cache_meta table (path-less / schema-not-applied conn) by
+    degrading to False, like the sibling reingest/backfill predicates."""
+    try:
+        return conn.execute(
+            "SELECT 1 FROM cache_meta "
+            "WHERE key='conversation_sessions_backfill_pending'"
+        ).fetchone() is not None
+    except sqlite3.OperationalError:
+        return False
+
+
+def _recompute_conversation_sessions(conn, session_ids=None) -> None:
+    """Recompute the ``conversation_sessions`` browse-rail rollup from
+    ``conversation_messages``. The caller holds the cache.db.lock flock and owns
+    the commit (this helper never commits).
+
+    ``session_ids is None`` -> FULL: wipe the whole rollup and rebuild it from a
+    single GROUP BY over every non-null session — the rare, flag-gated path
+    (rebuild / truncation / backfill / reingest / migration-013 history).
+
+    ``session_ids={...}`` -> SCOPED: for each <=400-id chunk, DELETE those rows
+    then re-INSERT the GROUP BY restricted to the chunk — the steady-state path
+    keyed on the per-file loop's touched set. DELETE+INSERT (NOT
+    INSERT…SELECT…ON CONFLICT, which trips SQLite's upsert-on-SELECT parse
+    ambiguity) also correctly drops a session whose rows all vanished — though in
+    steady state conversation_messages only gains rows, so that branch is just
+    belt-and-suspenders. The chunking keeps the ``session_id IN (…)`` parameter
+    list well under SQLite's variable limit.
+
+    The recomputed COUNT/MIN/MAX are byte-identical to the rail's prior live
+    aggregate over the same rows — that is the load-bearing invariant
+    (assert_rollup_matches_live in the maintenance test pins it)."""
+    if session_ids is None:
+        conn.execute("DELETE FROM conversation_sessions")
+        conn.execute(
+            "INSERT INTO conversation_sessions "
+            "(session_id, msg_count, started_utc, last_activity_utc) "
+            + _CONV_SESSIONS_SELECT + " GROUP BY session_id"
+        )
+        return
+    ids = [s for s in session_ids if s is not None]
+    for i in range(0, len(ids), 400):
+        chunk = ids[i:i + 400]
+        placeholders = ",".join("?" for _ in chunk)
+        conn.execute(
+            f"DELETE FROM conversation_sessions WHERE session_id IN ({placeholders})",
+            chunk,
+        )
+        conn.execute(
+            "INSERT INTO conversation_sessions "
+            "(session_id, msg_count, started_utc, last_activity_utc) "
+            + _CONV_SESSIONS_SELECT
+            + f" AND session_id IN ({placeholders}) GROUP BY session_id",
+            chunk,
+        )
+
+
 def _consume_search_split(conn) -> None:
     """#177 S6: flock-held consumer for ``conversation_search_split_pending``
     (set by cache migration 010). Cursor-resumable: backfills

package/bin/_cctally_dashboard.py

@@ -251,6 +251,32 @@ from typing import Any
 from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
 
 
+class _QuietThreadingHTTPServer(ThreadingHTTPServer):
+    """`ThreadingHTTPServer` that swallows client-disconnect tracebacks.
+
+    A backgrounded/closed/reloaded dashboard tab hangs up mid-response, so the
+    per-request thread's socket write raises one of the "peer went away"
+    exceptions, which `socketserver` routes through `handle_error`. On a local
+    dashboard that is expected and benign — dumping a socket-write stack trace
+    to the user's console for every such disconnect is pure noise (the original
+    report was repeated `BrokenPipeError` spam after hours idle). Everything
+    else still gets the full traceback via `super().handle_error`.
+
+    `daemon_threads = True` is folded in here (was an inline `srv.*` assignment):
+    SSE handler threads may block up to 15s on the keep-alive timeout — let them
+    die with the process.
+    """
+
+    daemon_threads = True
+
+    def handle_error(self, request, client_address):
+        exc = sys.exc_info()[1]
+        if isinstance(exc, (BrokenPipeError, ConnectionResetError, ConnectionAbortedError)):
+            # Client hung up mid-response; benign on a local dashboard.
+            return
+        super().handle_error(request, client_address)
+
+
 def _cctally():
     """Resolve the current ``cctally`` module at call-time (spec §5.5)."""
     return sys.modules["cctally"]
@@ -8251,8 +8277,9 @@ def cmd_dashboard(args: argparse.Namespace) -> int:
     update_check_thread.start()
 
     # HTTP server on its own thread so the main thread can block on signal.
-    srv = ThreadingHTTPServer((args.host, args.port), DashboardHTTPHandler)
-    srv.daemon_threads = True  # SSE handler threads may block up to 15s on keep-alive timeout — let them die with the process.
+    # `_QuietThreadingHTTPServer` folds in `daemon_threads = True` and silences
+    # client-disconnect tracebacks (spec §5).
+    srv = _QuietThreadingHTTPServer((args.host, args.port), DashboardHTTPHandler)
 
     bind_host = args.host
     bind_port = srv.server_address[1]

package/bin/_cctally_db.py

@@ -2360,6 +2360,27 @@ def _apply_cache_schema(conn: sqlite3.Connection) -> None:
             byte_offset INTEGER NOT NULL
         );
 
+        -- Browse-rail rollup (conversation_sessions). Materializes exactly the
+        -- four structural aggregates the rail's old live GROUP BY produced
+        -- (COUNT/MIN/MAX over conversation_messages per session_id) so
+        -- GET /api/conversations no longer scans the whole message table to
+        -- render a 50-row page. The explicit NOT NULL on the non-INTEGER PK
+        -- matters (SQLite's legacy NULL-in-PK bug); the rail keys on a concrete
+        -- session_id and the recompute's GROUP BY already filters nulls. The
+        -- index lets the only paginated ordering (recent) early-terminate at
+        -- LIMIT with no temp B-tree. Re-derivable like the rest of cache.db;
+        -- sync_cache keeps it honest (scoped DELETE+INSERT re-derive +
+        -- flag-gated full recompute) — migration 013 arms the one-time
+        -- history backfill.
+        CREATE TABLE IF NOT EXISTS conversation_sessions (
+            session_id        TEXT NOT NULL PRIMARY KEY,
+            msg_count         INTEGER NOT NULL DEFAULT 0,
+            started_utc       TEXT,
+            last_activity_utc TEXT
+        );
+        CREATE INDEX IF NOT EXISTS idx_conv_sessions_recent
+            ON conversation_sessions(last_activity_utc DESC, session_id DESC);
+
         CREATE TABLE IF NOT EXISTS codex_session_files (
             path             TEXT PRIMARY KEY,
             size_bytes       INTEGER NOT NULL,
@@ -3407,6 +3428,20 @@ def _012_create_conversation_ai_titles(conn: sqlite3.Connection) -> None:
     conn.commit()
 
 
+@cache_migration("013_create_conversation_sessions")
+def _013_create_conversation_sessions(conn: sqlite3.Connection) -> None:
+    """Flag-only arm for the conversation_sessions browse-rail rollup. The table
+    is created by _apply_cache_schema (every open); this sets
+    conversation_sessions_backfill_pending so sync_cache does the one-time full
+    GROUP BY recompute under the cache.db.lock flock. No data work here — the
+    dispatcher's central stamp (#140) marks a complete handler; a fresh install
+    stamps WITHOUT running the handler, so the flag is NOT set there, which is
+    correct (empty messages -> empty rollup; the incremental DELETE+INSERT
+    re-derive fills both in lockstep). Mirrors 012."""
+    _set_cache_meta(conn, "conversation_sessions_backfill_pending", "1")
+    conn.commit()
+
+
 # === Region 7d: Stats migration 008_recompute_weekly_cost_snapshots_dedup_fix ===
 
 @stats_migration("008_recompute_weekly_cost_snapshots_dedup_fix")

package/bin/_lib_conversation_query.py

@@ -352,18 +352,73 @@ def _session_latest_meta_map(conn, session_ids):
     return meta
 
 
+# Rail sort keys, in the rollup table's STRUCTURAL columns (Task A). ``recent``
+# rides idx_conv_sessions_recent(last_activity_utc DESC, session_id DESC) and
+# early-terminates at LIMIT with no temp B-tree; ``oldest`` scan-sorts the few
+# thousand rollup rows (microseconds). The live fallback re-expresses the SAME
+# orderings over conversation_messages aggregates via _SORTS_LIVE below.
+# When adding a sort key, add it to BOTH _SORTS and _SORTS_LIVE and exercise it
+# in test_conversation_rail_rollup_reconcile.py (its _dump asserts the test's
+# sort list covers exactly _SORTS.keys(), so a one-sided add fails loud).
 _SORTS = {
+    "recent": "last_activity_utc DESC, session_id DESC",
+    "oldest": "started_utc ASC, session_id ASC",
+}
+
+# The matching ORDER BY for the live GROUP BY fallback: the rollup's
+# last_activity_utc IS MAX(timestamp_utc) and started_utc IS MIN(timestamp_utc),
+# so the two branches paginate in byte-identical order (the load-bearing
+# invariant). Keep this table in lockstep with _SORTS.
+_SORTS_LIVE = {
     "recent": "MAX(timestamp_utc) DESC, session_id DESC",
     "oldest": "MIN(timestamp_utc) ASC, session_id ASC",
 }
 
 
-def list_conversations(conn, *, sort="recent", limit=50, offset=0) -> dict:
-    """All-history per-session browse rows (spec §3.1). NOT 365-day bounded."""
-    order = _SORTS.get(sort, _SORTS["recent"])
-    limit = max(1, min(int(limit), 200))
-    offset = max(0, int(offset))
-    rows = conn.execute(
+def _rollup_authoritative(conn) -> bool:
+    """True when the conversation_sessions rollup is authoritative — i.e. the
+    durable ``conversation_sessions_backfill_pending`` flag is NOT set, so the
+    rollup has been fully recomputed and the fast read is safe.
+
+    Returns True (authoritative) when the flag is absent, including the case
+    where cache_meta does not exist at all (a path-less / schema-not-applied or
+    in-memory connection). This mirrors Task A's
+    _conversation_sessions_backfill_pending OperationalError degrade-to-False —
+    here that means "no pending flag" -> authoritative -> read the rollup. A
+    populated cache.db always has cache_meta, so this only affects bare conns."""
+    try:
+        pending = conn.execute(
+            "SELECT 1 FROM cache_meta "
+            "WHERE key='conversation_sessions_backfill_pending'"
+        ).fetchone() is not None
+    except sqlite3.OperationalError:
+        return True
+    return not pending
+
+
+def _list_session_rows_rollup(conn, order, limit, offset):
+    """FAST path: read the pre-aggregated rail rows straight from the
+    conversation_sessions rollup (spec §3). No GROUP BY, no temp B-tree for the
+    ``recent`` sort. Returns (session_id, msg_count, started, last_activity)
+    tuples — the same shape the live aggregate yields, so the downstream
+    assembly is identical. No WHERE is needed: the rollup is non-null by
+    construction (PK NOT NULL; the recompute's GROUP BY already drops NULLs)."""
+    return conn.execute(
+        "SELECT session_id, msg_count, "
+        "       started_utc AS started, last_activity_utc AS last_activity "
+        "FROM conversation_sessions "
+        "ORDER BY " + order + " LIMIT ? OFFSET ?",
+        (limit + 1, offset),
+    ).fetchall()
+
+
+def _list_session_rows_live(conn, order, limit, offset):
+    """RETAINED fallback (Codex gate BLOCKER 2): the original live GROUP BY over
+    conversation_messages, used while the rollup is not authoritative (the flag
+    is set — e.g. an existing install before its first sync, or permanently
+    under ``--no-sync``). Byte-identical output to the rollup branch by
+    construction; ``order`` here is the _SORTS_LIVE aggregate expression."""
+    return conn.execute(
         "SELECT session_id, COUNT(*) AS msg_count, "
         "       MIN(timestamp_utc) AS started, MAX(timestamp_utc) AS last_activity "
         "FROM conversation_messages "
@@ -372,6 +427,33 @@ def list_conversations(conn, *, sort="recent", limit=50, offset=0) -> dict:
         "ORDER BY " + order + " LIMIT ? OFFSET ?",
         (limit + 1, offset),
     ).fetchall()
+
+
+def list_conversations(conn, *, sort="recent", limit=50, offset=0) -> dict:
+    """All-history per-session browse rows (spec §3.1). NOT 365-day bounded.
+
+    Reads the conversation_sessions rollup (Task A) when it is authoritative —
+    i.e. the durable ``conversation_sessions_backfill_pending`` flag is clear —
+    which lets the ``recent`` sort early-terminate on idx_conv_sessions_recent
+    with no full-table aggregate. While the flag is set (an existing install
+    before its first sync consumes the backfill, or permanently under
+    ``--no-sync``), it falls back to the RETAINED live GROUP BY over
+    conversation_messages — correct, possibly slower, never an empty rail.
+
+    The two branches are byte-identical for every (sort, limit, offset): the
+    rollup recomputes the same COUNT/MIN/MAX over the same rows, and the
+    fallback IS the old aggregate (pinned by the reconcile test over both
+    branches). Staleness bound: at most one sync tick (~5s) if a session gains
+    messages on the exact tick the rail reads — cosmetic only, and the rail
+    revalidates every visible tick anyway."""
+    limit = max(1, min(int(limit), 200))
+    offset = max(0, int(offset))
+    if _rollup_authoritative(conn):
+        order = _SORTS.get(sort, _SORTS["recent"])
+        rows = _list_session_rows_rollup(conn, order, limit, offset)
+    else:
+        order = _SORTS_LIVE.get(sort, _SORTS_LIVE["recent"])
+        rows = _list_session_rows_live(conn, order, limit, offset)
     has_more = len(rows) > limit
     rows = rows[:limit]
     session_ids = [r[0] for r in rows]