cctally 1.37.1 → 1.37.2

package/CHANGELOG.md

@@ -5,6 +5,12 @@ based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 
 ## [Unreleased]
 
+## [1.37.2] - 2026-06-12
+
+### Fixed
+- Dashboard: the web dashboard no longer hangs on startup when you have a large conversation history. It now binds its HTTP port and serves the six cost/usage panels immediately, building the first heavy cache sync on a background thread and pushing the enriched conversation data over SSE once it completes — instead of blocking the bind behind a multi-minute (and, if interrupted, indefinite) first sync. `--no-sync` startup is byte-identical (#179).
+- Dashboard conversation reader: the one-time conversation-enrichment reingest (the #177 re-derivation that runs once over existing transcripts) is now resumable. It walks transcript files under a sorted-path cursor, re-enriching one file per atomic transaction, so a Ctrl-C or crash mid-reingest resumes where it left off on the next sync instead of restarting the full clear-and-backfill from scratch — which previously could hold the cache writer lock and hang. `cache-sync --rebuild` remains the clean one-shot recovery path (#179).
+
 ## [1.37.1] - 2026-06-11
 
 ### Fixed

package/bin/_cctally_cache.py

@@ -268,6 +268,9 @@ _CACHE_MIGRATIONS = _cctally_db_sib._CACHE_MIGRATIONS
 # drop/recreate dance so the per-row delete trigger never fires O(rows) under
 # the held lock on a rebuild / truncation escalation.
 clear_conversation_messages = _cctally_db_sib.clear_conversation_messages
+# cache_meta key/value upsert helper — reused by the resumable reingest cursor
+# writes (#179) so the ON CONFLICT idiom lives in one place. Caller commits.
+_set_cache_meta = _cctally_db_sib._set_cache_meta
 
 
 # === BEGIN MOVED REGIONS ===
@@ -640,7 +643,9 @@ def sync_cache(
                 "DELETE FROM cache_meta WHERE key IN "
                 "('conversation_reingest_pending',"
                 " 'conversation_source_tool_use_reingest_pending',"
-                " 'conversation_reingest_enrichment_pending')")
+                " 'conversation_reingest_enrichment_pending',"
+                " 'conversation_reingest_cursor',"
+                " 'conversation_reingest_cursor_gen')")
             conn.commit()
             eprint("[cache-sync] rebuild: cleared Claude cached entries")
 
@@ -720,15 +725,12 @@ def sync_cache(
             except sqlite3.OperationalError:
                 _reingest = False
             if _reingest:
-                clear_conversation_messages(conn)
-                backfill_conversation_messages(conn)
-                conn.execute(
-                    "DELETE FROM cache_meta WHERE key IN "
-                    "('conversation_reingest_pending',"
-                    " 'conversation_source_tool_use_reingest_pending',"
-                    " 'conversation_reingest_enrichment_pending')"
-                )
-                conn.commit()
+                # #179: resumable per-file reingest (was a global clear_conversation_messages
+                # + offset-0 backfill that re-armed the entire ~2.5min rebuild on any
+                # interrupt). The helper checkpoints a sorted-path cursor and clears the
+                # three flags + cursor + gen atomically on completion. Never on the rebuild
+                # path (which already wipes + repopulates id-aware via the normal walk).
+                _resumable_reingest_conversation_messages(conn)
 
         paths: list[pathlib.Path] = list(_iter_claude_jsonl_files())
         stats.files_total = len(paths)
@@ -1157,6 +1159,95 @@ def backfill_conversation_messages(conn: sqlite3.Connection) -> int:
     return inserted
 
 
+_REINGEST_FLAG_KEYS = (
+    "conversation_reingest_pending",
+    "conversation_source_tool_use_reingest_pending",
+    "conversation_reingest_enrichment_pending",
+)
+
+
+def _reingest_parse_file(jp, path_str):
+    """Parse one Claude JSONL into enriched ``conversation_messages`` row tuples
+    (``_CONV_INSERT_SQL`` column order). Mirrors ``backfill_conversation_messages``'s
+    inner read+flatten, factored to a module-level seam so the resumable reingest
+    builds rows BEFORE any write (a parse failure does no DML) and tests can inject.
+    Raises ``OSError`` if the file can't be opened/read."""
+    rows = []
+    with open(jp, "r", encoding="utf-8", errors="replace") as fh:
+        for m in _iter_message_rows(fh, path_str):
+            rows.append(_conv_row_tuple(m, path_str))
+    return rows
+
+
+def _resumable_reingest_conversation_messages(conn):
+    """#179: resumable, lock-friendly replacement for the old global
+    ``clear_conversation_messages`` + offset-0 ``backfill_conversation_messages``
+    reingest, which re-armed the whole ~2.5min rebuild on any interrupt. Walks
+    every Claude JSONL in deterministic sorted-path order, re-enriching one file
+    per atomic transaction and checkpointing ``conversation_reingest_cursor`` so an
+    interrupt resumes instead of restarting. A ``conversation_reingest_cursor_gen``
+    fingerprint (the set of pending reingest flags) resets the cursor whenever the
+    pending-flag set changes, so a newly-armed flag forces a fresh pass. The caller
+    (``sync_cache``) already holds the cache.db flock; per-file commits bound only
+    the SQLite write transaction, not the flock. Clears the three flags + cursor +
+    gen atomically on completion."""
+    # 1. Generation guard: reset the cursor if the live pending-flag set differs.
+    set_flags = [k for k in _REINGEST_FLAG_KEYS
+                 if conn.execute("SELECT 1 FROM cache_meta WHERE key=?", (k,)).fetchone()]
+    gen = ",".join(sorted(set_flags))
+    gen_row = conn.execute(
+        "SELECT value FROM cache_meta WHERE key='conversation_reingest_cursor_gen'"
+    ).fetchone()
+    if (gen_row[0] if gen_row else None) != gen:
+        _set_cache_meta(conn, "conversation_reingest_cursor_gen", gen)
+        conn.execute("DELETE FROM cache_meta WHERE key='conversation_reingest_cursor'")
+        conn.commit()
+        cursor = ""
+    else:
+        crow = conn.execute(
+            "SELECT value FROM cache_meta WHERE key='conversation_reingest_cursor'"
+        ).fetchone()
+        cursor = crow[0] if crow and crow[0] is not None else ""
+
+    # 2. Per-file resumable walk in deterministic sorted-path order.
+    for jp in sorted(_iter_claude_jsonl_files(), key=str):
+        path_str = str(jp)
+        if path_str <= cursor:
+            continue
+        try:
+            rows = _reingest_parse_file(jp, path_str)   # parse FIRST — no DML on failure
+        except OSError as exc:
+            # Read/parse failed BEFORE any conversation_messages DML — the file's
+            # existing rows are untouched (preserved, not dropped). Only advance the
+            # cursor; this cursor-only write needs no rollback envelope (no message
+            # DML to undo, and an interrupt mid-commit just re-runs this file).
+            eprint(f"[conversation-reingest] could not read {jp}: {exc}; "
+                   "preserving existing rows")
+            _set_cache_meta(conn, "conversation_reingest_cursor", path_str)
+            conn.commit()
+            continue
+        try:
+            conn.execute("DELETE FROM conversation_messages WHERE source_path=?",
+                         (path_str,))
+            if rows:
+                conn.executemany(_CONV_INSERT_SQL, rows)
+            _set_cache_meta(conn, "conversation_reingest_cursor", path_str)
+            conn.commit()
+        except BaseException:
+            conn.rollback()
+            raise
+
+    # 3. Completion: clear flags + cursor + gen atomically.
+    conn.execute(
+        "DELETE FROM cache_meta WHERE key IN "
+        "('conversation_reingest_pending',"
+        " 'conversation_source_tool_use_reingest_pending',"
+        " 'conversation_reingest_enrichment_pending',"
+        " 'conversation_reingest_cursor',"
+        " 'conversation_reingest_cursor_gen')")
+    conn.commit()
+
+
 def iter_entries(
     conn: sqlite3.Connection,
     range_start: dt.datetime,

package/bin/_cctally_dashboard.py

@@ -7829,6 +7829,21 @@ def _dashboard_wait_for_signal(
         os.close(write_fd)
 
 
+def _dashboard_initial_snapshot(args, *, pinned_now, display_tz_pref_override):
+    """#179: build the dashboard's first snapshot WITHOUT the heavy sync so the
+    HTTP port binds promptly. The background ``_DashboardSyncThread`` (started in
+    cmd_dashboard before the ThreadingHTTPServer bind) runs the first full
+    ``sync_cache`` — including any pending conversation reingest — and SSE-pushes
+    the populated snapshot on completion. ``--no-sync`` already passed
+    ``skip_sync=True`` here, so that path is byte-identical; only the normal launch
+    changes (its previously-redundant foreground sync is removed, not duplicated —
+    the background thread's first tick was always going to sync anyway)."""
+    return sys.modules["cctally"]._tui_build_snapshot(
+        now_utc=pinned_now, skip_sync=True,
+        display_tz_pref_override=display_tz_pref_override,
+    )
+
+
 def cmd_dashboard(args: argparse.Namespace) -> int:
     """Launch the live web dashboard."""
     import signal as _signal
@@ -7906,9 +7921,13 @@ def cmd_dashboard(args: argparse.Namespace) -> int:
     # weekly, project, forecast subcommands already respect.
     pinned_now = _command_as_of() if os.environ.get("CCTALLY_AS_OF") else None
 
-    # Build initial snapshot — blocking, serves immediately.
-    initial = sys.modules["cctally"]._tui_build_snapshot(
-        now_utc=pinned_now, skip_sync=args.no_sync,
+    # #179: build the initial snapshot WITHOUT the heavy sync so the HTTP port
+    # binds promptly even on a heavy-history instance. The background
+    # _DashboardSyncThread (started below, before the bind) owns the first full
+    # sync_cache — including any pending conversation-enrichment reingest — and
+    # SSE-pushes the populated snapshot on completion.
+    initial = _dashboard_initial_snapshot(
+        args, pinned_now=pinned_now,
         display_tz_pref_override=display_tz_pref_override,
     )
     if args.no_sync:

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cctally",
-  "version": "1.37.1",
+  "version": "1.37.2",
   "description": "Claude Code usage tracker and local dashboard for Pro/Max subscription limits - weekly cost-per-percent trend, quota forecasts, threshold alerts. ccusage-compatible.",
   "homepage": "https://github.com/omrikais/cctally",
   "repository": {