cctally 1.37.0 → 1.37.2

package/CHANGELOG.md

@@ -5,6 +5,17 @@ based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 
 ## [Unreleased]
 
+## [1.37.2] - 2026-06-12
+
+### Fixed
+- Dashboard: the web dashboard no longer hangs on startup when you have a large conversation history. It now binds its HTTP port and serves the six cost/usage panels immediately, building the first heavy cache sync on a background thread and pushing the enriched conversation data over SSE once it completes — instead of blocking the bind behind a multi-minute (and, if interrupted, indefinite) first sync. `--no-sync` startup is byte-identical (#179).
+- Dashboard conversation reader: the one-time conversation-enrichment reingest (the #177 re-derivation that runs once over existing transcripts) is now resumable. It walks transcript files under a sorted-path cursor, re-enriching one file per atomic transaction, so a Ctrl-C or crash mid-reingest resumes where it left off on the next sync instead of restarting the full clear-and-backfill from scratch — which previously could hold the cache writer lock and hang. `cache-sync --rebuild` remains the clean one-shot recovery path (#179).
+
+## [1.37.1] - 2026-06-11
+
+### Fixed
+- Conversation viewer: the `TaskCreate`/`TaskUpdate`/`TaskList` checklist card rendered an empty "0 / 0" for tasks created inside subagents. Subagent Task tools record their result as a plain string (`Task #N created successfully: …`) rather than the structured shape main-session tools use, so the reader couldn't recover the task ids; it now parses both shapes. The running checklist is also reconstructed per-subagent, so parallel subagents no longer bleed their tasks into one another's cards, and a Task run whose results can't be parsed now falls back to plain tool chips instead of a misleading empty card. Existing transcripts pick this up on the next `cache-sync --rebuild`.
+
 ## [1.37.0] - 2026-06-11
 
 ### Added

package/bin/_cctally_cache.py

@@ -268,6 +268,9 @@ _CACHE_MIGRATIONS = _cctally_db_sib._CACHE_MIGRATIONS
 # drop/recreate dance so the per-row delete trigger never fires O(rows) under
 # the held lock on a rebuild / truncation escalation.
 clear_conversation_messages = _cctally_db_sib.clear_conversation_messages
+# cache_meta key/value upsert helper — reused by the resumable reingest cursor
+# writes (#179) so the ON CONFLICT idiom lives in one place. Caller commits.
+_set_cache_meta = _cctally_db_sib._set_cache_meta
 
 
 # === BEGIN MOVED REGIONS ===
@@ -640,7 +643,9 @@ def sync_cache(
                 "DELETE FROM cache_meta WHERE key IN "
                 "('conversation_reingest_pending',"
                 " 'conversation_source_tool_use_reingest_pending',"
-                " 'conversation_reingest_enrichment_pending')")
+                " 'conversation_reingest_enrichment_pending',"
+                " 'conversation_reingest_cursor',"
+                " 'conversation_reingest_cursor_gen')")
             conn.commit()
             eprint("[cache-sync] rebuild: cleared Claude cached entries")
 
@@ -720,15 +725,12 @@ def sync_cache(
             except sqlite3.OperationalError:
                 _reingest = False
             if _reingest:
-                clear_conversation_messages(conn)
-                backfill_conversation_messages(conn)
-                conn.execute(
-                    "DELETE FROM cache_meta WHERE key IN "
-                    "('conversation_reingest_pending',"
-                    " 'conversation_source_tool_use_reingest_pending',"
-                    " 'conversation_reingest_enrichment_pending')"
-                )
-                conn.commit()
+                # #179: resumable per-file reingest (was a global clear_conversation_messages
+                # + offset-0 backfill that re-armed the entire ~2.5min rebuild on any
+                # interrupt). The helper checkpoints a sorted-path cursor and clears the
+                # three flags + cursor + gen atomically on completion. Never on the rebuild
+                # path (which already wipes + repopulates id-aware via the normal walk).
+                _resumable_reingest_conversation_messages(conn)
 
         paths: list[pathlib.Path] = list(_iter_claude_jsonl_files())
         stats.files_total = len(paths)
@@ -1157,6 +1159,95 @@ def backfill_conversation_messages(conn: sqlite3.Connection) -> int:
     return inserted
 
 
+_REINGEST_FLAG_KEYS = (
+    "conversation_reingest_pending",
+    "conversation_source_tool_use_reingest_pending",
+    "conversation_reingest_enrichment_pending",
+)
+
+
+def _reingest_parse_file(jp, path_str):
+    """Parse one Claude JSONL into enriched ``conversation_messages`` row tuples
+    (``_CONV_INSERT_SQL`` column order). Mirrors ``backfill_conversation_messages``'s
+    inner read+flatten, factored to a module-level seam so the resumable reingest
+    builds rows BEFORE any write (a parse failure does no DML) and tests can inject.
+    Raises ``OSError`` if the file can't be opened/read."""
+    rows = []
+    with open(jp, "r", encoding="utf-8", errors="replace") as fh:
+        for m in _iter_message_rows(fh, path_str):
+            rows.append(_conv_row_tuple(m, path_str))
+    return rows
+
+
+def _resumable_reingest_conversation_messages(conn):
+    """#179: resumable, lock-friendly replacement for the old global
+    ``clear_conversation_messages`` + offset-0 ``backfill_conversation_messages``
+    reingest, which re-armed the whole ~2.5min rebuild on any interrupt. Walks
+    every Claude JSONL in deterministic sorted-path order, re-enriching one file
+    per atomic transaction and checkpointing ``conversation_reingest_cursor`` so an
+    interrupt resumes instead of restarting. A ``conversation_reingest_cursor_gen``
+    fingerprint (the set of pending reingest flags) resets the cursor whenever the
+    pending-flag set changes, so a newly-armed flag forces a fresh pass. The caller
+    (``sync_cache``) already holds the cache.db flock; per-file commits bound only
+    the SQLite write transaction, not the flock. Clears the three flags + cursor +
+    gen atomically on completion."""
+    # 1. Generation guard: reset the cursor if the live pending-flag set differs.
+    set_flags = [k for k in _REINGEST_FLAG_KEYS
+                 if conn.execute("SELECT 1 FROM cache_meta WHERE key=?", (k,)).fetchone()]
+    gen = ",".join(sorted(set_flags))
+    gen_row = conn.execute(
+        "SELECT value FROM cache_meta WHERE key='conversation_reingest_cursor_gen'"
+    ).fetchone()
+    if (gen_row[0] if gen_row else None) != gen:
+        _set_cache_meta(conn, "conversation_reingest_cursor_gen", gen)
+        conn.execute("DELETE FROM cache_meta WHERE key='conversation_reingest_cursor'")
+        conn.commit()
+        cursor = ""
+    else:
+        crow = conn.execute(
+            "SELECT value FROM cache_meta WHERE key='conversation_reingest_cursor'"
+        ).fetchone()
+        cursor = crow[0] if crow and crow[0] is not None else ""
+
+    # 2. Per-file resumable walk in deterministic sorted-path order.
+    for jp in sorted(_iter_claude_jsonl_files(), key=str):
+        path_str = str(jp)
+        if path_str <= cursor:
+            continue
+        try:
+            rows = _reingest_parse_file(jp, path_str)   # parse FIRST — no DML on failure
+        except OSError as exc:
+            # Read/parse failed BEFORE any conversation_messages DML — the file's
+            # existing rows are untouched (preserved, not dropped). Only advance the
+            # cursor; this cursor-only write needs no rollback envelope (no message
+            # DML to undo, and an interrupt mid-commit just re-runs this file).
+            eprint(f"[conversation-reingest] could not read {jp}: {exc}; "
+                   "preserving existing rows")
+            _set_cache_meta(conn, "conversation_reingest_cursor", path_str)
+            conn.commit()
+            continue
+        try:
+            conn.execute("DELETE FROM conversation_messages WHERE source_path=?",
+                         (path_str,))
+            if rows:
+                conn.executemany(_CONV_INSERT_SQL, rows)
+            _set_cache_meta(conn, "conversation_reingest_cursor", path_str)
+            conn.commit()
+        except BaseException:
+            conn.rollback()
+            raise
+
+    # 3. Completion: clear flags + cursor + gen atomically.
+    conn.execute(
+        "DELETE FROM cache_meta WHERE key IN "
+        "('conversation_reingest_pending',"
+        " 'conversation_source_tool_use_reingest_pending',"
+        " 'conversation_reingest_enrichment_pending',"
+        " 'conversation_reingest_cursor',"
+        " 'conversation_reingest_cursor_gen')")
+    conn.commit()
+
+
 def iter_entries(
     conn: sqlite3.Connection,
     range_start: dt.datetime,

package/bin/_cctally_dashboard.py

@@ -7829,6 +7829,21 @@ def _dashboard_wait_for_signal(
         os.close(write_fd)
 
 
+def _dashboard_initial_snapshot(args, *, pinned_now, display_tz_pref_override):
+    """#179: build the dashboard's first snapshot WITHOUT the heavy sync so the
+    HTTP port binds promptly. The background ``_DashboardSyncThread`` (started in
+    cmd_dashboard before the ThreadingHTTPServer bind) runs the first full
+    ``sync_cache`` — including any pending conversation reingest — and SSE-pushes
+    the populated snapshot on completion. ``--no-sync`` already passed
+    ``skip_sync=True`` here, so that path is byte-identical; only the normal launch
+    changes (its previously-redundant foreground sync is removed, not duplicated —
+    the background thread's first tick was always going to sync anyway)."""
+    return sys.modules["cctally"]._tui_build_snapshot(
+        now_utc=pinned_now, skip_sync=True,
+        display_tz_pref_override=display_tz_pref_override,
+    )
+
+
 def cmd_dashboard(args: argparse.Namespace) -> int:
     """Launch the live web dashboard."""
     import signal as _signal
@@ -7906,9 +7921,13 @@ def cmd_dashboard(args: argparse.Namespace) -> int:
     # weekly, project, forecast subcommands already respect.
     pinned_now = _command_as_of() if os.environ.get("CCTALLY_AS_OF") else None
 
-    # Build initial snapshot — blocking, serves immediately.
-    initial = sys.modules["cctally"]._tui_build_snapshot(
-        now_utc=pinned_now, skip_sync=args.no_sync,
+    # #179: build the initial snapshot WITHOUT the heavy sync so the HTTP port
+    # binds promptly even on a heavy-history instance. The background
+    # _DashboardSyncThread (started below, before the bind) owns the first full
+    # sync_cache — including any pending conversation-enrichment reingest — and
+    # SSE-pushes the populated snapshot on completion.
+    initial = _dashboard_initial_snapshot(
+        args, pinned_now=pinned_now,
         display_tz_pref_override=display_tz_pref_override,
     )
     if args.no_sync:

package/bin/_lib_conversation.py

@@ -8,6 +8,7 @@ mid-write tail line. Spec §1, §2.
 """
 from __future__ import annotations
 import json
+import re
 from dataclasses import dataclass
 
 HUMAN = "human"
@@ -288,45 +289,68 @@ def _attach_ask_answers(blocks, obj):
         results[0]["ask_annotations"] = bounded_anno
 
 
+# Subagent Task tools record toolUseResult=null and put the identity in the
+# human-readable result text instead; the id is the only thing the fold needs
+# from the result (subject/status come from the call input). Anchored to the
+# line start so unrelated output mentioning a task id mid-sentence never matches.
+# Shapes verified against real subagent transcripts (Claude Code 2.1.173).
+_TASK_CREATE_RESULT_RE = re.compile(r"^Task #(\d+) created\b")
+_TASK_UPDATE_RESULT_RE = re.compile(r"^Updated task #(\d+)\b")
+
+
 def _attach_task_meta(blocks, obj):
     """Stash a Task* tool's record-level identity onto its single tool_result
-    block so the query-kernel fold has a robust id (NOT a result-string parse).
-    Task ids are monotonic + never reused, so the explicit id is the only stable
-    fold key. Self-identifying by toolUseResult shape:
+    block so the query-kernel fold has a robust id. Task ids are monotonic +
+    never reused, so the explicit id is the only stable fold key. Two result
+    shapes, both self-identifying off the single tool_result block:
+
+    Structured (MAIN-session Task tools) — toolUseResult carries the identity:
       TaskCreate -> {"task": {"id": ...}}            -> block["task_id"]
       TaskUpdate -> {"taskId": ...}                   -> block["task_id"]
       TaskList   -> {"tasks": [{id,subject,status}]}  -> block["task_list"]
+
+    String-content (SUBAGENT Task tools) — toolUseResult is null and the id
+    lives in the result text ("Task #7 created successfully: ..." / "Updated
+    task #3 status"); we recover the id from block["text"]. Subagent-driven
+    workflows make this the dominant shape, so missing it left every subagent
+    Task run rendering as an empty "0 / 0" card.
+
     Same exactly-one-result-block guard as _attach_subagent_result. Subjects
     bounded through _bound_input.
 
     The ``task.id`` (not ``task.task_id``) gate deliberately ignores the
     look-alike local_bash spawn result {"task": {"task_id": ..., ...}}, which is
     a different tool family and carries no checklist id."""
-    tur = obj.get("toolUseResult")
-    if not isinstance(tur, dict):
-        return
     results = [b for b in blocks if b.get("kind") == "tool_result"]
     if len(results) != 1:
         return
     block = results[0]
-    task = tur.get("task")
-    if isinstance(task, dict) and task.get("id") is not None:
-        block["task_id"] = str(task["id"])
-        return
-    if tur.get("taskId") is not None:
-        block["task_id"] = str(tur["taskId"])
-        return
-    tasks = tur.get("tasks")
-    if isinstance(tasks, list):
-        snap = []
-        for t in tasks:
-            if not isinstance(t, dict) or t.get("id") is None:
-                continue
-            bounded, _ = _bound_input({"subject": t.get("subject") or ""})
-            snap.append({"id": str(t["id"]),
-                         "subject": bounded.get("subject", ""),
-                         "status": t.get("status") or "pending"})
-        block["task_list"] = snap
+    tur = obj.get("toolUseResult")
+    if isinstance(tur, dict):
+        task = tur.get("task")
+        if isinstance(task, dict) and task.get("id") is not None:
+            block["task_id"] = str(task["id"])
+            return
+        if tur.get("taskId") is not None:
+            block["task_id"] = str(tur["taskId"])
+            return
+        tasks = tur.get("tasks")
+        if isinstance(tasks, list):
+            snap = []
+            for t in tasks:
+                if not isinstance(t, dict) or t.get("id") is None:
+                    continue
+                bounded, _ = _bound_input({"subject": t.get("subject") or ""})
+                snap.append({"id": str(t["id"]),
+                             "subject": bounded.get("subject", ""),
+                             "status": t.get("status") or "pending"})
+            block["task_list"] = snap
+            return
+    # String-content fallback (subagent Task tools): no structured identity.
+    m = (_TASK_CREATE_RESULT_RE.match(block.get("text") or "")
+         or _TASK_UPDATE_RESULT_RE.match(block.get("text") or ""))
+    if m:
+        block["task_id"] = m.group(1)
 
 
 def _stringify(c):

package/bin/_lib_conversation_query.py

@@ -711,24 +711,37 @@ _TASK_TRIO = ("TaskCreate", "TaskUpdate", "TaskList")
 def _fold_task_runs(items, task_link):
     """Reconstruct the running to-do list from the chronological Task* op stream
     and stamp the resulting todos[] snapshot onto the FIRST tool_call of each
-    Task* run. State spans the whole session; key on the explicit task id (never
-    reused). `deleted` drops a task; a TaskList result reseeds the whole snapshot.
+    Task* run. Key on the explicit task id (never reused). `deleted` drops a
+    task; a TaskList result reseeds the whole snapshot.
+
+    Scoped PER subagent thread (``subagent_key``): the main session (key None)
+    and each subagent keep INDEPENDENT running checklists, so parallel subagents
+    with disjoint task-id ranges never bleed into one another's cards. Within a
+    thread, state still spans the whole session.
+
+    Degradation guard: a thread's run is stamped only once that thread has
+    recognized a real create/list (``seen``). A Task* run with no recognizable
+    create — a future result shape we don't parse, or pre-fix legacy rows —
+    leaves ``task_snapshot`` ABSENT, so the frontend falls back to generic chips
+    instead of a misleading empty "0 / 0" card.
 
     Mirrors the ask_answers join: the parser stashed the record-level identity
     onto the tool_result block, the Phase-1 sweep popped it into ``task_link``
     keyed by tool_use_id, and this fold joins it back. The frontend stays a pure
     todos[] renderer — all running-list state lives here."""
-    order = []
-    state = {}
+    threads = {}      # subagent_key -> {"order": [...], "state": {...}, "seen": bool}
 
-    def snapshot():
-        return [dict(content=state[i]["content"], status=state[i]["status"],
-                     **({"activeForm": state[i]["activeForm"]} if state[i].get("activeForm") else {}))
-                for i in order if i in state]
+    def snapshot(th):
+        st, order = th["state"], th["order"]
+        return [dict(content=st[i]["content"], status=st[i]["status"],
+                     **({"activeForm": st[i]["activeForm"]} if st[i].get("activeForm") else {}))
+                for i in order if i in st]
 
     for it in items:
         if it.get("kind") != "assistant":
             continue
+        th = threads.setdefault(it.get("subagent_key"),
+                                {"order": [], "state": {}, "seen": False})
         first_task_call = None
         for b in it["blocks"]:
             if b.get("kind") != "tool_call" or b.get("name") not in _TASK_TRIO:
@@ -741,30 +754,32 @@ def _fold_task_runs(items, task_link):
             if name == "TaskCreate":
                 tid = link.get("task_id")
                 if tid is not None:
-                    if tid not in state:
-                        order.append(tid)
-                    state[tid] = {"content": inp.get("subject") or "", "status": "pending",
-                                  "activeForm": inp.get("activeForm") or ""}
+                    th["seen"] = True
+                    if tid not in th["state"]:
+                        th["order"].append(tid)
+                    th["state"][tid] = {"content": inp.get("subject") or "", "status": "pending",
+                                        "activeForm": inp.get("activeForm") or ""}
             elif name == "TaskUpdate":
                 tid = str(inp.get("taskId")) if inp.get("taskId") is not None else link.get("task_id")
                 status = inp.get("status")
                 if tid is not None:
                     if status == "deleted":
-                        state.pop(tid, None)
-                    elif tid in state and status:
-                        state[tid]["status"] = status
+                        th["state"].pop(tid, None)
+                    elif tid in th["state"] and status:
+                        th["state"][tid]["status"] = status
             elif name == "TaskList":
                 snap = link.get("task_list")
                 if snap is not None:
-                    order = []
-                    state = {}
+                    th["seen"] = True
+                    th["order"] = []
+                    th["state"] = {}
                     for t in snap:
                         tid = t["id"]
-                        order.append(tid)
-                        state[tid] = {"content": t.get("subject") or "",
-                                      "status": t.get("status") or "pending", "activeForm": ""}
-        if first_task_call is not None:
-            first_task_call["task_snapshot"] = snapshot()
+                        th["order"].append(tid)
+                        th["state"][tid] = {"content": t.get("subject") or "",
+                                            "status": t.get("status") or "pending", "activeForm": ""}
+        if first_task_call is not None and th["seen"]:
+            first_task_call["task_snapshot"] = snapshot(th)
 
 
 def _latest(logical, col):

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cctally",
-  "version": "1.37.0",
+  "version": "1.37.2",
   "description": "Claude Code usage tracker and local dashboard for Pro/Max subscription limits - weekly cost-per-percent trend, quota forecasts, threshold alerts. ccusage-compatible.",
   "homepage": "https://github.com/omrikais/cctally",
   "repository": {