cctally 1.29.0 → 1.30.0

package/CHANGELOG.md

@@ -5,6 +5,18 @@ based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 
 ## [Unreleased]
 
+## [1.30.0] - 2026-06-09
+
+### Added
+- **Keyboard navigation in the dashboard conversation reader.** With a conversation open, `j` / `k` move a focused-turn cursor between turns (scrolling each into view and auto-loading the next page when you reach the end), `[` / `]` collapse-all / expand-all the disclosure sections in the thread, and `g` jumps back to the top. The bindings are listed in the dashboard help overlay (`?`) under a new "Conversations" group and are inert while a modal or the rail search/filter input is active.
+- **Syntax-highlighted code blocks and copy buttons in the conversation reader.** Fenced code in transcripts is now syntax-highlighted with a language label, and one-click copy buttons appear on code blocks, tool output, and message text.
+- **Derived conversation titles.** Each conversation now shows a short derived title in the sidebar rail, the reader header, and full-text search results — and cross-session search now matches on that title too; the rail also groups conversations under date dividers.
+- Jump-to-message now expands the owning collapsed subagent thread when the target message lands inside one, instead of silently scrolling to a hidden turn (#160).
+
+### Changed
+- **The dashboard conversation viewer (introduced in 1.29.0) has been redesigned end-to-end.** Transcripts now render as serif prose (self-hosted Newsreader) on a readable ~68-character measure with higher-contrast text, laid out along a timeline spine with role-differentiated turns; assistant turns are walked in document order so each tool call renders paired with its result as an inline tool-I/O chip (chevron + preview) and stray orphan tool-result runs are collapsed; subagent sidechains render as weighted thread cards; system-command messages fold into an expandable pill; and inline-SVG icons replace the previous emoji throughout, with disclosure sections opening on a smooth animation and turns staggering in on first load behind a refined jump-flash (#161, #162, #164, #165, #168).
+- Internal (no action needed on upgrade): a new cache migration (`003`) re-ingests existing conversation history id-aware so older transcripts pick up the tool-call linkage ids the paired tool-I/O view needs. The cache is fully re-derivable and the migration runs automatically on the next DB open.
+
 ## [1.29.0] - 2026-06-08
 
 ### Added

package/bin/_cctally_cache.py

@@ -612,6 +612,12 @@ def sync_cache(
             # run a redundant (idempotent but wasteful) offset-0 backfill pass.
             conn.execute(
                 "DELETE FROM cache_meta WHERE key='conversation_backfill_pending'")
+            # Issue #164: a rebuild also clears + repopulates the message index
+            # id-aware via the normal offset-0 walk, so drop the 003 reingest
+            # flag too — the post-rebuild sync must not run a redundant
+            # (idempotent but wasteful) clear+backfill pass.
+            conn.execute(
+                "DELETE FROM cache_meta WHERE key='conversation_reingest_pending'")
             conn.commit()
             eprint("[cache-sync] rebuild: cleared Claude cached entries")
 
@@ -647,6 +653,36 @@ def sync_cache(
                 )
                 conn.commit()
 
+            # Issue #164: consume the deferred conversation_messages re-ingest.
+            # Cache migration 003 is flag-only — it sets
+            # ``conversation_reingest_pending`` rather than clearing inline
+            # (clearing in the handler would run WITHOUT this flock, racing a
+            # concurrent sync, and would empty the reader on stats-only /
+            # eager-migration opens or ``dashboard --no-sync``). The destructive
+            # clear + id-aware offset-0 re-derive live here, UNDER the held
+            # flock. Distinct from 002's backfill-without-clear: 003 is
+            # clear-then-backfill, re-deriving the WHOLE index id-aware so
+            # existing history pairs tool_use<->tool_result. The clear is
+            # storm-free (#138); the offset-0 backfill walks every JSONL from 0;
+            # the flag is dropped LAST so a crash mid-walk re-runs cleanly on the
+            # next sync. Never on the rebuild path (which already wipes +
+            # repopulates the index id-aware via the normal walk).
+            try:
+                _reingest = conn.execute(
+                    "SELECT 1 FROM cache_meta "
+                    "WHERE key='conversation_reingest_pending'"
+                ).fetchone() is not None
+            except sqlite3.OperationalError:
+                _reingest = False
+            if _reingest:
+                clear_conversation_messages(conn)
+                backfill_conversation_messages(conn)
+                conn.execute(
+                    "DELETE FROM cache_meta "
+                    "WHERE key='conversation_reingest_pending'"
+                )
+                conn.commit()
+
         paths: list[pathlib.Path] = list(_iter_claude_jsonl_files())
         stats.files_total = len(paths)
 

package/bin/_cctally_db.py

@@ -2991,6 +2991,24 @@ def _002_conversation_messages_backfill(conn: sqlite3.Connection) -> None:
     conn.commit()
 
 
+@cache_migration("003_conversation_reingest_tool_ids")
+def _003_conversation_reingest_tool_ids(conn: sqlite3.Connection) -> None:
+    """Flag-only re-ingest of conversation_messages so tool_use.id /
+    tool_result.tool_use_id / preview land on existing history (#164).
+
+    The destructive clear + offset-0 backfill run in sync_cache UNDER the
+    cache.db.lock flock — NOT here. Clearing in the handler would violate the
+    lock discipline cache-001 follows and would empty the reader on
+    stats-only / eager-migration opens or ``dashboard --no-sync``. A distinct
+    flag from 002's conversation_backfill_pending: 002 = backfill-without-clear;
+    003 = clear-then-backfill. The dispatcher stamps this migration centrally
+    on the existing-install path (issue #140); a fresh install stamps it
+    without running (empty table), and the flag — if ever set — is a harmless
+    no-op there."""
+    _set_cache_meta(conn, "conversation_reingest_pending", "1")
+    conn.commit()
+
+
 # === Region 7d: Stats migration 008_recompute_weekly_cost_snapshots_dedup_fix ===
 
 @stats_migration("008_recompute_weekly_cost_snapshots_dedup_fix")

package/bin/_lib_conversation.py

@@ -136,12 +136,15 @@ def _blocks_and_text(content):
                 blocks.append({"kind": "thinking", "text": b.get("thinking", "") or ""})
             elif bt == "tool_use":
                 blocks.append({"kind": "tool_use", "name": b.get("name"),
-                               "input_summary": _summarize(b.get("input"))})
+                               "input_summary": _summarize(b.get("input")),
+                               "id": b.get("id"),
+                               "preview": tool_preview(b.get("name"), b.get("input"))})
             elif bt == "tool_result":
                 raw = _stringify(b.get("content"))
                 blocks.append({"kind": "tool_result", "text": raw[:_TOOL_RESULT_CAP],
                                "truncated": len(raw) > _TOOL_RESULT_CAP,
-                               "is_error": bool(b.get("is_error"))})
+                               "is_error": bool(b.get("is_error")),
+                               "tool_use_id": b.get("tool_use_id")})
             elif bt in ("image", "document"):
                 blocks.append({"kind": bt, **_media(b.get("source"))})
             elif bt == "tool_reference":
@@ -170,6 +173,39 @@ def _summarize(inp):
     return s[:200]
 
 
+_PREVIEW_FIELDS = {
+    "Read": "file_path", "Write": "file_path", "Edit": "file_path",
+    "MultiEdit": "file_path", "NotebookEdit": "file_path",
+    "Bash": "command", "Grep": "pattern", "Glob": "pattern",
+    "Task": "description", "WebFetch": "url", "WebSearch": "query",
+}
+
+
+def tool_preview(name, inp):
+    """One-line, full-fidelity preview for a tool call's collapsed chip (#164,
+    C5). Runs on the RAW input dict before _summarize truncates to 200 chars.
+    Known tools map to their primary arg; Bash takes the first command line;
+    Task falls back to subagent_type; unknown/mcp tools take the first
+    string-valued arg, else the tool name. Always returns a single-line str."""
+    if not isinstance(inp, dict):
+        return ""
+    field = _PREVIEW_FIELDS.get(name or "")
+    val = None
+    if field is not None:
+        val = inp.get(field)
+        if val is None and name == "Task":
+            val = inp.get("subagent_type")
+    if val is None:
+        # generic fallback: first string-valued arg, else the tool name
+        for v in inp.values():
+            if isinstance(v, str) and v:
+                val = v
+                break
+    if not isinstance(val, str) or not val:
+        return name or ""
+    return val.splitlines()[0]
+
+
 def _media(source):
     if not isinstance(source, dict):
         return {"media_type": None, "bytes": 0}

package/bin/_lib_conversation_query.py

@@ -13,6 +13,7 @@ deduped session_entries row (idx_entries_dedup), via the shared pricing helper
 from __future__ import annotations
 import json as _json
 import os
+import re
 import sqlite3
 
 # Public surface (Plan 2): shipped in the npm tarball + brew formula + public
@@ -21,6 +22,75 @@ import sqlite3
 from _lib_pricing import _calculate_entry_cost
 
 
+# Mirror of dashboard/web/src/conversations/systemMarkers.ts::MARKER_RE — anchored
+# whole-string (fullmatch), unrolled-lazy body for linear time (no ReDoS), \1
+# backref forces each close tag to match its open tag. Used to SKIP slash-command
+# plumbing when deriving a conversation title (#165 Q2). MUST stay equivalent to
+# the TS predicate over ASCII whitespace (parity-tested); exotic Unicode/control
+# whitespace is an explicit non-goal. See docs/dashboard-gotchas.md.
+_MARKER_TAGS = ("command-name", "command-message", "command-args", "local-command-caveat")
+_MARKER_RE = re.compile(
+    r"\s*(?:<(" + "|".join(_MARKER_TAGS) + r")>(?:(?!</\1>)[\s\S])*</\1>\s*)+"
+)
+
+
+def _is_system_marker(text) -> bool:
+    """True iff `text` is ONLY concatenated command-marker wrappers (slash-command
+    plumbing) — the title-derivation skip predicate. `fullmatch` reproduces the TS
+    `^\\s*…\\s*$` anchor (no `$`-before-trailing-`\\n` foot-gun)."""
+    return bool(text) and _MARKER_RE.fullmatch(text) is not None
+
+
+_TITLE_MAX = 120
+
+
+def _title_from_text(text) -> str:
+    """First non-blank LINE of `text`, trimmed, sliced to _TITLE_MAX with a
+    trailing '…' ONLY when truncated (rstrip before the ellipsis). '' if none.
+    Semantics IDENTICAL to the client deriveReaderTitle (#165 P2.5)."""
+    for line in (text or "").split("\n"):
+        s = line.strip()
+        if s:
+            return (s[:_TITLE_MAX].rstrip() + "…") if len(s) > _TITLE_MAX else s
+    return ""
+
+
+def _session_titles_map(conn, session_ids):
+    """{sid: title} for the first non-marker, non-blank MAIN-session human line
+    per session (read-time, no migration). Windowed to the earliest 12 human
+    rows/session (rides idx_conv_session_ts); Python skips system markers. A
+    session whose first 12 human rows are all markers/blank is simply absent
+    (caller falls back). NOTE (Codex P1.2): the window ranks the full per-session
+    human partition before rn<=12 — confirmed index-ordered + bounded by the page
+    (≤200 sessions); per-session human counts are modest. If EXPLAIN QUERY PLAN
+    ever shows a temp B-tree sort here, switch to a per-session correlated
+    LIMIT 12 candidate fetch."""
+    if not session_ids:
+        return {}
+    titles = {}
+    ph = ",".join("?" for _ in session_ids)
+    rows = conn.execute(
+        "SELECT session_id, text FROM ("
+        "  SELECT session_id, text, "
+        "         ROW_NUMBER() OVER (PARTITION BY session_id "
+        "                            ORDER BY timestamp_utc, id) AS rn "
+        f"  FROM conversation_messages "
+        f"  WHERE session_id IN ({ph}) AND entry_type='human' "
+        "        AND is_sidechain=0 AND COALESCE(text,'') <> ''"
+        ") WHERE rn <= 12 ORDER BY session_id, rn",
+        tuple(session_ids),
+    ).fetchall()
+    for sid, text in rows:
+        if sid in titles:
+            continue                 # already resolved to the first non-marker
+        if _is_system_marker(text):
+            continue
+        t = _title_from_text(text)
+        if t:
+            titles[sid] = t
+    return titles
+
+
 def _project_label(cwd) -> str:
     """Basename of the project cwd (dashboard label posture — no reveal). Falls
     back to the raw path for root-ish cwds, '' when absent."""
@@ -155,9 +225,11 @@ def list_conversations(conn, *, sort="recent", limit=50, offset=0) -> dict:
     models = _session_models_map(conn, session_ids)
     # cwd/git_branch as the latest non-null (reader posture), NOT a lexical MAX().
     meta = _session_latest_meta_map(conn, session_ids)
+    titles = _session_titles_map(conn, session_ids)
     conversations = [
         {
             "session_id": sid,
+            "title": titles.get(sid) or _project_label(meta.get(sid, (None, None))[0]) or sid,
             "project_label": _project_label(meta.get(sid, (None, None))[0]),
             "git_branch": meta.get(sid, (None, None))[1],
             "started_utc": started,
@@ -239,8 +311,26 @@ def get_conversation(conn, session_id, *, after=None, limit=500):
     # item. A turn → exactly ONE item → cost counted exactly once. Humans,
     # tool_results, and assistant rows with a null msg_id emit as simple items at
     # their own position.
+    # ---- Phase 1: build items + index every assistant item's tool_use ids ----
+    # A tool_result is NOT guaranteed to sort after its tool_use (a grounded
+    # transcript scan found a matched result ordered BEFORE its use, plus orphan
+    # results with no in-session use), so this is a build-and-index-ALL pass
+    # FOLLOWED by a fold pass — never a single forward pass. None ids are never
+    # indexed (the id-less degradation guard).
     items = []
-    turn_index = {}        # (msg_id, req_id) -> index into items
+    turn_index = {}                # (msg_id, req_id) -> index into items
+    tooluse_index = {}             # tool_use id -> (item, block_dict)
+    tool_result_items = []         # placeholder items deferred to Phase 2
+
+    def _index_tool_uses(item):
+        # Index every tool_use id -> its (item, block). Idempotent: re-scanning
+        # a turn's blocks re-maps the same id to the same (item, block). Anthropic
+        # tool_use ids are unique within a session; a collision would be
+        # last-writer-wins (a result then folds to one deterministic owner).
+        for b in item["blocks"]:
+            if b.get("kind") == "tool_use" and b.get("id") is not None:
+                tooluse_index[b["id"]] = (item, b)
+
     for row in logical:
         (rid, u, ts, etype, text, blocks, model, msg_id, req_id,
          is_sc, cwd, branch, source_path, parent_uuid) = row
@@ -249,11 +339,64 @@ def get_conversation(conn, session_id, *, after=None, limit=500):
             idx = turn_index.get(key)
             if idx is None:
                 turn_index[key] = len(items)
-                items.append(_build_turn([row]))
+                it = _build_turn([row])
+                items.append(it)
+                _index_tool_uses(it)
             else:
                 _extend_turn(items[idx], row)
+                _index_tool_uses(items[idx])     # re-index the turn (idempotent; new fragment may add ids)
+        elif etype == "tool_result":
+            it = _build_simple(row)
+            items.append(it)
+            tool_result_items.append(it)
         else:
-            items.append(_build_simple(row))
+            it = _build_simple(row)
+            items.append(it)
+            if etype == "assistant":             # null-msg_id assistant: index its uses too
+                _index_tool_uses(it)
+
+    # ---- Phase 2: fold each tool_result item into its owning assistant item ----
+    drop = set()                                 # id() of folded placeholder items
+    for tr in tool_result_items:
+        tr_blocks = [b for b in tr["blocks"] if b.get("kind") == "tool_result"]
+        non_result = [b for b in tr["blocks"] if b.get("kind") != "tool_result"]
+        owners = []
+        resolved = []
+        for b in tr_blocks:
+            tid = b.get("tool_use_id")
+            hit = tooluse_index.get(tid) if tid is not None else None
+            if hit is None:
+                owners = None                    # an unresolved block -> keep standalone
+                break
+            owners.append(hit[0])
+            resolved.append((hit[1], b))
+        # fold iff every result block resolved to exactly ONE owning item, no leftovers
+        owner_ids = {id(o) for o in owners} if owners is not None else set()
+        if owners and not non_result and len(owner_ids) == 1:
+            owner = owners[0]
+            for use_block, res_block in resolved:
+                use_block["result"] = {"text": res_block.get("text", ""),
+                                       "truncated": bool(res_block.get("truncated")),
+                                       "is_error": bool(res_block.get("is_error"))}
+            owner["member_uuids"].append(tr["anchor"]["uuid"])
+            drop.add(id(tr))
+        # else: leave tr standalone (orphan / multi-owner / mixed) — a folded
+        # row's uuid then joins EXACTLY ONE item's member_uuids (the #160 anchor).
+
+    if drop:
+        items = [it for it in items if id(it) not in drop]
+
+    # ---- Phase 3: sweep every assistant item's tool_use -> tool_call ----
+    # Covers turn items AND _build_simple null-msg_id assistant items. Matched
+    # requests already carry `result`; unmatched get `result: None`
+    # (request-only). Post-migration the client never receives a bare tool_use.
+    for it in items:
+        if it["kind"] == "assistant":
+            for b in it["blocks"]:
+                if b.get("kind") == "tool_use":
+                    b["kind"] = "tool_call"
+                    b["tool_use_id"] = b.pop("id", None)
+                    b.setdefault("result", None)
 
     costs = _turn_cost_map(conn, list(turn_index))
     # Stamp per-item cost first, then derive the header from the SUM of the
@@ -480,6 +623,19 @@ def _attach_costs(conn, page):
     return page
 
 
+def _attach_titles(conn, page):
+    """Stamp each final-page hit with its session's derived title — ONE batched
+    _session_titles_map over the distinct page session_ids (parallel to
+    _attach_costs). Fallback project_label → session_id, matching
+    list_conversations (#165 Q4)."""
+    sids = list({h["session_id"] for h in page})
+    titles = _session_titles_map(conn, sids)
+    for h in page:
+        sid = h["session_id"]
+        h["title"] = titles.get(sid) or h.get("project_label") or sid
+    return page
+
+
 def _like_pattern(q):
     """Build the LIKE pattern for `q`. Escape the ESCAPE char (\\) FIRST, then
     the wildcards — otherwise a query containing a backslash (incl. a trailing
@@ -566,7 +722,8 @@ def _search_fts(conn, q, limit, offset):
     snips = _fts_snippets(conn, fts_q, [r[0] for r in page])
     hits = [_row_to_hit(uuid, sid, ts, cwd, snips.get(rid, ""), mid, rqd)
             for (rid, sid, uuid, ts, cwd, mid, rqd) in page]
-    return {"query": q, "mode": "fts", "hits": _attach_costs(conn, hits),
+    return {"query": q, "mode": "fts",
+            "hits": _attach_titles(conn, _attach_costs(conn, hits)),
             "total": total}
 
 
@@ -599,7 +756,8 @@ def _search_like(conn, q, limit, offset):
     hits = [_row_to_hit(uuid, sid, ts, cwd,
                         _manual_snippet(texts.get(rid, ""), q), mid, rqd)
             for (rid, sid, uuid, ts, cwd, mid, rqd) in page]
-    return {"query": q, "mode": "like", "hits": _attach_costs(conn, hits),
+    return {"query": q, "mode": "like",
+            "hits": _attach_titles(conn, _attach_costs(conn, hits)),
             "total": total}