cctally 1.44.3 → 1.46.0

package/CHANGELOG.md

@@ -5,6 +5,16 @@ based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 
 ## [Unreleased]
 
+## [1.46.0] - 2026-06-15
+
+### Added
+- Conversation viewer: the open reader now live-tails an active conversation in near-real time — new turns appear within about a second of the session's file changing (instead of waiting for the periodic 5-second dashboard tick) via a dedicated per-conversation SSE stream that watches only the open session's transcript; the live-tail is on by default and can be turned off with the new `dashboard.live_tail` config key (or the dashboard's Settings → "Conversation viewer" toggle), and `--no-sync` keeps it passive so frozen-data debugging is unaffected.
+
+## [1.45.0] - 2026-06-15
+
+### Added
+- Conversation viewer: the dashboard now marks any assistant turn that suffered a **prompt-cache failure** — where Claude re-created the bulk of its cached prompt prefix instead of reading it (most often on the first turn after you send a new message — usually well inside the cache TTL, not just on long idle gaps), re-billing those tokens at the higher cache-write rate. The failing turn gets an amber `⚡ CACHE REBUILT · <tokens> · +$<wasted>` chip in its header, and the conversation outline gains a matching amber landmark, a `⚡` quick-jump button (`c` / `C` keys), and a "Cache" count in the at-a-glance stats card. Detection is conservative (per model + thread: `cache_read` collapses below half its running high-water mark **and** ≥75% of the turn's context is freshly re-created, with the unavoidable first cache prime, model switches, and context compaction all excluded), so a typical session shows about one marker. The markers are on by default and can be turned off via the new `dashboard.cache_failure_markers` config key (or the dashboard's Settings → "Conversation viewer" toggle). The wasted-cost figure is a display-only estimate recomputed per read — no new database tables and no migration.
+
 ## [1.44.3] - 2026-06-15
 
 ### Fixed

package/bin/_cctally_cache.py

@@ -465,6 +465,20 @@ class IngestStats:
     # new inserts.
     rows_changed: int = 0
     lock_contended: bool = False
+    # Targeted (only_paths) live-tail fast-path fields. Default-clean so the
+    # only_paths=None callers (every existing caller) read targeted_clean=True
+    # and are otherwise unaffected.
+    files_failed: int = 0
+    deferred_reason: "str | None" = None
+
+    @property
+    def targeted_clean(self) -> bool:
+        """True ⇔ a targeted ingest fully applied: not contended, not deferred,
+        and no per-file failure. The watch loop emits + advances `seen` only
+        when this is True."""
+        return (not self.lock_contended
+                and self.deferred_reason is None
+                and self.files_failed == 0)
 
 
 def _progress_stderr(stats: IngestStats, *, force: bool = False) -> None:
@@ -563,11 +577,43 @@ def _ensure_session_files_row(conn: sqlite3.Connection, source_path: str) -> Non
     conn.commit()
 
 
+# Flags whose presence means the cache is mid-migration / mid-reingest. A
+# targeted (only_paths) ingest DECLINES when any is set and defers to the next
+# full background sync — inserting through a half-migrated FTS shape or skipping
+# a pending backfill would diverge from what a full sync produces (spec §
+# "Targeted ingest contract"). Enumerated against the flag-consumption blocks
+# guarded by the `if not rebuild and not targeted:` branch in sync_cache (the
+# full-sync-only `_consume_*` calls); keep this tuple in sync with those.
+_TARGETED_DECLINE_FLAGS = (
+    "conversation_backfill_pending",
+    "ai_titles_backfill_pending",
+    "conversation_reingest_pending",
+    "conversation_source_tool_use_reingest_pending",
+    "conversation_reingest_enrichment_pending",
+    "conversation_media_reingest_pending",
+    "conversation_search_split_pending",
+    "conversation_promote_command_args_pending",
+    "conversation_sessions_backfill_pending",
+)
+
+
+def _targeted_has_pending_global_work(conn) -> bool:
+    placeholders = ",".join("?" for _ in _TARGETED_DECLINE_FLAGS)
+    try:
+        row = conn.execute(
+            f"SELECT 1 FROM cache_meta WHERE key IN ({placeholders}) LIMIT 1",
+            _TARGETED_DECLINE_FLAGS).fetchone()
+    except sqlite3.OperationalError:
+        return False
+    return row is not None
+
+
 def sync_cache(
     conn: sqlite3.Connection,
     *,
     progress: Callable[[IngestStats], None] | None = None,
     rebuild: bool = False,
+    only_paths: "set[str] | None" = None,
 ) -> IngestStats:
     """Read-through delta ingest. Acquires an exclusive fcntl.flock; if
     another process holds it, returns immediately with lock_contended=True
@@ -592,6 +638,14 @@ def sync_cache(
             stats.lock_contended = True
             return stats
 
+        targeted = only_paths is not None
+        if targeted:
+            if rebuild:
+                raise ValueError("sync_cache: only_paths is incompatible with rebuild")
+            if _targeted_has_pending_global_work(conn):
+                stats.deferred_reason = "pending_global_flags"
+                return stats
+
         # Walk-complete sentinel gating (cctally-dev#93, D5b/D6b). Capture
         # whether cache 001 was already applied at the moment this sync
         # acquired the lock. The end-of-loop marker write is gated on this so
@@ -742,7 +796,7 @@ def sync_cache(
         # path, which already cleared the flag and repopulates via the normal
         # walk). A path-less/:memory: conn has no cache_meta only if the schema
         # was never applied; the try/except tolerates that.
-        if not rebuild:
+        if not rebuild and not targeted:
             try:
                 _pending = conn.execute(
                     "SELECT 1 FROM cache_meta "
@@ -865,7 +919,10 @@ def sync_cache(
             # HUMAN(text=args); the split-FTS UPDATE triggers re-index the args.
             _consume_promote_command_args(conn)
 
-        paths: list[pathlib.Path] = list(_iter_claude_jsonl_files())
+        if targeted:
+            paths = [pathlib.Path(p) for p in only_paths if pathlib.Path(p).is_file()]
+        else:
+            paths = list(_iter_claude_jsonl_files())
         stats.files_total = len(paths)
 
         # This SELECT does NOT open an implicit transaction (Python's
@@ -910,22 +967,29 @@ def sync_cache(
         # commit lands BEFORE the per-file read+parse loop, so no write
         # lock is held into that loop (same discipline as the truncation
         # escalation just below).
-        on_disk_paths = {str(jp) for jp in paths}
-        orphaned_tracked_paths = [
-            p for p, (size_bytes, _, _) in existing.items()
-            if size_bytes and p not in on_disk_paths
-        ]
-        if orphaned_tracked_paths:
-            eprint(
-                f"[cache] {len(orphaned_tracked_paths)} tracked file(s) no "
-                f"longer on disk; invalidating walk-complete marker "
-                f"(run `cache-sync --rebuild` to prune orphaned entries)"
-            )
-            conn.execute(
-                "DELETE FROM cache_meta WHERE key='claude_ingest_walk_complete'"
-            )
-            conn.commit()
-            walk_clean = False  # orphaned rows -> cache doesn't mirror disk (D5a)
+        # Targeted (only_paths) sync narrows `paths` to the requested file(s),
+        # so the orphan scan below — which infers "deleted from disk" from a
+        # tracked path's absence in `paths` — would mistake EVERY other tracked
+        # file for an orphan and nuke the walk-complete marker. Skip it entirely
+        # for targeted: the live-tail fast path never prunes orphans (the full
+        # background sync owns that).
+        if not targeted:
+            on_disk_paths = {str(jp) for jp in paths}
+            orphaned_tracked_paths = [
+                p for p, (size_bytes, _, _) in existing.items()
+                if size_bytes and p not in on_disk_paths
+            ]
+            if orphaned_tracked_paths:
+                eprint(
+                    f"[cache] {len(orphaned_tracked_paths)} tracked file(s) no "
+                    f"longer on disk; invalidating walk-complete marker "
+                    f"(run `cache-sync --rebuild` to prune orphaned entries)"
+                )
+                conn.execute(
+                    "DELETE FROM cache_meta WHERE key='claude_ingest_walk_complete'"
+                )
+                conn.commit()
+                walk_clean = False  # orphaned rows -> cache doesn't mirror disk (D5a)
 
         # Pre-scan for any truncation among tracked files. Under the
         # ccusage-parity ON CONFLICT DO UPDATE, source_path is PINNED to
@@ -958,6 +1022,14 @@ def sync_cache(
                 truncated_paths.add(str(jp))
 
         if truncated_paths:
+            if targeted:
+                # The targeted fast path must NEVER trigger the global
+                # full-cache wipe-and-re-ingest escalation below — that would
+                # turn a 1s live-tail tick into a multi-minute rebuild and drop
+                # every other session's rows. Decline and defer to the next full
+                # background sync, which owns the truncation escalation.
+                stats.deferred_reason = "truncation"
+                return stats
             eprint(
                 f"[cache-sync] truncation detected on {len(truncated_paths)} "
                 f"file(s) — re-ingesting all files (safe under ccusage-parity "
@@ -1035,6 +1107,7 @@ def sync_cache(
             except OSError as exc:
                 eprint(f"[cache] stat failed for {jp}: {exc}")
                 walk_clean = False  # skipped a file without ingesting (D5a)
+                stats.files_failed += 1
                 continue
 
             size = st.st_size
@@ -1118,6 +1191,7 @@ def sync_cache(
             except OSError as exc:
                 eprint(f"[cache] could not read {jp}: {exc}")
                 walk_clean = False  # skipped a file without ingesting (D5a)
+                stats.files_failed += 1
                 continue
 
             # Python's sqlite3 module starts an implicit transaction on the
@@ -1240,6 +1314,7 @@ def sync_cache(
                 eprint(f"[cache] db error on {jp}: {exc}")
                 conn.rollback()
                 walk_clean = False  # rolled back this file without ingesting (D5a)
+                stats.files_failed += 1
                 continue
 
             if progress is not None:
@@ -1280,7 +1355,7 @@ def sync_cache(
         # completeness. A lock-contended sync returned early above and never
         # reaches here. Presence (not the timestamp) is the gate signal; the
         # value stores the completion instant for doctor/debugging.
-        if walk_clean and applied_at_start:
+        if walk_clean and applied_at_start and not targeted:
             conn.execute(
                 "INSERT INTO cache_meta(key, value) "
                 "VALUES('claude_ingest_walk_complete', ?) "

package/bin/_cctally_config.py

@@ -302,6 +302,8 @@ ALLOWED_CONFIG_KEYS = (
     "alerts.command_template",
     "dashboard.bind",
     "dashboard.expose_transcripts",
+    "dashboard.cache_failure_markers",
+    "dashboard.live_tail",
     "update.check.enabled",
     "update.check.ttl_hours",
     "statusline.visual_burn_rate",
@@ -478,6 +480,46 @@ def _config_known_value(config: dict, key: str) -> "object":
             except ValueError:
                 return False
         return False
+    if key == "dashboard.cache_failure_markers":
+        # Boolean opt-OUT (spec §5). Default TRUE — absence is treated as ON
+        # (unlike dashboard.expose_transcripts, an opt-IN default of False).
+        # A hand-edited junk value surfaces the True default rather than
+        # crashing (mirrors dashboard.bind / expose_transcripts).
+        block = config.get("dashboard") if isinstance(config, dict) else None
+        if not isinstance(block, dict):
+            block = {}
+        stored = block.get("cache_failure_markers")
+        if stored is None:
+            return True
+        if isinstance(stored, bool):
+            return stored
+        # Only str spellings are normalizable; any other JSON scalar surfaces
+        # the default (the shared normalizer's .strip() would AttributeError on
+        # a bare int — uncaught by `except ValueError`).
+        if isinstance(stored, str):
+            try:
+                return c._normalize_alerts_enabled_value(stored)
+            except ValueError:
+                return True
+        return True
+    if key == "dashboard.live_tail":
+        # Boolean opt-OUT (spec §4.2). Default TRUE — absence is ON. A
+        # hand-edited junk value surfaces the True default. Mirrors
+        # dashboard.cache_failure_markers exactly.
+        block = config.get("dashboard") if isinstance(config, dict) else None
+        if not isinstance(block, dict):
+            block = {}
+        stored = block.get("live_tail")
+        if stored is None:
+            return True
+        if isinstance(stored, bool):
+            return stored
+        if isinstance(stored, str):
+            try:
+                return c._normalize_alerts_enabled_value(stored)
+            except ValueError:
+                return True
+        return True
     if key in ("update.check.enabled", "update.check.ttl_hours"):
         # Defaults mirror `_is_update_check_due` (True / 24 hours).
         # Hand-edited junk surfaces as the default — matches dashboard.bind.
@@ -864,6 +906,83 @@ def _cmd_config_set(args: argparse.Namespace) -> int:
                 f"{'true' if canonical else 'false'}"
             )
         return 0
+    if key == "dashboard.cache_failure_markers":
+        # Same read-modify-write posture as dashboard.expose_transcripts:
+        # validate first, then write under config_writer_lock with
+        # _load_config_unlocked (load_config inside the writer-lock
+        # self-deadlocks — fcntl.flock is per-fd). Preserves sibling
+        # dashboard.bind / dashboard.expose_transcripts. Reuse the shared
+        # bool-normalizer; catch + re-message with the actual key name (it
+        # hardcodes "alerts.enabled" in its ValueError text).
+        try:
+            canonical = c._normalize_alerts_enabled_value(raw)
+        except ValueError:
+            print(
+                f"cctally: invalid boolean value for "
+                f"dashboard.cache_failure_markers: "
+                f"{raw!r} (expected true|false|yes|no|1|0|on|off)",
+                file=sys.stderr,
+            )
+            return 2
+        with config_writer_lock():
+            config = _load_config_unlocked()
+            existing = config.get("dashboard")
+            if existing is not None and not isinstance(existing, dict):
+                print(
+                    "cctally: dashboard config error: dashboard must be an object",
+                    file=sys.stderr,
+                )
+                return 2
+            block = dict(existing or {})
+            block["cache_failure_markers"] = canonical
+            config["dashboard"] = block
+            save_config(config)
+        if getattr(args, "emit_json", False):
+            print(
+                json.dumps(
+                    {"dashboard": {"cache_failure_markers": canonical}}, indent=2
+                )
+            )
+        else:
+            print(
+                f"dashboard.cache_failure_markers="
+                f"{'true' if canonical else 'false'}"
+            )
+        return 0
+    if key == "dashboard.live_tail":
+        # Mirror dashboard.cache_failure_markers exactly: validate the bool
+        # first, then read-modify-write under config_writer_lock with
+        # _load_config_unlocked (load_config under the lock self-deadlocks).
+        # Preserves sibling dashboard.bind / expose_transcripts /
+        # cache_failure_markers. Re-message the shared normalizer's ValueError
+        # with the actual key name.
+        try:
+            canonical = c._normalize_alerts_enabled_value(raw)
+        except ValueError:
+            print(
+                f"cctally: invalid boolean value for dashboard.live_tail: "
+                f"{raw!r} (expected true|false|yes|no|1|0|on|off)",
+                file=sys.stderr,
+            )
+            return 2
+        with config_writer_lock():
+            config = _load_config_unlocked()
+            existing = config.get("dashboard")
+            if existing is not None and not isinstance(existing, dict):
+                print(
+                    "cctally: dashboard config error: dashboard must be an object",
+                    file=sys.stderr,
+                )
+                return 2
+            block = dict(existing or {})
+            block["live_tail"] = canonical
+            config["dashboard"] = block
+            save_config(config)
+        if getattr(args, "emit_json", False):
+            print(json.dumps({"dashboard": {"live_tail": canonical}}, indent=2))
+        else:
+            print(f"dashboard.live_tail={'true' if canonical else 'false'}")
+        return 0
     if key in (
         "statusline.visual_burn_rate",
         "statusline.cost_source",
@@ -1236,6 +1355,37 @@ def _cmd_config_unset(args: argparse.Namespace) -> int:
                 save_config(config)
             # idempotent: silent on missing key
         return 0
+    if key == "dashboard.cache_failure_markers":
+        # Mirror the dashboard.expose_transcripts unset branch: drop only the
+        # cache_failure_markers leaf; if the dashboard block ends up empty, drop
+        # the parent too so config.json stays tidy. Sibling dashboard.bind /
+        # expose_transcripts survive. Unsetting restores the True (opt-out)
+        # default at read time.
+        with config_writer_lock():
+            config = _load_config_unlocked()
+            block = config.get("dashboard")
+            if isinstance(block, dict) and "cache_failure_markers" in block:
+                del block["cache_failure_markers"]
+                if not block:
+                    config.pop("dashboard", None)
+                save_config(config)
+            # idempotent: silent on missing key
+        return 0
+    if key == "dashboard.live_tail":
+        # Mirror the cache_failure_markers unset branch: drop only the
+        # live_tail leaf; if the dashboard block ends up empty, drop the parent
+        # too. Sibling dashboard.bind / expose_transcripts / cache_failure_markers
+        # survive. Unsetting restores the True (opt-out) default at read time.
+        with config_writer_lock():
+            config = _load_config_unlocked()
+            block = config.get("dashboard")
+            if isinstance(block, dict) and "live_tail" in block:
+                del block["live_tail"]
+                if not block:
+                    config.pop("dashboard", None)
+                save_config(config)
+            # idempotent: silent on missing key
+        return 0
     if key in (
         "statusline.visual_burn_rate",
         "statusline.cost_source",