cctally 1.45.0 → 1.47.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +10 -0
- package/bin/_cctally_cache.py +94 -19
- package/bin/_cctally_config.py +68 -0
- package/bin/_cctally_dashboard.py +144 -16
- package/bin/_lib_conversation_query.py +48 -3
- package/bin/_lib_conversation_watch.py +67 -0
- package/dashboard/static/assets/index-BOR4FBvt.js +68 -0
- package/dashboard/static/assets/index-CuAtCrys.css +1 -0
- package/dashboard/static/dashboard.html +2 -2
- package/package.json +2 -1
- package/dashboard/static/assets/index-Ci-5UHk3.js +0 -68
- package/dashboard/static/assets/index-Dgq5Vvml.css +0 -1
package/CHANGELOG.md
CHANGED
|
@@ -5,6 +5,16 @@ based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
|
|
|
5
5
|
|
|
6
6
|
## [Unreleased]
|
|
7
7
|
|
|
8
|
+
## [1.47.0] - 2026-06-16
|
|
9
|
+
|
|
10
|
+
### Added
|
|
11
|
+
- **Dashboard:** the Recent Sessions modal now shows cache-rebuild events for the selected conversation — rebuild count, wasted $, tokens re-created, a session cache-value-saved figure, and a worst-first list of jump-links that open the conversation viewer at the message that triggered each rebuild (respects `dashboard.cache_failure_markers`).
|
|
12
|
+
|
|
13
|
+
## [1.46.0] - 2026-06-15
|
|
14
|
+
|
|
15
|
+
### Added
|
|
16
|
+
- Conversation viewer: the open reader now live-tails an active conversation in near-real time — new turns appear within about a second of the session's file changing (instead of waiting for the periodic 5-second dashboard tick) via a dedicated per-conversation SSE stream that watches only the open session's transcript; the live-tail is on by default and can be turned off with the new `dashboard.live_tail` config key (or the dashboard's Settings → "Conversation viewer" toggle), and `--no-sync` keeps it passive so frozen-data debugging is unaffected.
|
|
17
|
+
|
|
8
18
|
## [1.45.0] - 2026-06-15
|
|
9
19
|
|
|
10
20
|
### Added
|
package/bin/_cctally_cache.py
CHANGED
|
@@ -465,6 +465,20 @@ class IngestStats:
|
|
|
465
465
|
# new inserts.
|
|
466
466
|
rows_changed: int = 0
|
|
467
467
|
lock_contended: bool = False
|
|
468
|
+
# Targeted (only_paths) live-tail fast-path fields. Default-clean so the
|
|
469
|
+
# only_paths=None callers (every existing caller) read targeted_clean=True
|
|
470
|
+
# and are otherwise unaffected.
|
|
471
|
+
files_failed: int = 0
|
|
472
|
+
deferred_reason: "str | None" = None
|
|
473
|
+
|
|
474
|
+
@property
|
|
475
|
+
def targeted_clean(self) -> bool:
|
|
476
|
+
"""True ⇔ a targeted ingest fully applied: not contended, not deferred,
|
|
477
|
+
and no per-file failure. The watch loop emits + advances `seen` only
|
|
478
|
+
when this is True."""
|
|
479
|
+
return (not self.lock_contended
|
|
480
|
+
and self.deferred_reason is None
|
|
481
|
+
and self.files_failed == 0)
|
|
468
482
|
|
|
469
483
|
|
|
470
484
|
def _progress_stderr(stats: IngestStats, *, force: bool = False) -> None:
|
|
@@ -563,11 +577,43 @@ def _ensure_session_files_row(conn: sqlite3.Connection, source_path: str) -> Non
|
|
|
563
577
|
conn.commit()
|
|
564
578
|
|
|
565
579
|
|
|
580
|
+
# Flags whose presence means the cache is mid-migration / mid-reingest. A
|
|
581
|
+
# targeted (only_paths) ingest DECLINES when any is set and defers to the next
|
|
582
|
+
# full background sync — inserting through a half-migrated FTS shape or skipping
|
|
583
|
+
# a pending backfill would diverge from what a full sync produces (spec §
|
|
584
|
+
# "Targeted ingest contract"). Enumerated against the flag-consumption blocks
|
|
585
|
+
# guarded by the `if not rebuild and not targeted:` branch in sync_cache (the
|
|
586
|
+
# full-sync-only `_consume_*` calls); keep this tuple in sync with those.
|
|
587
|
+
_TARGETED_DECLINE_FLAGS = (
|
|
588
|
+
"conversation_backfill_pending",
|
|
589
|
+
"ai_titles_backfill_pending",
|
|
590
|
+
"conversation_reingest_pending",
|
|
591
|
+
"conversation_source_tool_use_reingest_pending",
|
|
592
|
+
"conversation_reingest_enrichment_pending",
|
|
593
|
+
"conversation_media_reingest_pending",
|
|
594
|
+
"conversation_search_split_pending",
|
|
595
|
+
"conversation_promote_command_args_pending",
|
|
596
|
+
"conversation_sessions_backfill_pending",
|
|
597
|
+
)
|
|
598
|
+
|
|
599
|
+
|
|
600
|
+
def _targeted_has_pending_global_work(conn) -> bool:
|
|
601
|
+
placeholders = ",".join("?" for _ in _TARGETED_DECLINE_FLAGS)
|
|
602
|
+
try:
|
|
603
|
+
row = conn.execute(
|
|
604
|
+
f"SELECT 1 FROM cache_meta WHERE key IN ({placeholders}) LIMIT 1",
|
|
605
|
+
_TARGETED_DECLINE_FLAGS).fetchone()
|
|
606
|
+
except sqlite3.OperationalError:
|
|
607
|
+
return False
|
|
608
|
+
return row is not None
|
|
609
|
+
|
|
610
|
+
|
|
566
611
|
def sync_cache(
|
|
567
612
|
conn: sqlite3.Connection,
|
|
568
613
|
*,
|
|
569
614
|
progress: Callable[[IngestStats], None] | None = None,
|
|
570
615
|
rebuild: bool = False,
|
|
616
|
+
only_paths: "set[str] | None" = None,
|
|
571
617
|
) -> IngestStats:
|
|
572
618
|
"""Read-through delta ingest. Acquires an exclusive fcntl.flock; if
|
|
573
619
|
another process holds it, returns immediately with lock_contended=True
|
|
@@ -592,6 +638,14 @@ def sync_cache(
|
|
|
592
638
|
stats.lock_contended = True
|
|
593
639
|
return stats
|
|
594
640
|
|
|
641
|
+
targeted = only_paths is not None
|
|
642
|
+
if targeted:
|
|
643
|
+
if rebuild:
|
|
644
|
+
raise ValueError("sync_cache: only_paths is incompatible with rebuild")
|
|
645
|
+
if _targeted_has_pending_global_work(conn):
|
|
646
|
+
stats.deferred_reason = "pending_global_flags"
|
|
647
|
+
return stats
|
|
648
|
+
|
|
595
649
|
# Walk-complete sentinel gating (cctally-dev#93, D5b/D6b). Capture
|
|
596
650
|
# whether cache 001 was already applied at the moment this sync
|
|
597
651
|
# acquired the lock. The end-of-loop marker write is gated on this so
|
|
@@ -742,7 +796,7 @@ def sync_cache(
|
|
|
742
796
|
# path, which already cleared the flag and repopulates via the normal
|
|
743
797
|
# walk). A path-less/:memory: conn has no cache_meta only if the schema
|
|
744
798
|
# was never applied; the try/except tolerates that.
|
|
745
|
-
if not rebuild:
|
|
799
|
+
if not rebuild and not targeted:
|
|
746
800
|
try:
|
|
747
801
|
_pending = conn.execute(
|
|
748
802
|
"SELECT 1 FROM cache_meta "
|
|
@@ -865,7 +919,10 @@ def sync_cache(
|
|
|
865
919
|
# HUMAN(text=args); the split-FTS UPDATE triggers re-index the args.
|
|
866
920
|
_consume_promote_command_args(conn)
|
|
867
921
|
|
|
868
|
-
|
|
922
|
+
if targeted:
|
|
923
|
+
paths = [pathlib.Path(p) for p in only_paths if pathlib.Path(p).is_file()]
|
|
924
|
+
else:
|
|
925
|
+
paths = list(_iter_claude_jsonl_files())
|
|
869
926
|
stats.files_total = len(paths)
|
|
870
927
|
|
|
871
928
|
# This SELECT does NOT open an implicit transaction (Python's
|
|
@@ -910,22 +967,29 @@ def sync_cache(
|
|
|
910
967
|
# commit lands BEFORE the per-file read+parse loop, so no write
|
|
911
968
|
# lock is held into that loop (same discipline as the truncation
|
|
912
969
|
# escalation just below).
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
970
|
+
# Targeted (only_paths) sync narrows `paths` to the requested file(s),
|
|
971
|
+
# so the orphan scan below — which infers "deleted from disk" from a
|
|
972
|
+
# tracked path's absence in `paths` — would mistake EVERY other tracked
|
|
973
|
+
# file for an orphan and nuke the walk-complete marker. Skip it entirely
|
|
974
|
+
# for targeted: the live-tail fast path never prunes orphans (the full
|
|
975
|
+
# background sync owns that).
|
|
976
|
+
if not targeted:
|
|
977
|
+
on_disk_paths = {str(jp) for jp in paths}
|
|
978
|
+
orphaned_tracked_paths = [
|
|
979
|
+
p for p, (size_bytes, _, _) in existing.items()
|
|
980
|
+
if size_bytes and p not in on_disk_paths
|
|
981
|
+
]
|
|
982
|
+
if orphaned_tracked_paths:
|
|
983
|
+
eprint(
|
|
984
|
+
f"[cache] {len(orphaned_tracked_paths)} tracked file(s) no "
|
|
985
|
+
f"longer on disk; invalidating walk-complete marker "
|
|
986
|
+
f"(run `cache-sync --rebuild` to prune orphaned entries)"
|
|
987
|
+
)
|
|
988
|
+
conn.execute(
|
|
989
|
+
"DELETE FROM cache_meta WHERE key='claude_ingest_walk_complete'"
|
|
990
|
+
)
|
|
991
|
+
conn.commit()
|
|
992
|
+
walk_clean = False # orphaned rows -> cache doesn't mirror disk (D5a)
|
|
929
993
|
|
|
930
994
|
# Pre-scan for any truncation among tracked files. Under the
|
|
931
995
|
# ccusage-parity ON CONFLICT DO UPDATE, source_path is PINNED to
|
|
@@ -958,6 +1022,14 @@ def sync_cache(
|
|
|
958
1022
|
truncated_paths.add(str(jp))
|
|
959
1023
|
|
|
960
1024
|
if truncated_paths:
|
|
1025
|
+
if targeted:
|
|
1026
|
+
# The targeted fast path must NEVER trigger the global
|
|
1027
|
+
# full-cache wipe-and-re-ingest escalation below — that would
|
|
1028
|
+
# turn a 1s live-tail tick into a multi-minute rebuild and drop
|
|
1029
|
+
# every other session's rows. Decline and defer to the next full
|
|
1030
|
+
# background sync, which owns the truncation escalation.
|
|
1031
|
+
stats.deferred_reason = "truncation"
|
|
1032
|
+
return stats
|
|
961
1033
|
eprint(
|
|
962
1034
|
f"[cache-sync] truncation detected on {len(truncated_paths)} "
|
|
963
1035
|
f"file(s) — re-ingesting all files (safe under ccusage-parity "
|
|
@@ -1035,6 +1107,7 @@ def sync_cache(
|
|
|
1035
1107
|
except OSError as exc:
|
|
1036
1108
|
eprint(f"[cache] stat failed for {jp}: {exc}")
|
|
1037
1109
|
walk_clean = False # skipped a file without ingesting (D5a)
|
|
1110
|
+
stats.files_failed += 1
|
|
1038
1111
|
continue
|
|
1039
1112
|
|
|
1040
1113
|
size = st.st_size
|
|
@@ -1118,6 +1191,7 @@ def sync_cache(
|
|
|
1118
1191
|
except OSError as exc:
|
|
1119
1192
|
eprint(f"[cache] could not read {jp}: {exc}")
|
|
1120
1193
|
walk_clean = False # skipped a file without ingesting (D5a)
|
|
1194
|
+
stats.files_failed += 1
|
|
1121
1195
|
continue
|
|
1122
1196
|
|
|
1123
1197
|
# Python's sqlite3 module starts an implicit transaction on the
|
|
@@ -1240,6 +1314,7 @@ def sync_cache(
|
|
|
1240
1314
|
eprint(f"[cache] db error on {jp}: {exc}")
|
|
1241
1315
|
conn.rollback()
|
|
1242
1316
|
walk_clean = False # rolled back this file without ingesting (D5a)
|
|
1317
|
+
stats.files_failed += 1
|
|
1243
1318
|
continue
|
|
1244
1319
|
|
|
1245
1320
|
if progress is not None:
|
|
@@ -1280,7 +1355,7 @@ def sync_cache(
|
|
|
1280
1355
|
# completeness. A lock-contended sync returned early above and never
|
|
1281
1356
|
# reaches here. Presence (not the timestamp) is the gate signal; the
|
|
1282
1357
|
# value stores the completion instant for doctor/debugging.
|
|
1283
|
-
if walk_clean and applied_at_start:
|
|
1358
|
+
if walk_clean and applied_at_start and not targeted:
|
|
1284
1359
|
conn.execute(
|
|
1285
1360
|
"INSERT INTO cache_meta(key, value) "
|
|
1286
1361
|
"VALUES('claude_ingest_walk_complete', ?) "
|
package/bin/_cctally_config.py
CHANGED
|
@@ -303,6 +303,7 @@ ALLOWED_CONFIG_KEYS = (
|
|
|
303
303
|
"dashboard.bind",
|
|
304
304
|
"dashboard.expose_transcripts",
|
|
305
305
|
"dashboard.cache_failure_markers",
|
|
306
|
+
"dashboard.live_tail",
|
|
306
307
|
"update.check.enabled",
|
|
307
308
|
"update.check.ttl_hours",
|
|
308
309
|
"statusline.visual_burn_rate",
|
|
@@ -501,6 +502,24 @@ def _config_known_value(config: dict, key: str) -> "object":
|
|
|
501
502
|
except ValueError:
|
|
502
503
|
return True
|
|
503
504
|
return True
|
|
505
|
+
if key == "dashboard.live_tail":
|
|
506
|
+
# Boolean opt-OUT (spec §4.2). Default TRUE — absence is ON. A
|
|
507
|
+
# hand-edited junk value surfaces the True default. Mirrors
|
|
508
|
+
# dashboard.cache_failure_markers exactly.
|
|
509
|
+
block = config.get("dashboard") if isinstance(config, dict) else None
|
|
510
|
+
if not isinstance(block, dict):
|
|
511
|
+
block = {}
|
|
512
|
+
stored = block.get("live_tail")
|
|
513
|
+
if stored is None:
|
|
514
|
+
return True
|
|
515
|
+
if isinstance(stored, bool):
|
|
516
|
+
return stored
|
|
517
|
+
if isinstance(stored, str):
|
|
518
|
+
try:
|
|
519
|
+
return c._normalize_alerts_enabled_value(stored)
|
|
520
|
+
except ValueError:
|
|
521
|
+
return True
|
|
522
|
+
return True
|
|
504
523
|
if key in ("update.check.enabled", "update.check.ttl_hours"):
|
|
505
524
|
# Defaults mirror `_is_update_check_due` (True / 24 hours).
|
|
506
525
|
# Hand-edited junk surfaces as the default — matches dashboard.bind.
|
|
@@ -930,6 +949,40 @@ def _cmd_config_set(args: argparse.Namespace) -> int:
|
|
|
930
949
|
f"{'true' if canonical else 'false'}"
|
|
931
950
|
)
|
|
932
951
|
return 0
|
|
952
|
+
if key == "dashboard.live_tail":
|
|
953
|
+
# Mirror dashboard.cache_failure_markers exactly: validate the bool
|
|
954
|
+
# first, then read-modify-write under config_writer_lock with
|
|
955
|
+
# _load_config_unlocked (load_config under the lock self-deadlocks).
|
|
956
|
+
# Preserves sibling dashboard.bind / expose_transcripts /
|
|
957
|
+
# cache_failure_markers. Re-message the shared normalizer's ValueError
|
|
958
|
+
# with the actual key name.
|
|
959
|
+
try:
|
|
960
|
+
canonical = c._normalize_alerts_enabled_value(raw)
|
|
961
|
+
except ValueError:
|
|
962
|
+
print(
|
|
963
|
+
f"cctally: invalid boolean value for dashboard.live_tail: "
|
|
964
|
+
f"{raw!r} (expected true|false|yes|no|1|0|on|off)",
|
|
965
|
+
file=sys.stderr,
|
|
966
|
+
)
|
|
967
|
+
return 2
|
|
968
|
+
with config_writer_lock():
|
|
969
|
+
config = _load_config_unlocked()
|
|
970
|
+
existing = config.get("dashboard")
|
|
971
|
+
if existing is not None and not isinstance(existing, dict):
|
|
972
|
+
print(
|
|
973
|
+
"cctally: dashboard config error: dashboard must be an object",
|
|
974
|
+
file=sys.stderr,
|
|
975
|
+
)
|
|
976
|
+
return 2
|
|
977
|
+
block = dict(existing or {})
|
|
978
|
+
block["live_tail"] = canonical
|
|
979
|
+
config["dashboard"] = block
|
|
980
|
+
save_config(config)
|
|
981
|
+
if getattr(args, "emit_json", False):
|
|
982
|
+
print(json.dumps({"dashboard": {"live_tail": canonical}}, indent=2))
|
|
983
|
+
else:
|
|
984
|
+
print(f"dashboard.live_tail={'true' if canonical else 'false'}")
|
|
985
|
+
return 0
|
|
933
986
|
if key in (
|
|
934
987
|
"statusline.visual_burn_rate",
|
|
935
988
|
"statusline.cost_source",
|
|
@@ -1318,6 +1371,21 @@ def _cmd_config_unset(args: argparse.Namespace) -> int:
|
|
|
1318
1371
|
save_config(config)
|
|
1319
1372
|
# idempotent: silent on missing key
|
|
1320
1373
|
return 0
|
|
1374
|
+
if key == "dashboard.live_tail":
|
|
1375
|
+
# Mirror the cache_failure_markers unset branch: drop only the
|
|
1376
|
+
# live_tail leaf; if the dashboard block ends up empty, drop the parent
|
|
1377
|
+
# too. Sibling dashboard.bind / expose_transcripts / cache_failure_markers
|
|
1378
|
+
# survive. Unsetting restores the True (opt-out) default at read time.
|
|
1379
|
+
with config_writer_lock():
|
|
1380
|
+
config = _load_config_unlocked()
|
|
1381
|
+
block = config.get("dashboard")
|
|
1382
|
+
if isinstance(block, dict) and "live_tail" in block:
|
|
1383
|
+
del block["live_tail"]
|
|
1384
|
+
if not block:
|
|
1385
|
+
config.pop("dashboard", None)
|
|
1386
|
+
save_config(config)
|
|
1387
|
+
# idempotent: silent on missing key
|
|
1388
|
+
return 0
|
|
1321
1389
|
if key in (
|
|
1322
1390
|
"statusline.visual_burn_rate",
|
|
1323
1391
|
"statusline.cost_source",
|
|
@@ -317,7 +317,7 @@ from _lib_subscription_weeks import _compute_subscription_weeks
|
|
|
317
317
|
from _lib_blocks import _group_entries_into_blocks
|
|
318
318
|
from _cctally_config import save_config, _load_config_unlocked
|
|
319
319
|
from _cctally_db import _render_migration_error_banner
|
|
320
|
-
from _cctally_cache import get_entries, open_cache_db
|
|
320
|
+
from _cctally_cache import get_entries, open_cache_db, sync_cache
|
|
321
321
|
|
|
322
322
|
|
|
323
323
|
# === Module-level back-ref shims for helpers that STAY in bin/cctally ======
|
|
@@ -623,6 +623,12 @@ BLOCK_DURATION = sys.modules["cctally"].BLOCK_DURATION
|
|
|
623
623
|
# === STATIC_DIR — dashboard static-asset root ==============================
|
|
624
624
|
STATIC_DIR = pathlib.Path(__file__).resolve().parent.parent / "dashboard" / "static"
|
|
625
625
|
|
|
626
|
+
# Conversation live-tail watch loop (spec §4.1). Single-file stat poll: cheap.
|
|
627
|
+
_LIVE_TAIL_POLL_INTERVAL = 1.0 # seconds between stat polls of the open file(s)
|
|
628
|
+
_LIVE_TAIL_DEBOUNCE = 0.25 # settle window after first detected growth
|
|
629
|
+
_LIVE_TAIL_KEEPALIVE = 15.0 # idle keep-alive cadence (proxy guard)
|
|
630
|
+
_LIVE_TAIL_FILE_RESET_EVERY = 10 # re-resolve the session file set every N cycles
|
|
631
|
+
|
|
626
632
|
|
|
627
633
|
# === Dashboard bind validators (config + cmd_dashboard) ====================
|
|
628
634
|
|
|
@@ -4921,8 +4927,10 @@ def snapshot_to_envelope(snap: "DataSnapshot", *,
|
|
|
4921
4927
|
_dash_cfg = _cfg_for_alerts.get("dashboard") if isinstance(
|
|
4922
4928
|
_cfg_for_alerts.get("dashboard"), dict) else {}
|
|
4923
4929
|
_cfm = _dash_cfg.get("cache_failure_markers", True)
|
|
4930
|
+
_lt = _dash_cfg.get("live_tail", True)
|
|
4924
4931
|
dashboard_prefs = {
|
|
4925
4932
|
"cache_failure_markers": _cfm if isinstance(_cfm, bool) else True,
|
|
4933
|
+
"live_tail": _lt if isinstance(_lt, bool) else True,
|
|
4926
4934
|
}
|
|
4927
4935
|
|
|
4928
4936
|
# Mirror update-state.json + update-suppress.json into the envelope
|
|
@@ -5268,6 +5276,30 @@ def _qs_str(q: dict, key: str, default: str | None) -> str | None:
|
|
|
5268
5276
|
_CONV_SEARCH_KINDS = ("all", "prompts", "assistant", "tools", "thinking")
|
|
5269
5277
|
|
|
5270
5278
|
|
|
5279
|
+
def _cached_file_sigs(conn, paths):
|
|
5280
|
+
"""{path: size_bytes} from session_files for the given paths — the cache's
|
|
5281
|
+
own view of how far each file is ingested. Size-only by design, matching the
|
|
5282
|
+
watch kernel's size-only signature (`file_sig`) and sync_cache's size-only
|
|
5283
|
+
delta signal: mtime is NOT consulted, because a size-unchanged ingest does
|
|
5284
|
+
not refresh session_files.mtime_ns and a stale mtime would re-detect
|
|
5285
|
+
'changed' every cycle forever. Used to baseline the live-tail watch so a file
|
|
5286
|
+
the cache hasn't caught up on reads as 'changed' on cycle 1 (spec §2.4).
|
|
5287
|
+
Paths with no row are simply absent → treated as changed."""
|
|
5288
|
+
out = {}
|
|
5289
|
+
if not paths:
|
|
5290
|
+
return out
|
|
5291
|
+
placeholders = ",".join("?" for _ in paths)
|
|
5292
|
+
try:
|
|
5293
|
+
rows = conn.execute(
|
|
5294
|
+
f"SELECT path, size_bytes FROM session_files "
|
|
5295
|
+
f"WHERE path IN ({placeholders})", list(paths)).fetchall()
|
|
5296
|
+
except sqlite3.OperationalError:
|
|
5297
|
+
return out
|
|
5298
|
+
for p, size in rows:
|
|
5299
|
+
out[p] = size
|
|
5300
|
+
return out
|
|
5301
|
+
|
|
5302
|
+
|
|
5271
5303
|
class DashboardHTTPHandler(BaseHTTPRequestHandler):
|
|
5272
5304
|
"""Routes:
|
|
5273
5305
|
GET / → dashboard.html
|
|
@@ -5406,6 +5438,10 @@ class DashboardHTTPHandler(BaseHTTPRequestHandler):
|
|
|
5406
5438
|
# #177 S6: in-conversation find → rendered-turn anchors. Matched
|
|
5407
5439
|
# BEFORE the <id> reader catch-all (same precedence as /outline).
|
|
5408
5440
|
self._handle_get_conversation_find(path)
|
|
5441
|
+
elif path.startswith("/api/conversation/") and path.endswith("/events"):
|
|
5442
|
+
# Live-tail SSE for the open reader (spec §2). Matched BEFORE the
|
|
5443
|
+
# <id> reader catch-all.
|
|
5444
|
+
self._handle_get_conversation_events(path)
|
|
5409
5445
|
elif path.startswith("/api/conversation/"):
|
|
5410
5446
|
self._handle_get_conversation_detail(path)
|
|
5411
5447
|
else:
|
|
@@ -5879,7 +5915,7 @@ class DashboardHTTPHandler(BaseHTTPRequestHandler):
|
|
|
5879
5915
|
"field": f"dashboard.{leaf}"},
|
|
5880
5916
|
)
|
|
5881
5917
|
return
|
|
5882
|
-
if leaf
|
|
5918
|
+
if leaf not in ("cache_failure_markers", "live_tail"):
|
|
5883
5919
|
self._respond_json(
|
|
5884
5920
|
400,
|
|
5885
5921
|
{"error": f"unknown dashboard settings key: {leaf}",
|
|
@@ -5887,18 +5923,16 @@ class DashboardHTTPHandler(BaseHTTPRequestHandler):
|
|
|
5887
5923
|
)
|
|
5888
5924
|
return
|
|
5889
5925
|
dashboard_validated = {}
|
|
5890
|
-
|
|
5891
|
-
if
|
|
5892
|
-
|
|
5893
|
-
|
|
5894
|
-
|
|
5895
|
-
|
|
5896
|
-
|
|
5897
|
-
|
|
5898
|
-
|
|
5899
|
-
|
|
5900
|
-
dashboard_block["cache_failure_markers"]
|
|
5901
|
-
)
|
|
5926
|
+
for _leaf in ("cache_failure_markers", "live_tail"):
|
|
5927
|
+
if _leaf in dashboard_block:
|
|
5928
|
+
if not isinstance(dashboard_block[_leaf], bool):
|
|
5929
|
+
self._respond_json(
|
|
5930
|
+
400,
|
|
5931
|
+
{"error": f"dashboard.{_leaf} must be a JSON boolean",
|
|
5932
|
+
"field": f"dashboard.{_leaf}"},
|
|
5933
|
+
)
|
|
5934
|
+
return
|
|
5935
|
+
dashboard_validated[_leaf] = dashboard_block[_leaf]
|
|
5902
5936
|
|
|
5903
5937
|
# Pre-validate update shape. Only `update.check.{enabled,ttl_hours}`
|
|
5904
5938
|
# is settable today; any other key under `update` or `update.check`
|
|
@@ -6235,14 +6269,15 @@ class DashboardHTTPHandler(BaseHTTPRequestHandler):
|
|
|
6235
6269
|
"anomaly_threshold_pp": stored_threshold,
|
|
6236
6270
|
}
|
|
6237
6271
|
if dashboard_validated is not None:
|
|
6238
|
-
# Echo the persisted
|
|
6239
|
-
#
|
|
6272
|
+
# Echo the persisted dashboard-writable leaves (cache_failure_markers
|
|
6273
|
+
# + live_tail) so the SettingsOverlay can repaint without a
|
|
6240
6274
|
# follow-up GET. Default true (opt-out) when nothing is persisted.
|
|
6241
6275
|
persisted_dash = merged.get("dashboard") or {}
|
|
6242
6276
|
out["dashboard"] = {
|
|
6243
6277
|
"cache_failure_markers": bool(
|
|
6244
6278
|
persisted_dash.get("cache_failure_markers", True)
|
|
6245
6279
|
),
|
|
6280
|
+
"live_tail": bool(persisted_dash.get("live_tail", True)),
|
|
6246
6281
|
}
|
|
6247
6282
|
out["saved_at"] = (
|
|
6248
6283
|
dt.datetime.now(dt.timezone.utc)
|
|
@@ -7481,6 +7516,99 @@ class DashboardHTTPHandler(BaseHTTPRequestHandler):
|
|
|
7481
7516
|
return
|
|
7482
7517
|
self._respond_json(200, body)
|
|
7483
7518
|
|
|
7519
|
+
def _handle_get_conversation_events(self, path: str) -> None:
|
|
7520
|
+
"""``GET /api/conversation/<id>/events`` — per-conversation live-tail
|
|
7521
|
+
SSE (spec §2). Fail-closed behind the same transcript privacy gate as
|
|
7522
|
+
the other conversation routes. Watches only this session's file(s);
|
|
7523
|
+
emits ``event: tail`` on growth, ``: keep-alive`` when idle. Passive
|
|
7524
|
+
(no ingest, no emit) under ``--no-sync``."""
|
|
7525
|
+
if not self._require_transcripts_allowed():
|
|
7526
|
+
return
|
|
7527
|
+
import time as _time
|
|
7528
|
+
import urllib.parse as _u
|
|
7529
|
+
watch = sys.modules["cctally"]._load_sibling("_lib_conversation_watch")
|
|
7530
|
+
cq = self._conversation_query()
|
|
7531
|
+
session_id = _u.unquote(path[len("/api/conversation/"):-len("/events")])
|
|
7532
|
+
if not session_id:
|
|
7533
|
+
self.send_error(404, "conversation not found")
|
|
7534
|
+
return
|
|
7535
|
+
|
|
7536
|
+
self.send_response(200)
|
|
7537
|
+
self.send_header("Content-Type", "text/event-stream; charset=utf-8")
|
|
7538
|
+
self.send_header("Cache-Control", "no-cache")
|
|
7539
|
+
self.send_header("Connection", "keep-alive")
|
|
7540
|
+
self.send_header("X-Accel-Buffering", "no")
|
|
7541
|
+
self.end_headers()
|
|
7542
|
+
|
|
7543
|
+
passive = bool(type(self).no_sync)
|
|
7544
|
+
|
|
7545
|
+
try:
|
|
7546
|
+
conn = open_cache_db()
|
|
7547
|
+
except (sqlite3.DatabaseError, OSError):
|
|
7548
|
+
# Cache unavailable — degrade to keep-alive only; client backstop
|
|
7549
|
+
# tick still surfaces turns. (Headers already sent; can't 500.)
|
|
7550
|
+
passive = True
|
|
7551
|
+
conn = None
|
|
7552
|
+
|
|
7553
|
+
def _resolve():
|
|
7554
|
+
return cq.session_source_paths(conn, session_id) if conn else []
|
|
7555
|
+
|
|
7556
|
+
def _ingest(changed):
|
|
7557
|
+
return sync_cache(conn, only_paths=set(changed))
|
|
7558
|
+
|
|
7559
|
+
try:
|
|
7560
|
+
if passive:
|
|
7561
|
+
# Frozen-data contract: no ingest, no emit. Keep-alive only.
|
|
7562
|
+
while True:
|
|
7563
|
+
_time.sleep(_LIVE_TAIL_KEEPALIVE)
|
|
7564
|
+
self.wfile.write(b": keep-alive\n\n")
|
|
7565
|
+
self.wfile.flush()
|
|
7566
|
+
|
|
7567
|
+
files = _resolve()
|
|
7568
|
+
# Best-effort connect ingest for immediacy, then baseline `seen`
|
|
7569
|
+
# from the cache's own offsets (session_files) so any pre-connect
|
|
7570
|
+
# growth the connect-ingest declined is still caught on cycle 1.
|
|
7571
|
+
try:
|
|
7572
|
+
if files:
|
|
7573
|
+
sync_cache(conn, only_paths=set(files))
|
|
7574
|
+
except sqlite3.DatabaseError:
|
|
7575
|
+
pass
|
|
7576
|
+
seen = _cached_file_sigs(conn, files)
|
|
7577
|
+
|
|
7578
|
+
idle = 0.0
|
|
7579
|
+
cycles = 0
|
|
7580
|
+
while True:
|
|
7581
|
+
_time.sleep(_LIVE_TAIL_POLL_INTERVAL)
|
|
7582
|
+
cycles += 1
|
|
7583
|
+
changed = watch.changed_paths(files, seen)
|
|
7584
|
+
if changed:
|
|
7585
|
+
_time.sleep(_LIVE_TAIL_DEBOUNCE)
|
|
7586
|
+
new_seen, emitted = watch.watch_step(
|
|
7587
|
+
files, seen, ingest_fn=_ingest,
|
|
7588
|
+
committed_sig_fn=lambda p: _cached_file_sigs(conn, [p]).get(p))
|
|
7589
|
+
seen = new_seen
|
|
7590
|
+
if emitted:
|
|
7591
|
+
self.wfile.write(
|
|
7592
|
+
("event: tail\ndata: "
|
|
7593
|
+
+ json.dumps({"sessionId": session_id})
|
|
7594
|
+
+ "\n\n").encode("utf-8"))
|
|
7595
|
+
self.wfile.flush()
|
|
7596
|
+
idle = 0.0
|
|
7597
|
+
continue
|
|
7598
|
+
idle += _LIVE_TAIL_POLL_INTERVAL
|
|
7599
|
+
if idle >= _LIVE_TAIL_KEEPALIVE:
|
|
7600
|
+
self.wfile.write(b": keep-alive\n\n")
|
|
7601
|
+
self.wfile.flush()
|
|
7602
|
+
idle = 0.0
|
|
7603
|
+
if cycles % _LIVE_TAIL_FILE_RESET_EVERY == 0:
|
|
7604
|
+
files = _resolve()
|
|
7605
|
+
seen = {p: s for p, s in seen.items() if p in set(files)}
|
|
7606
|
+
except (BrokenPipeError, ConnectionResetError):
|
|
7607
|
+
pass # client disconnect is normal
|
|
7608
|
+
finally:
|
|
7609
|
+
if conn is not None:
|
|
7610
|
+
conn.close()
|
|
7611
|
+
|
|
7484
7612
|
def _handle_get_conversation_search(self) -> None:
|
|
7485
7613
|
"""``GET /api/conversation/search?q=...&kind=...`` — cross-session
|
|
7486
7614
|
FTS/LIKE search (spec §3.3). Matched BEFORE the ``<id>`` reader in
|
|
@@ -313,6 +313,18 @@ def _cache_failure_wasted_usd(model, lost):
|
|
|
313
313
|
return write - read
|
|
314
314
|
|
|
315
315
|
|
|
316
|
+
def _cache_read_saved_usd(model, cache_read):
|
|
317
|
+
"""Marginal USD the cache SAVED this turn: the `cache_read` prefix priced at
|
|
318
|
+
the full input rate minus its actual cache-READ rate. Display-only (same
|
|
319
|
+
caveat as `_cache_failure_wasted_usd`): NEVER summed into a cost-snapshot /
|
|
320
|
+
budget / reconciled figure. `input_tokens` and `cache_read_input_tokens` are
|
|
321
|
+
independent keys in the Claude `_calculate_entry_cost` (no subset
|
|
322
|
+
subtraction), so passing each alone yields the two rates cleanly."""
|
|
323
|
+
full = _calculate_entry_cost(model or "", {"input_tokens": cache_read})
|
|
324
|
+
read = _calculate_entry_cost(model or "", {"cache_read_input_tokens": cache_read})
|
|
325
|
+
return full - read
|
|
326
|
+
|
|
327
|
+
|
|
316
328
|
def _stamp_cache_failures(items):
|
|
317
329
|
"""Stamp ``item["cache_failure"]`` on each assistant turn that re-creates the
|
|
318
330
|
bulk of its cached prefix instead of reading it (spec §1). Mutates `items` in
|
|
@@ -451,6 +463,20 @@ def _session_latest_meta_map(conn, session_ids):
|
|
|
451
463
|
return meta
|
|
452
464
|
|
|
453
465
|
|
|
466
|
+
def session_source_paths(conn, session_id):
|
|
467
|
+
"""Distinct JSONL source files backing one session — the file-set the
|
|
468
|
+
live-tail watch loop polls (spec §2.3). Reads ``conversation_messages``,
|
|
469
|
+
the reader's own source of truth, NOT ``session_files`` (whose ``session_id``
|
|
470
|
+
is lazy / filename-fallback). Returns a list of path strings; empty for an
|
|
471
|
+
unknown or not-yet-ingested session.
|
|
472
|
+
"""
|
|
473
|
+
rows = conn.execute(
|
|
474
|
+
"SELECT DISTINCT source_path FROM conversation_messages "
|
|
475
|
+
"WHERE session_id=? AND source_path IS NOT NULL",
|
|
476
|
+
(session_id,)).fetchall()
|
|
477
|
+
return [r[0] for r in rows]
|
|
478
|
+
|
|
479
|
+
|
|
454
480
|
# Rail sort keys, in the rollup table's STRUCTURAL columns (Task A). ``recent``
|
|
455
481
|
# rides idx_conv_sessions_recent(last_activity_utc DESC, session_id DESC) and
|
|
456
482
|
# early-terminates at LIMIT with no temp B-tree; ``oldest`` scan-sorts the few
|
|
@@ -1106,6 +1132,8 @@ def get_conversation_outline(conn, session_id):
|
|
|
1106
1132
|
cf_count = 0
|
|
1107
1133
|
cf_tokens = 0
|
|
1108
1134
|
cf_wasted = 0.0
|
|
1135
|
+
cf_rebuilds = [] # per-rebuild list (worst-first), spec §1
|
|
1136
|
+
cache_saved = 0.0 # session cache-value-saved, spec §1
|
|
1109
1137
|
for it in items:
|
|
1110
1138
|
kind = it["kind"]
|
|
1111
1139
|
turn_counts["total"] += 1
|
|
@@ -1147,6 +1175,9 @@ def get_conversation_outline(conn, session_id):
|
|
|
1147
1175
|
t["tokens"] = tok
|
|
1148
1176
|
for k in tokens:
|
|
1149
1177
|
tokens[k] += tok.get(k, 0)
|
|
1178
|
+
cr_tokens = tok.get("cache_read", 0) or 0
|
|
1179
|
+
if cr_tokens > 0:
|
|
1180
|
+
cache_saved += _cache_read_saved_usd(it.get("model"), cr_tokens)
|
|
1150
1181
|
# Copy the cache-failure marker onto the OutlineTurn exactly where
|
|
1151
1182
|
# tokens is copied (assistant-only, rides the same source row) and
|
|
1152
1183
|
# accumulate the session-level aggregate (spec §2).
|
|
@@ -1156,6 +1187,13 @@ def get_conversation_outline(conn, session_id):
|
|
|
1156
1187
|
cf_count += 1
|
|
1157
1188
|
cf_tokens += cf.get("tokens_recreated", 0)
|
|
1158
1189
|
cf_wasted += cf.get("est_wasted_usd", 0.0)
|
|
1190
|
+
cf_rebuilds.append({
|
|
1191
|
+
"uuid": t["uuid"],
|
|
1192
|
+
"subagent_key": t["subagent_key"],
|
|
1193
|
+
"ts": t["ts"],
|
|
1194
|
+
"tokens_recreated": cf.get("tokens_recreated", 0),
|
|
1195
|
+
"est_wasted_usd": cf.get("est_wasted_usd", 0.0),
|
|
1196
|
+
})
|
|
1159
1197
|
if kind == "meta":
|
|
1160
1198
|
t["meta_kind"] = it.get("meta_kind")
|
|
1161
1199
|
t["skill_name"] = it.get("skill_name")
|
|
@@ -1171,9 +1209,16 @@ def get_conversation_outline(conn, session_id):
|
|
|
1171
1209
|
"duration_seconds": duration, "tokens": tokens,
|
|
1172
1210
|
"cost_usd": asm["header_cost"]}
|
|
1173
1211
|
if cf_count:
|
|
1174
|
-
stats["cache_failures"] = {
|
|
1175
|
-
|
|
1176
|
-
|
|
1212
|
+
stats["cache_failures"] = {
|
|
1213
|
+
"count": cf_count,
|
|
1214
|
+
"tokens_recreated": cf_tokens,
|
|
1215
|
+
"est_wasted_usd": cf_wasted,
|
|
1216
|
+
"rebuilds": sorted(
|
|
1217
|
+
cf_rebuilds,
|
|
1218
|
+
key=lambda r: (-r["est_wasted_usd"], r["ts"] is None, r["ts"] or ""),
|
|
1219
|
+
),
|
|
1220
|
+
}
|
|
1221
|
+
stats["cache_saved_usd"] = cache_saved
|
|
1177
1222
|
return {"session_id": session_id,
|
|
1178
1223
|
"subagent_meta": asm["subagent_meta"],
|
|
1179
1224
|
"stats": stats,
|