cctally 1.43.2 → 1.44.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/bin/_cctally_cache.py +257 -8
- package/bin/_cctally_dashboard.py +29 -2
- package/bin/_cctally_db.py +59 -0
- package/bin/_lib_conversation.py +49 -0
- package/bin/_lib_conversation_query.py +122 -8
- package/bin/cctally +1 -0
- package/dashboard/static/assets/{index-qxISkiGC.js → index-C_xsUDIl.js} +9 -9
- package/dashboard/static/assets/{index-mPwkGfE9.css → index-Drpkfv6k.css} +1 -1
- package/dashboard/static/dashboard.html +2 -2
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -5,6 +5,21 @@ based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
|
|
|
5
5
|
|
|
6
6
|
## [Unreleased]
|
|
7
7
|
|
|
8
|
+
## [1.44.0] - 2026-06-14
|
|
9
|
+
|
|
10
|
+
### Changed
|
|
11
|
+
- Conversation viewer: the dashboard conversation browse-rail is now served from a `conversation_sessions` rollup (a small per-session COUNT/MIN/MAX table maintained inside `sync_cache` under the cache lock, indexed for the recent ordering) instead of a full-table `GROUP BY` over every message on each page load, so the rail stays fast as your history grows and when a backgrounded tab refocuses after queuing SSE ticks; the rollup is re-derivable like the rest of `cache.db` (a one-time backfill arms automatically on the next sync via cache migration `013`, with a live `GROUP BY` fallback until it lands, so the rail's output is byte-identical either way), and the rail now pauses its per-tick page-1 refetch while the tab is hidden, firing exactly one refetch on the hidden→visible transition so a freshly-revealed reader is current without re-querying every five seconds while idle.
|
|
12
|
+
|
|
13
|
+
## [1.43.3] - 2026-06-14
|
|
14
|
+
|
|
15
|
+
### Added
|
|
16
|
+
- Conversation viewer: each conversation is now titled by Claude Code's AI-generated session title — in the sessions rail and as the reader header — instead of the first prompt, falling back in order to the first prompt, the project label, then the session id; a title that Claude rewrites mid-session updates an already-open reader live (#193).
|
|
17
|
+
- Conversation viewer: a subagent thread is now titled by the description from the `Task` that spawned it, in both the thread-card header and the matching outline landmark (falling back to the subagent's first prompt), and a Bash tool call shows its own description on the dimmed chip line (falling back to the command, which always stays in the expanded `$ …` body); older transcripts and tool calls without a stored description keep their prior rendering (#193).
|
|
18
|
+
|
|
19
|
+
### Fixed
|
|
20
|
+
- Conversation viewer: a short single-line "You" prompt no longer shows an empty gap below its text — the per-message copy/permalink actions (previously an always-reserved but hidden-until-hover row that padded the bottom of the bordered box) now float into the bubble's top-right corner, so the box hugs its prose; assistant turns are unchanged (#192).
|
|
21
|
+
- Conversation viewer: the outline no longer highlights two entries at once when you scroll onto a landmark inside a prompt's section — a subagent card, a section heading, or a plan/question (most visible when a subagent is the last outline entry) — it now marks only that exact landmark instead of also lighting the enclosing "You" prompt, while scrolling onto ordinary prose still highlights its section prompt as before (#192).
|
|
22
|
+
|
|
8
23
|
## [1.43.2] - 2026-06-13
|
|
9
24
|
|
|
10
25
|
### Added
|
package/bin/_cctally_cache.py
CHANGED
|
@@ -189,6 +189,18 @@ _CONV_INSERT_SQL = (
|
|
|
189
189
|
" VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
|
|
190
190
|
)
|
|
191
191
|
|
|
192
|
+
# #193: last non-null write wins (ai-title carries no timestamp; see spec S1). NO
|
|
193
|
+
# byte_offset guard — it can't order a cross-file resumed session. Ordering is
|
|
194
|
+
# made deterministic by ingest order: backfill_ai_titles walks files
|
|
195
|
+
# mtime-ascending so the newest file's last title is written last; the
|
|
196
|
+
# incremental fused walk appends only new bytes in file order.
|
|
197
|
+
_AI_TITLE_UPSERT_SQL = (
|
|
198
|
+
"INSERT INTO conversation_ai_titles(session_id,ai_title,source_path,byte_offset) "
|
|
199
|
+
"VALUES(?,?,?,?) "
|
|
200
|
+
"ON CONFLICT(session_id) DO UPDATE SET "
|
|
201
|
+
"ai_title=excluded.ai_title, source_path=excluded.source_path, byte_offset=excluded.byte_offset"
|
|
202
|
+
)
|
|
203
|
+
|
|
192
204
|
|
|
193
205
|
def _conv_row_tuple(m, path_str):
|
|
194
206
|
"""Flatten a ``MessageRow`` into the ``_CONV_INSERT_SQL`` column order.
|
|
@@ -212,19 +224,23 @@ def _conv_row_tuple(m, path_str):
|
|
|
212
224
|
|
|
213
225
|
def _iter_sync_entries(fh, path_str):
|
|
214
226
|
"""Fused single-pass sync walker (#138). Yields
|
|
215
|
-
``(byte_offset, cost_or_None, msgrow_or_None)`` for each
|
|
216
|
-
``fh``'s current position that produces a cost entry
|
|
217
|
-
message row.
|
|
227
|
+
``(byte_offset, cost_or_None, msgrow_or_None, aititle_or_None)`` for each
|
|
228
|
+
JSONL line from ``fh``'s current position that produces a cost entry, a
|
|
229
|
+
conversation message row, and/or an ai-title record.
|
|
218
230
|
|
|
219
231
|
Each line is read once (readline()+tell()) and ``json.loads``-parsed ONCE,
|
|
220
|
-
then classified by
|
|
232
|
+
then classified by the pure per-line parsers (#138 one-parse-per-line stays
|
|
233
|
+
intact — ``parse_ai_title`` runs on the SAME already-parsed ``obj``):
|
|
221
234
|
|
|
222
235
|
* ``cost_or_None`` is ``(UsageEntry, msg_id, req_id)`` when the line is a
|
|
223
236
|
billable assistant entry (``_lib_jsonl.parse_cost_entry``), else None.
|
|
224
237
|
* ``msgrow_or_None`` is a ``MessageRow`` when the line is a user/assistant
|
|
225
238
|
turn carrying a uuid (``_lib_conversation.parse_message_row``), else None.
|
|
239
|
+
* ``aititle_or_None`` is an ``AiTitleRow`` when the line is an ai-title
|
|
240
|
+
carrying a non-empty sessionId+aiTitle (#193), else None.
|
|
226
241
|
|
|
227
|
-
The
|
|
242
|
+
The three are independent — a normal assistant line yields the first two;
|
|
243
|
+
an ai-title line (a non-user/assistant type) yields only the third. This replaces
|
|
228
244
|
the former cost walk + re-seek-and-walk over the identical byte span: with a
|
|
229
245
|
single walk the "identical span" invariant is structural (one stop point),
|
|
230
246
|
not a prose-enforced ``mrow.byte_offset >= final_offset`` runtime break. A
|
|
@@ -252,8 +268,9 @@ def _iter_sync_entries(fh, path_str):
|
|
|
252
268
|
continue
|
|
253
269
|
cost = _lib_jsonl.parse_cost_entry(obj, path_str)
|
|
254
270
|
mrow = _lib_conversation.parse_message_row(obj, offset)
|
|
255
|
-
|
|
256
|
-
|
|
271
|
+
ai = _lib_conversation.parse_ai_title(obj, offset)
|
|
272
|
+
if cost is not None or mrow is not None or ai is not None:
|
|
273
|
+
yield offset, cost, mrow, ai
|
|
257
274
|
|
|
258
275
|
|
|
259
276
|
def _iter_claude_jsonl_files():
|
|
@@ -616,6 +633,12 @@ def sync_cache(
|
|
|
616
633
|
# under the lock (#138) — NOT a bare DELETE that fires conv_fts_ad
|
|
617
634
|
# per row.
|
|
618
635
|
clear_conversation_messages(conn)
|
|
636
|
+
# #193: ai-titles share the message lifecycle on a rebuild — wipe the
|
|
637
|
+
# table (so a title for a since-deleted session can't linger) and the
|
|
638
|
+
# pending-backfill flag in lockstep. The per-file fused walk below
|
|
639
|
+
# repopulates from offset 0, satisfying any deferred backfill.
|
|
640
|
+
conn.execute("DELETE FROM conversation_ai_titles")
|
|
641
|
+
conn.execute("DELETE FROM cache_meta WHERE key='ai_titles_backfill_pending'")
|
|
619
642
|
# Clear the walk-complete sentinel atomically with the wipe
|
|
620
643
|
# (cctally-dev#93, D5/D2): a stale "complete" marker must never
|
|
621
644
|
# survive a destructive rebuild. The end-of-loop write below
|
|
@@ -691,6 +714,15 @@ def sync_cache(
|
|
|
691
714
|
"DELETE FROM cache_meta WHERE key IN "
|
|
692
715
|
"('conversation_promote_command_args_pending',"
|
|
693
716
|
" 'conversation_promote_command_args_cursor')")
|
|
717
|
+
# Browse-rail rollup: a rebuild re-derives conversation_messages from
|
|
718
|
+
# offset 0, so wipe the rollup here (in the same destructive txn,
|
|
719
|
+
# alongside clear_conversation_messages, so a crash-recovery read
|
|
720
|
+
# can't surface ghost rows) and arm the durable backfill flag. The
|
|
721
|
+
# post-walk recompute (after the per-file loop, still under the
|
|
722
|
+
# flock) consumes the flag and rebuilds the rollup from the freshly
|
|
723
|
+
# re-ingested messages, then drops it last (crash-safe).
|
|
724
|
+
conn.execute("DELETE FROM conversation_sessions")
|
|
725
|
+
_set_cache_meta(conn, "conversation_sessions_backfill_pending", "1")
|
|
694
726
|
conn.commit()
|
|
695
727
|
eprint("[cache-sync] rebuild: cleared Claude cached entries")
|
|
696
728
|
|
|
@@ -724,6 +756,33 @@ def sync_cache(
|
|
|
724
756
|
"DELETE FROM cache_meta "
|
|
725
757
|
"WHERE key='conversation_backfill_pending'"
|
|
726
758
|
)
|
|
759
|
+
# Browse-rail rollup: a #139 offset-0 backfill bulk-inserts
|
|
760
|
+
# history into conversation_messages, so arm the durable
|
|
761
|
+
# recompute flag (idempotent; covers a partial-migration state
|
|
762
|
+
# where the rollup is empty but messages just landed). The
|
|
763
|
+
# post-walk recompute rebuilds it and drops the flag last.
|
|
764
|
+
_set_cache_meta(conn, "conversation_sessions_backfill_pending", "1")
|
|
765
|
+
conn.commit()
|
|
766
|
+
|
|
767
|
+
# #193: consume the deferred ai-title backfill. Cache migration 012 is
|
|
768
|
+
# flag-only (sets ``ai_titles_backfill_pending``); the offset-0 walk
|
|
769
|
+
# over all history via backfill_ai_titles (mtime-ascending,
|
|
770
|
+
# last-write-wins) runs HERE under the held flock — same #139
|
|
771
|
+
# contract as the message backfill above. Touches ONLY
|
|
772
|
+
# conversation_ai_titles; the flag is dropped LAST so a crash mid-walk
|
|
773
|
+
# re-runs cleanly. Never on the rebuild path (which already cleared
|
|
774
|
+
# the flag + repopulates via the normal walk).
|
|
775
|
+
try:
|
|
776
|
+
_ai_pending = conn.execute(
|
|
777
|
+
"SELECT 1 FROM cache_meta WHERE key='ai_titles_backfill_pending'"
|
|
778
|
+
).fetchone() is not None
|
|
779
|
+
except sqlite3.OperationalError:
|
|
780
|
+
_ai_pending = False
|
|
781
|
+
if _ai_pending:
|
|
782
|
+
backfill_ai_titles(conn)
|
|
783
|
+
conn.execute(
|
|
784
|
+
"DELETE FROM cache_meta WHERE key='ai_titles_backfill_pending'"
|
|
785
|
+
)
|
|
727
786
|
conn.commit()
|
|
728
787
|
|
|
729
788
|
# Issue #164: consume the deferred conversation_messages re-ingest.
|
|
@@ -781,6 +840,13 @@ def sync_cache(
|
|
|
781
840
|
# three flags + cursor + gen atomically on completion. Never on the rebuild
|
|
782
841
|
# path (which already wipes + repopulates id-aware via the normal walk).
|
|
783
842
|
_resumable_reingest_conversation_messages(conn)
|
|
843
|
+
# Browse-rail rollup: a #179 reingest DELETEs + re-inserts every
|
|
844
|
+
# file's conversation_messages rows (bumping autoincrement ids and
|
|
845
|
+
# potentially MIN/MAX), so arm the durable recompute flag
|
|
846
|
+
# (idempotent; covers a partial-migration state). The post-walk
|
|
847
|
+
# recompute rebuilds the rollup and drops the flag last.
|
|
848
|
+
_set_cache_meta(conn, "conversation_sessions_backfill_pending", "1")
|
|
849
|
+
conn.commit()
|
|
784
850
|
|
|
785
851
|
# #177 S6: consume the migration-010 search-column split under the
|
|
786
852
|
# SAME held flock, AFTER the reingest so any just-re-ingested rows
|
|
@@ -905,6 +971,11 @@ def sync_cache(
|
|
|
905
971
|
# triggers → truncate → 'delete-all' → recreate, so conv_fts_ad
|
|
906
972
|
# never fires O(rows) inside the held lock.
|
|
907
973
|
clear_conversation_messages(conn)
|
|
974
|
+
# #193: truncation escalates to a full offset-0 re-ingest, so wipe
|
|
975
|
+
# conversation_ai_titles too (parallel to the session_entries +
|
|
976
|
+
# conversation_messages full-reset). The per-file fused walk below
|
|
977
|
+
# repopulates it from offset 0.
|
|
978
|
+
conn.execute("DELETE FROM conversation_ai_titles")
|
|
908
979
|
# Clear the walk-complete sentinel atomically with the truncation
|
|
909
980
|
# full-reset (cctally-dev#93, D5/D2): the cache is being wiped, so
|
|
910
981
|
# any "complete" marker is now stale. The end-of-loop write below
|
|
@@ -926,6 +997,13 @@ def sync_cache(
|
|
|
926
997
|
conn.execute(
|
|
927
998
|
"UPDATE session_files SET size_bytes = 0, last_byte_offset = 0"
|
|
928
999
|
)
|
|
1000
|
+
# Browse-rail rollup: truncation escalates to a full offset-0
|
|
1001
|
+
# re-ingest of conversation_messages, so wipe the rollup here (in the
|
|
1002
|
+
# same destructive txn, alongside clear_conversation_messages) and
|
|
1003
|
+
# arm the durable backfill flag. The post-walk recompute rebuilds it
|
|
1004
|
+
# from the re-ingested messages and drops the flag last (crash-safe).
|
|
1005
|
+
conn.execute("DELETE FROM conversation_sessions")
|
|
1006
|
+
_set_cache_meta(conn, "conversation_sessions_backfill_pending", "1")
|
|
929
1007
|
conn.commit()
|
|
930
1008
|
stats.files_reset_truncated += len(truncated_paths)
|
|
931
1009
|
# Force every file to re-ingest from offset 0: clearing the
|
|
@@ -935,6 +1013,14 @@ def sync_cache(
|
|
|
935
1013
|
# avoids a redundant per-file DELETE that would be a no-op).
|
|
936
1014
|
existing = {}
|
|
937
1015
|
|
|
1016
|
+
# Browse-rail rollup: accumulate the session_ids whose
|
|
1017
|
+
# conversation_messages this walk touched, so the post-walk recompute can
|
|
1018
|
+
# scope its DELETE+INSERT re-derive to just those sessions (steady
|
|
1019
|
+
# state). Pure Python —
|
|
1020
|
+
# updated only AFTER each per-file conn.commit() below, never inside the
|
|
1021
|
+
# zero-write-lock read/parse region, so it adds no DML there.
|
|
1022
|
+
touched_sessions: set = set()
|
|
1023
|
+
|
|
938
1024
|
for jp in paths:
|
|
939
1025
|
path_str = str(jp)
|
|
940
1026
|
# Backfill session_id/project_path for A2 `session` subcommand.
|
|
@@ -976,6 +1062,7 @@ def sync_cache(
|
|
|
976
1062
|
# so a slow JSONL doesn't hold a SQLite lock.
|
|
977
1063
|
rows: list[tuple[Any, ...]] = []
|
|
978
1064
|
conv_rows: list[tuple[Any, ...]] = []
|
|
1065
|
+
ai_rows: list[tuple[Any, ...]] = [] # #193: ai-title upserts
|
|
979
1066
|
final_offset = start_offset
|
|
980
1067
|
try:
|
|
981
1068
|
with open(jp, "r", encoding="utf-8", errors="replace") as fh:
|
|
@@ -987,7 +1074,7 @@ def sync_cache(
|
|
|
987
1074
|
# walk over the identical span — the "identical span"
|
|
988
1075
|
# invariant is now structural (a single stop point) rather
|
|
989
1076
|
# than a prose-enforced ``>= final_offset`` runtime break.
|
|
990
|
-
for offset, cost, mrow in _iter_sync_entries(fh, path_str):
|
|
1077
|
+
for offset, cost, mrow, ai in _iter_sync_entries(fh, path_str):
|
|
991
1078
|
if cost is not None:
|
|
992
1079
|
entry, msg_id, req_id = cost
|
|
993
1080
|
usage = entry.usage
|
|
@@ -1015,6 +1102,11 @@ def sync_cache(
|
|
|
1015
1102
|
))
|
|
1016
1103
|
if mrow is not None:
|
|
1017
1104
|
conv_rows.append(_conv_row_tuple(mrow, path_str))
|
|
1105
|
+
if ai is not None:
|
|
1106
|
+
# #193: accumulate ai-title upserts in file order; the
|
|
1107
|
+
# executemany below applies them after conv_rows.
|
|
1108
|
+
ai_rows.append((ai.session_id, ai.ai_title,
|
|
1109
|
+
path_str, ai.byte_offset))
|
|
1018
1110
|
# ``final_offset`` is the single walk's stop — captured AFTER
|
|
1019
1111
|
# the loop drains (or rewinds a partial mid-write tail line).
|
|
1020
1112
|
# It is what session_files.last_byte_offset is written from,
|
|
@@ -1115,6 +1207,10 @@ def sync_cache(
|
|
|
1115
1207
|
# (parallel to the cost path's lifecycle).
|
|
1116
1208
|
if conv_rows:
|
|
1117
1209
|
conn.executemany(_CONV_INSERT_SQL, conv_rows)
|
|
1210
|
+
# #193: ai-title upserts for this file, in file order (last wins).
|
|
1211
|
+
# Committed atomically with the session_files cursor below.
|
|
1212
|
+
if ai_rows:
|
|
1213
|
+
conn.executemany(_AI_TITLE_UPSERT_SQL, ai_rows)
|
|
1118
1214
|
# UPSERT preserves session_id / project_path columns populated
|
|
1119
1215
|
# by _ensure_session_files_row at the top of this loop. A plain
|
|
1120
1216
|
# INSERT OR REPLACE would wipe them on every changed-file sync.
|
|
@@ -1134,6 +1230,12 @@ def sync_cache(
|
|
|
1134
1230
|
)
|
|
1135
1231
|
conn.commit()
|
|
1136
1232
|
stats.files_processed += 1
|
|
1233
|
+
# Browse-rail rollup: record the session_ids this file just
|
|
1234
|
+
# committed so the post-walk recompute can scope its DELETE+INSERT
|
|
1235
|
+
# re-derive to them. cr[0] is session_id per _conv_row_tuple's
|
|
1236
|
+
# column order. Lands
|
|
1237
|
+
# AFTER the commit (pure Python; no DML, no extra write lock).
|
|
1238
|
+
touched_sessions.update(cr[0] for cr in conv_rows if cr[0] is not None)
|
|
1137
1239
|
except sqlite3.DatabaseError as exc:
|
|
1138
1240
|
eprint(f"[cache] db error on {jp}: {exc}")
|
|
1139
1241
|
conn.rollback()
|
|
@@ -1146,6 +1248,31 @@ def sync_cache(
|
|
|
1146
1248
|
if progress is not None:
|
|
1147
1249
|
progress(stats)
|
|
1148
1250
|
|
|
1251
|
+
# Browse-rail rollup maintenance (single post-walk recompute, under the
|
|
1252
|
+
# still-held flock, after every per-file commit and before the
|
|
1253
|
+
# walk-complete marker). Keyed on the DURABLE flag, not an in-memory
|
|
1254
|
+
# bool: a crash between a destructive path's commit (rebuild /
|
|
1255
|
+
# truncation / #139 backfill / #179 reingest, each of which armed the
|
|
1256
|
+
# flag in its own committed txn) and this recompute leaves the flag set,
|
|
1257
|
+
# so the next sync full-recomputes — never strands stale rollup rows
|
|
1258
|
+
# (Codex gate BLOCKER 1). Flag set -> full GROUP BY over all sessions
|
|
1259
|
+
# (rare, ~90ms), then drop the flag LAST (drop-it-last contract). Else ->
|
|
1260
|
+
# scoped re-derive (DELETE+INSERT, not a SQL UPSERT) over just the
|
|
1261
|
+
# sessions this walk touched (steady state,
|
|
1262
|
+
# ~1 session/tick). Both recomputes derive COUNT/MIN/MAX from the same
|
|
1263
|
+
# rows the rail's old live aggregate read, so the rollup stays
|
|
1264
|
+
# byte-identical to that aggregate.
|
|
1265
|
+
if _conversation_sessions_backfill_pending(conn):
|
|
1266
|
+
_recompute_conversation_sessions(conn)
|
|
1267
|
+
conn.execute(
|
|
1268
|
+
"DELETE FROM cache_meta "
|
|
1269
|
+
"WHERE key='conversation_sessions_backfill_pending'"
|
|
1270
|
+
)
|
|
1271
|
+
conn.commit()
|
|
1272
|
+
elif touched_sessions:
|
|
1273
|
+
_recompute_conversation_sessions(conn, touched_sessions)
|
|
1274
|
+
conn.commit()
|
|
1275
|
+
|
|
1149
1276
|
# Walk-complete sentinel write (cctally-dev#93, D5a). Still inside the
|
|
1150
1277
|
# held fcntl lock, before the finally-unlock. Only when the entire walk
|
|
1151
1278
|
# was clean AND cache 001 was already applied at the start of this run
|
|
@@ -1226,6 +1353,46 @@ def backfill_conversation_messages(conn: sqlite3.Connection) -> int:
|
|
|
1226
1353
|
return inserted
|
|
1227
1354
|
|
|
1228
1355
|
|
|
1356
|
+
def backfill_ai_titles(conn: sqlite3.Connection) -> int:
|
|
1357
|
+
"""One-time backfill of ``conversation_ai_titles`` for existing installs
|
|
1358
|
+
(#193). Walks EVERY Claude JSONL from offset 0 via
|
|
1359
|
+
``_lib_conversation.iter_ai_titles`` and upserts.
|
|
1360
|
+
|
|
1361
|
+
Files are walked MTIME-ASCENDING so that, for a session whose ai-title spans
|
|
1362
|
+
multiple files (a ``--resume``), the most-recently-modified file's last
|
|
1363
|
+
non-null title is written last (last-write-wins; see _AI_TITLE_UPSERT_SQL).
|
|
1364
|
+
Per-file commit; the caller (``sync_cache``, consuming the
|
|
1365
|
+
``ai_titles_backfill_pending`` flag) holds the ``cache.db.lock`` flock for the
|
|
1366
|
+
duration. Touches ONLY ``conversation_ai_titles`` — the cost/message cursors
|
|
1367
|
+
are untouched. Idempotent: a re-run rewrites the same current title (the
|
|
1368
|
+
last-write-wins ordering is stable under the deterministic mtime walk).
|
|
1369
|
+
Returns rows upserted."""
|
|
1370
|
+
n = 0
|
|
1371
|
+
|
|
1372
|
+
def _mtime(p):
|
|
1373
|
+
try:
|
|
1374
|
+
return p.stat().st_mtime
|
|
1375
|
+
except OSError:
|
|
1376
|
+
return 0.0 # vanished mid-walk; sorts first, the open() below skips it
|
|
1377
|
+
|
|
1378
|
+
files = sorted(_iter_claude_jsonl_files(), key=_mtime)
|
|
1379
|
+
for jp in files:
|
|
1380
|
+
path_str = str(jp)
|
|
1381
|
+
rows: list[tuple[Any, ...]] = []
|
|
1382
|
+
try:
|
|
1383
|
+
with open(jp, "r", encoding="utf-8", errors="replace") as fh:
|
|
1384
|
+
for r in _lib_conversation.iter_ai_titles(fh, path_str):
|
|
1385
|
+
rows.append((r.session_id, r.ai_title, path_str, r.byte_offset))
|
|
1386
|
+
except OSError as exc:
|
|
1387
|
+
eprint(f"[ai-title-backfill] could not read {jp}: {exc}")
|
|
1388
|
+
continue
|
|
1389
|
+
if rows:
|
|
1390
|
+
conn.executemany(_AI_TITLE_UPSERT_SQL, rows)
|
|
1391
|
+
n += len(rows)
|
|
1392
|
+
conn.commit()
|
|
1393
|
+
return n
|
|
1394
|
+
|
|
1395
|
+
|
|
1229
1396
|
_REINGEST_FLAG_KEYS = (
|
|
1230
1397
|
"conversation_reingest_pending",
|
|
1231
1398
|
"conversation_source_tool_use_reingest_pending",
|
|
@@ -1317,6 +1484,88 @@ def _resumable_reingest_conversation_messages(conn):
|
|
|
1317
1484
|
conn.commit()
|
|
1318
1485
|
|
|
1319
1486
|
|
|
1487
|
+
# === Browse-rail rollup (conversation_sessions) maintenance =================
|
|
1488
|
+
# Keeps conversation_sessions — the four structural aggregates the old live
|
|
1489
|
+
# GROUP BY produced — in lockstep with conversation_messages so
|
|
1490
|
+
# GET /api/conversations renders a page without scanning the whole message
|
|
1491
|
+
# table. Maintained entirely inside sync_cache under the cache.db.lock flock:
|
|
1492
|
+
# the steady-state per-file loop is insert-only, so a scoped re-derive
|
|
1493
|
+
# (DELETE+INSERT, not a SQL UPSERT) over the touched sessions suffices; the
|
|
1494
|
+
# rare heavy/destructive paths (rebuild,
|
|
1495
|
+
# truncation-escalation, #139 backfill, #179 reingest) set the durable
|
|
1496
|
+
# ``conversation_sessions_backfill_pending`` cache_meta flag (migration 013
|
|
1497
|
+
# arms it too) which forces one full recompute, crash-safe across a
|
|
1498
|
+
# destructive-commit/recompute crash window. The CALLER owns the commit.
|
|
1499
|
+
|
|
1500
|
+
# All non-null sessions, recomputed from conversation_messages. Shared by the
|
|
1501
|
+
# full and scoped recompute so both paths derive byte-identical aggregates.
|
|
1502
|
+
_CONV_SESSIONS_SELECT = (
|
|
1503
|
+
"SELECT session_id, COUNT(*), MIN(timestamp_utc), MAX(timestamp_utc) "
|
|
1504
|
+
"FROM conversation_messages WHERE session_id IS NOT NULL"
|
|
1505
|
+
)
|
|
1506
|
+
|
|
1507
|
+
|
|
1508
|
+
def _conversation_sessions_backfill_pending(conn) -> bool:
|
|
1509
|
+
"""True while the durable ``conversation_sessions_backfill_pending`` flag is
|
|
1510
|
+
set — the signal that the rollup needs one full GROUP BY recompute (armed by
|
|
1511
|
+
migration 013 and by every heavy/destructive conversation_messages path).
|
|
1512
|
+
Tolerates a missing cache_meta table (path-less / schema-not-applied conn) by
|
|
1513
|
+
degrading to False, like the sibling reingest/backfill predicates."""
|
|
1514
|
+
try:
|
|
1515
|
+
return conn.execute(
|
|
1516
|
+
"SELECT 1 FROM cache_meta "
|
|
1517
|
+
"WHERE key='conversation_sessions_backfill_pending'"
|
|
1518
|
+
).fetchone() is not None
|
|
1519
|
+
except sqlite3.OperationalError:
|
|
1520
|
+
return False
|
|
1521
|
+
|
|
1522
|
+
|
|
1523
|
+
def _recompute_conversation_sessions(conn, session_ids=None) -> None:
|
|
1524
|
+
"""Recompute the ``conversation_sessions`` browse-rail rollup from
|
|
1525
|
+
``conversation_messages``. The caller holds the cache.db.lock flock and owns
|
|
1526
|
+
the commit (this helper never commits).
|
|
1527
|
+
|
|
1528
|
+
``session_ids is None`` -> FULL: wipe the whole rollup and rebuild it from a
|
|
1529
|
+
single GROUP BY over every non-null session — the rare, flag-gated path
|
|
1530
|
+
(rebuild / truncation / backfill / reingest / migration-013 history).
|
|
1531
|
+
|
|
1532
|
+
``session_ids={...}`` -> SCOPED: for each <=400-id chunk, DELETE those rows
|
|
1533
|
+
then re-INSERT the GROUP BY restricted to the chunk — the steady-state path
|
|
1534
|
+
keyed on the per-file loop's touched set. DELETE+INSERT (NOT
|
|
1535
|
+
INSERT…SELECT…ON CONFLICT, which trips SQLite's upsert-on-SELECT parse
|
|
1536
|
+
ambiguity) also correctly drops a session whose rows all vanished — though in
|
|
1537
|
+
steady state conversation_messages only gains rows, so that branch is just
|
|
1538
|
+
belt-and-suspenders. The chunking keeps the ``session_id IN (…)`` parameter
|
|
1539
|
+
list well under SQLite's variable limit.
|
|
1540
|
+
|
|
1541
|
+
The recomputed COUNT/MIN/MAX are byte-identical to the rail's prior live
|
|
1542
|
+
aggregate over the same rows — that is the load-bearing invariant
|
|
1543
|
+
(assert_rollup_matches_live in the maintenance test pins it)."""
|
|
1544
|
+
if session_ids is None:
|
|
1545
|
+
conn.execute("DELETE FROM conversation_sessions")
|
|
1546
|
+
conn.execute(
|
|
1547
|
+
"INSERT INTO conversation_sessions "
|
|
1548
|
+
"(session_id, msg_count, started_utc, last_activity_utc) "
|
|
1549
|
+
+ _CONV_SESSIONS_SELECT + " GROUP BY session_id"
|
|
1550
|
+
)
|
|
1551
|
+
return
|
|
1552
|
+
ids = [s for s in session_ids if s is not None]
|
|
1553
|
+
for i in range(0, len(ids), 400):
|
|
1554
|
+
chunk = ids[i:i + 400]
|
|
1555
|
+
placeholders = ",".join("?" for _ in chunk)
|
|
1556
|
+
conn.execute(
|
|
1557
|
+
f"DELETE FROM conversation_sessions WHERE session_id IN ({placeholders})",
|
|
1558
|
+
chunk,
|
|
1559
|
+
)
|
|
1560
|
+
conn.execute(
|
|
1561
|
+
"INSERT INTO conversation_sessions "
|
|
1562
|
+
"(session_id, msg_count, started_utc, last_activity_utc) "
|
|
1563
|
+
+ _CONV_SESSIONS_SELECT
|
|
1564
|
+
+ f" AND session_id IN ({placeholders}) GROUP BY session_id",
|
|
1565
|
+
chunk,
|
|
1566
|
+
)
|
|
1567
|
+
|
|
1568
|
+
|
|
1320
1569
|
def _consume_search_split(conn) -> None:
|
|
1321
1570
|
"""#177 S6: flock-held consumer for ``conversation_search_split_pending``
|
|
1322
1571
|
(set by cache migration 010). Cursor-resumable: backfills
|
|
@@ -251,6 +251,32 @@ from typing import Any
|
|
|
251
251
|
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
|
|
252
252
|
|
|
253
253
|
|
|
254
|
+
class _QuietThreadingHTTPServer(ThreadingHTTPServer):
|
|
255
|
+
"""`ThreadingHTTPServer` that swallows client-disconnect tracebacks.
|
|
256
|
+
|
|
257
|
+
A backgrounded/closed/reloaded dashboard tab hangs up mid-response, so the
|
|
258
|
+
per-request thread's socket write raises one of the "peer went away"
|
|
259
|
+
exceptions, which `socketserver` routes through `handle_error`. On a local
|
|
260
|
+
dashboard that is expected and benign — dumping a socket-write stack trace
|
|
261
|
+
to the user's console for every such disconnect is pure noise (the original
|
|
262
|
+
report was repeated `BrokenPipeError` spam after hours idle). Everything
|
|
263
|
+
else still gets the full traceback via `super().handle_error`.
|
|
264
|
+
|
|
265
|
+
`daemon_threads = True` is folded in here (was an inline `srv.*` assignment):
|
|
266
|
+
SSE handler threads may block up to 15s on the keep-alive timeout — let them
|
|
267
|
+
die with the process.
|
|
268
|
+
"""
|
|
269
|
+
|
|
270
|
+
daemon_threads = True
|
|
271
|
+
|
|
272
|
+
def handle_error(self, request, client_address):
|
|
273
|
+
exc = sys.exc_info()[1]
|
|
274
|
+
if isinstance(exc, (BrokenPipeError, ConnectionResetError, ConnectionAbortedError)):
|
|
275
|
+
# Client hung up mid-response; benign on a local dashboard.
|
|
276
|
+
return
|
|
277
|
+
super().handle_error(request, client_address)
|
|
278
|
+
|
|
279
|
+
|
|
254
280
|
def _cctally():
|
|
255
281
|
"""Resolve the current ``cctally`` module at call-time (spec §5.5)."""
|
|
256
282
|
return sys.modules["cctally"]
|
|
@@ -8251,8 +8277,9 @@ def cmd_dashboard(args: argparse.Namespace) -> int:
|
|
|
8251
8277
|
update_check_thread.start()
|
|
8252
8278
|
|
|
8253
8279
|
# HTTP server on its own thread so the main thread can block on signal.
|
|
8254
|
-
|
|
8255
|
-
|
|
8280
|
+
# `_QuietThreadingHTTPServer` folds in `daemon_threads = True` and silences
|
|
8281
|
+
# client-disconnect tracebacks (spec §5).
|
|
8282
|
+
srv = _QuietThreadingHTTPServer((args.host, args.port), DashboardHTTPHandler)
|
|
8256
8283
|
|
|
8257
8284
|
bind_host = args.host
|
|
8258
8285
|
bind_port = srv.server_address[1]
|
package/bin/_cctally_db.py
CHANGED
|
@@ -2349,6 +2349,38 @@ def _apply_cache_schema(conn: sqlite3.Connection) -> None:
|
|
|
2349
2349
|
CREATE INDEX IF NOT EXISTS idx_conv_turnkey
|
|
2350
2350
|
ON conversation_messages(msg_id, req_id);
|
|
2351
2351
|
|
|
2352
|
+
-- #193: per-session AI-generated title, isolated from the six places
|
|
2353
|
+
-- that iterate conversation_messages. The explicit NOT NULL on the
|
|
2354
|
+
-- non-INTEGER PRIMARY KEY matters (SQLite's legacy NULL-in-PK bug);
|
|
2355
|
+
-- _session_titles_map can only key on a concrete session_id.
|
|
2356
|
+
CREATE TABLE IF NOT EXISTS conversation_ai_titles (
|
|
2357
|
+
session_id TEXT NOT NULL PRIMARY KEY,
|
|
2358
|
+
ai_title TEXT NOT NULL,
|
|
2359
|
+
source_path TEXT,
|
|
2360
|
+
byte_offset INTEGER NOT NULL
|
|
2361
|
+
);
|
|
2362
|
+
|
|
2363
|
+
-- Browse-rail rollup (conversation_sessions). Materializes exactly the
|
|
2364
|
+
-- four structural aggregates the rail's old live GROUP BY produced
|
|
2365
|
+
-- (COUNT/MIN/MAX over conversation_messages per session_id) so
|
|
2366
|
+
-- GET /api/conversations no longer scans the whole message table to
|
|
2367
|
+
-- render a 50-row page. The explicit NOT NULL on the non-INTEGER PK
|
|
2368
|
+
-- matters (SQLite's legacy NULL-in-PK bug); the rail keys on a concrete
|
|
2369
|
+
-- session_id and the recompute's GROUP BY already filters nulls. The
|
|
2370
|
+
-- index lets the only paginated ordering (recent) early-terminate at
|
|
2371
|
+
-- LIMIT with no temp B-tree. Re-derivable like the rest of cache.db;
|
|
2372
|
+
-- sync_cache keeps it honest (scoped DELETE+INSERT re-derive +
|
|
2373
|
+
-- flag-gated full recompute) — migration 013 arms the one-time
|
|
2374
|
+
-- history backfill.
|
|
2375
|
+
CREATE TABLE IF NOT EXISTS conversation_sessions (
|
|
2376
|
+
session_id TEXT NOT NULL PRIMARY KEY,
|
|
2377
|
+
msg_count INTEGER NOT NULL DEFAULT 0,
|
|
2378
|
+
started_utc TEXT,
|
|
2379
|
+
last_activity_utc TEXT
|
|
2380
|
+
);
|
|
2381
|
+
CREATE INDEX IF NOT EXISTS idx_conv_sessions_recent
|
|
2382
|
+
ON conversation_sessions(last_activity_utc DESC, session_id DESC);
|
|
2383
|
+
|
|
2352
2384
|
CREATE TABLE IF NOT EXISTS codex_session_files (
|
|
2353
2385
|
path TEXT PRIMARY KEY,
|
|
2354
2386
|
size_bytes INTEGER NOT NULL,
|
|
@@ -3383,6 +3415,33 @@ def _011_conversation_promote_command_args(conn: sqlite3.Connection) -> None:
|
|
|
3383
3415
|
conn.commit()
|
|
3384
3416
|
|
|
3385
3417
|
|
|
3418
|
+
@cache_migration("012_create_conversation_ai_titles")
|
|
3419
|
+
def _012_create_conversation_ai_titles(conn: sqlite3.Connection) -> None:
|
|
3420
|
+
"""Flag-only arm for #193. The conversation_ai_titles table itself is created
|
|
3421
|
+
by _apply_cache_schema (runs on every open, fresh + existing installs); this
|
|
3422
|
+
migration sets ``ai_titles_backfill_pending`` so sync_cache walks all history
|
|
3423
|
+
once via backfill_ai_titles under the cache.db.lock flock. No data work here
|
|
3424
|
+
-> the dispatcher's central stamp (#140) marks a complete handler; a fresh
|
|
3425
|
+
install stamps WITHOUT a populated history (its incremental walk fills the
|
|
3426
|
+
table as it ingests, and the consumed backfill no-ops). Mirrors 002/010/011."""
|
|
3427
|
+
_set_cache_meta(conn, "ai_titles_backfill_pending", "1")
|
|
3428
|
+
conn.commit()
|
|
3429
|
+
|
|
3430
|
+
|
|
3431
|
+
@cache_migration("013_create_conversation_sessions")
|
|
3432
|
+
def _013_create_conversation_sessions(conn: sqlite3.Connection) -> None:
|
|
3433
|
+
"""Flag-only arm for the conversation_sessions browse-rail rollup. The table
|
|
3434
|
+
is created by _apply_cache_schema (every open); this sets
|
|
3435
|
+
conversation_sessions_backfill_pending so sync_cache does the one-time full
|
|
3436
|
+
GROUP BY recompute under the cache.db.lock flock. No data work here — the
|
|
3437
|
+
dispatcher's central stamp (#140) marks a complete handler; a fresh install
|
|
3438
|
+
stamps WITHOUT running the handler, so the flag is NOT set there, which is
|
|
3439
|
+
correct (empty messages -> empty rollup; the incremental DELETE+INSERT
|
|
3440
|
+
re-derive fills both in lockstep). Mirrors 012."""
|
|
3441
|
+
_set_cache_meta(conn, "conversation_sessions_backfill_pending", "1")
|
|
3442
|
+
conn.commit()
|
|
3443
|
+
|
|
3444
|
+
|
|
3386
3445
|
# === Region 7d: Stats migration 008_recompute_weekly_cost_snapshots_dedup_fix ===
|
|
3387
3446
|
|
|
3388
3447
|
@stats_migration("008_recompute_weekly_cost_snapshots_dedup_fix")
|
package/bin/_lib_conversation.py
CHANGED
|
@@ -263,6 +263,55 @@ def parse_message_row(obj, offset):
|
|
|
263
263
|
return _normalize(obj, t, offset)
|
|
264
264
|
|
|
265
265
|
|
|
266
|
+
@dataclass
|
|
267
|
+
class AiTitleRow:
|
|
268
|
+
"""Pure per-line AI-title record (no I/O). Parallels MessageRow but for the
|
|
269
|
+
main-session ``{"type":"ai-title","aiTitle":...,"sessionId":...}`` lines that
|
|
270
|
+
parse_message_row drops (type not in user/assistant). #193."""
|
|
271
|
+
session_id: "str | None"
|
|
272
|
+
ai_title: str
|
|
273
|
+
byte_offset: int
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def parse_ai_title(obj, offset):
|
|
277
|
+
"""Return an AiTitleRow when ``obj`` is an ai-title line with BOTH a non-empty
|
|
278
|
+
string sessionId and a non-empty string aiTitle, else None. Skips the null /
|
|
279
|
+
blank rewrites CC emits as the title evolves, and any malformed line. #193."""
|
|
280
|
+
if obj.get("type") != "ai-title":
|
|
281
|
+
return None
|
|
282
|
+
sid = obj.get("sessionId")
|
|
283
|
+
title = obj.get("aiTitle")
|
|
284
|
+
if not (isinstance(sid, str) and sid.strip()):
|
|
285
|
+
return None
|
|
286
|
+
if not (isinstance(title, str) and title.strip()):
|
|
287
|
+
return None
|
|
288
|
+
return AiTitleRow(sid, title, offset)
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
def iter_ai_titles(fh, path_str):
|
|
292
|
+
"""Yield AiTitleRow for each ai-title line from ``fh``'s current position.
|
|
293
|
+
Mirrors iter_message_rows: caller owns the open file handle; pure parse.
|
|
294
|
+
``path_str`` is accepted for signature-parity (ai-title rows don't use it)."""
|
|
295
|
+
while True:
|
|
296
|
+
offset = fh.tell()
|
|
297
|
+
line = fh.readline()
|
|
298
|
+
if not line:
|
|
299
|
+
return
|
|
300
|
+
if not line.endswith("\n"):
|
|
301
|
+
fh.seek(offset)
|
|
302
|
+
return
|
|
303
|
+
s = line.strip()
|
|
304
|
+
if not s:
|
|
305
|
+
continue
|
|
306
|
+
try:
|
|
307
|
+
obj = json.loads(s)
|
|
308
|
+
except json.JSONDecodeError:
|
|
309
|
+
continue
|
|
310
|
+
row = parse_ai_title(obj, offset)
|
|
311
|
+
if row is not None:
|
|
312
|
+
yield row
|
|
313
|
+
|
|
314
|
+
|
|
266
315
|
def _normalize(obj, t, offset):
|
|
267
316
|
msg = obj.get("message")
|
|
268
317
|
if not isinstance(msg, dict):
|