cctally 1.39.0 → 1.41.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/bin/_cctally_cache.py +103 -6
- package/bin/_cctally_dashboard.py +85 -3
- package/bin/_cctally_db.py +179 -45
- package/bin/_lib_conversation.py +75 -21
- package/bin/_lib_conversation_query.py +513 -40
- package/dashboard/static/assets/{index-JAqj_fCI.css → index-BKlFTF9C.css} +1 -1
- package/dashboard/static/assets/index-DZ_higAv.js +68 -0
- package/dashboard/static/dashboard.html +2 -2
- package/package.json +1 -1
- package/dashboard/static/assets/index-E2kVc-MU.js +0 -67
package/CHANGELOG.md
CHANGED
|
@@ -5,6 +5,17 @@ based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
|
|
|
5
5
|
|
|
6
6
|
## [Unreleased]
|
|
7
7
|
|
|
8
|
+
## [1.41.0] - 2026-06-13
|
|
9
|
+
|
|
10
|
+
### Added
|
|
11
|
+
- Conversation viewer: search now reaches past prose into commands, file paths, error strings, and the assistant's thinking, with exact kind facets and a browser-style in-conversation find bar — the rail gains an `All · Prompts · Assistant · Tools · Thinking` chip row, a result count line, match badges (`tool`, `thinking`), and a `Load N more` button, while pressing `/` over an open conversation opens a floating find pill (`Enter`/`Shift+Enter` and `n`/`N` step matches, `Esc` closes, wrap-around, exact `k / N` counter, `<mark>` term highlighting in prose) backed by a new `GET /api/conversation/<id>/find` endpoint and a `kind` param on `/api/conversation/search`; the tool/thinking facets light up after a one-time, lossless search-index split (cache migration `010`) runs automatically under the cache lock on the next sync, reporting `search_depth: "prose-only"` (with the Tools/Thinking chips disabled and an `indexing…` hint) until it completes, and degrading to a basic substring scan on hosts without FTS5 (#177 S6).
|
|
12
|
+
|
|
13
|
+
## [1.40.0] - 2026-06-12
|
|
14
|
+
|
|
15
|
+
### Added
|
|
16
|
+
- Conversation viewer: a full-session navigation layer — a collapsible outline sidebar (toggle `o`) listing every turn as a landmark with nested thinking entries and scroll-sync highlighting, a stats overview (turn counts, duration, tokens, cost, models, tool histogram, and an error row that jumps to the first error), jump-to-next keys with no wrap-around (`e`/`E` errors, `u`/`U` prompts, `b`/`B` subagents, `p`/`P` plans/questions) mirrored by a clickable glyph cluster with counts, and focus modes (`v` cycles All → Chat → Prompts → Errors; hidden turns coalesce into a clickable `· N hidden ·` marker, and a jump whose target the mode would hide resets to All before landing) (#177 Session 5).
|
|
17
|
+
- Conversation viewer: per-turn timestamps and inter-turn markers — each turn header shows a quiet `· HH:mm` in your `display.tz` (precise instant in a tooltip), with a `⏸ 42 min later` gap rule between turns ten or more minutes apart, a `— Jun 13 —` rule when the calendar day changes, and a combined `⏸ 9.5 h later · Jun 13` when both apply — plus per-turn token footers that extend the assistant cost line to `$0.0214 · in 1.2k · out 4.8k · cache 310k` (tokens-only when a turn has usage but no attributable cost, cost-only for un-reingested turns) (#177 Session 5).
|
|
18
|
+
|
|
8
19
|
## [1.39.0] - 2026-06-12
|
|
9
20
|
|
|
10
21
|
### Added
|
package/bin/_cctally_cache.py
CHANGED
|
@@ -184,8 +184,9 @@ _CONV_INSERT_SQL = (
|
|
|
184
184
|
"(session_id,uuid,parent_uuid,source_path,byte_offset,"
|
|
185
185
|
" timestamp_utc,entry_type,text,blocks_json,model,msg_id,"
|
|
186
186
|
" req_id,cwd,git_branch,is_sidechain,source_tool_use_id,"
|
|
187
|
-
" stop_reason,attribution_skill,attribution_plugin,
|
|
188
|
-
"
|
|
187
|
+
" stop_reason,attribution_skill,attribution_plugin,"
|
|
188
|
+
" search_tool,search_thinking,search_aux)"
|
|
189
|
+
" VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
|
|
189
190
|
)
|
|
190
191
|
|
|
191
192
|
|
|
@@ -193,15 +194,19 @@ def _conv_row_tuple(m, path_str):
|
|
|
193
194
|
"""Flatten a ``MessageRow`` into the ``_CONV_INSERT_SQL`` column order.
|
|
194
195
|
|
|
195
196
|
The #177 enrichment fields (stop_reason / attribution_skill /
|
|
196
|
-
attribution_plugin / search_aux) are
|
|
197
|
-
— same order as the SQL column list —
|
|
198
|
-
|
|
197
|
+
attribution_plugin / search_tool / search_thinking / search_aux) are
|
|
198
|
+
TAIL-APPENDED after source_tool_use_id — same order as the SQL column list —
|
|
199
|
+
so both ingest paths (fused per-file walk + backfill_conversation_messages)
|
|
200
|
+
carry them through this one tuple. #177 S6: ``search_aux`` is always written
|
|
201
|
+
as ``''`` (documented-dead); the split ``search_tool``/``search_thinking``
|
|
202
|
+
columns carry the non-prose index."""
|
|
199
203
|
return (
|
|
200
204
|
m.session_id, m.uuid, m.parent_uuid, path_str, m.byte_offset,
|
|
201
205
|
m.timestamp_utc, m.entry_type, m.text, m.blocks_json, m.model,
|
|
202
206
|
m.msg_id, m.req_id, m.cwd, m.git_branch, m.is_sidechain,
|
|
203
207
|
m.source_tool_use_id,
|
|
204
|
-
m.stop_reason, m.attribution_skill, m.attribution_plugin,
|
|
208
|
+
m.stop_reason, m.attribution_skill, m.attribution_plugin,
|
|
209
|
+
m.search_tool, m.search_thinking, m.search_aux,
|
|
205
210
|
)
|
|
206
211
|
|
|
207
212
|
|
|
@@ -651,6 +656,31 @@ def sync_cache(
|
|
|
651
656
|
" 'conversation_media_reingest_pending',"
|
|
652
657
|
" 'conversation_reingest_cursor',"
|
|
653
658
|
" 'conversation_reingest_cursor_gen')")
|
|
659
|
+
# #177 S6: a rebuild repopulates search_tool/search_thinking via the
|
|
660
|
+
# offset-0 walk (the parser derives them), so the migration-010
|
|
661
|
+
# backfill is redundant. But a LEGACY-shape DB still carries the old
|
|
662
|
+
# prose+aux FTS pair that the split triggers can't write — swap to the
|
|
663
|
+
# split shape NOW (the table is empty post-clear, so the walk below
|
|
664
|
+
# populates it through the new triggers), then drop the pending flag +
|
|
665
|
+
# cursor so the post-rebuild sync runs no redundant backfill/swap.
|
|
666
|
+
# MISSING this site re-arms the flag on every cache-sync --rebuild.
|
|
667
|
+
try:
|
|
668
|
+
_split_pending = conn.execute(
|
|
669
|
+
"SELECT 1 FROM cache_meta "
|
|
670
|
+
"WHERE key='conversation_search_split_pending'"
|
|
671
|
+
).fetchone() is not None
|
|
672
|
+
except sqlite3.OperationalError:
|
|
673
|
+
_split_pending = False
|
|
674
|
+
if _split_pending:
|
|
675
|
+
_fts_off = conn.execute(
|
|
676
|
+
"SELECT 1 FROM cache_meta WHERE key='fts5_unavailable'"
|
|
677
|
+
).fetchone() is not None
|
|
678
|
+
if not _fts_off and not _cctally_db_sib._conversation_fts_is_split(conn):
|
|
679
|
+
_cctally_db_sib._swap_conversation_fts_to_split(conn)
|
|
680
|
+
conn.execute(
|
|
681
|
+
"DELETE FROM cache_meta WHERE key IN "
|
|
682
|
+
"('conversation_search_split_pending',"
|
|
683
|
+
" 'conversation_search_split_cursor')")
|
|
654
684
|
conn.commit()
|
|
655
685
|
eprint("[cache-sync] rebuild: cleared Claude cached entries")
|
|
656
686
|
|
|
@@ -742,6 +772,14 @@ def sync_cache(
|
|
|
742
772
|
# path (which already wipes + repopulates id-aware via the normal walk).
|
|
743
773
|
_resumable_reingest_conversation_messages(conn)
|
|
744
774
|
|
|
775
|
+
# #177 S6: consume the migration-010 search-column split under the
|
|
776
|
+
# SAME held flock, AFTER the reingest so any just-re-ingested rows
|
|
777
|
+
# already carry fresh search_tool/search_thinking before the backfill
|
|
778
|
+
# touches the tail. Cursor-resumable; the legacy triggers are blind to
|
|
779
|
+
# the search_tool/search_thinking UPDATEs (they fire on text only), so
|
|
780
|
+
# old search keeps working until the final swap.
|
|
781
|
+
_consume_search_split(conn)
|
|
782
|
+
|
|
745
783
|
paths: list[pathlib.Path] = list(_iter_claude_jsonl_files())
|
|
746
784
|
stats.files_total = len(paths)
|
|
747
785
|
|
|
@@ -1260,6 +1298,65 @@ def _resumable_reingest_conversation_messages(conn):
|
|
|
1260
1298
|
conn.commit()
|
|
1261
1299
|
|
|
1262
1300
|
|
|
1301
|
+
def _consume_search_split(conn) -> None:
|
|
1302
|
+
"""#177 S6: flock-held consumer for ``conversation_search_split_pending``
|
|
1303
|
+
(set by cache migration 010). Cursor-resumable: backfills
|
|
1304
|
+
search_tool/search_thinking from each row's ``blocks_json`` via the SHARED
|
|
1305
|
+
``_lib_conversation._derive_search_columns`` chokepoint (so the values are
|
|
1306
|
+
byte-identical to live ingest), checkpointing
|
|
1307
|
+
``conversation_search_split_cursor`` per 500-row batch. These UPDATEs are
|
|
1308
|
+
INVISIBLE to the LEGACY triggers (which fire on text/search_aux only), so the
|
|
1309
|
+
old prose search keeps working untouched until the final swap (spec F5).
|
|
1310
|
+
|
|
1311
|
+
When the cursor completes, swap the legacy two-table FTS to the consolidated
|
|
1312
|
+
split shape + rebuild (one short transaction), then delete the pending +
|
|
1313
|
+
cursor meta keys. FTS5-unavailable (``fts5_unavailable`` set): the base-column
|
|
1314
|
+
backfill still runs (it is FTS-independent), the vtable swap is SKIPPED, the
|
|
1315
|
+
flag still clears, and the rebuild-on-availability recovery path
|
|
1316
|
+
(_apply_cache_schema) lands the split shape later (spec F6). Interrupted at
|
|
1317
|
+
any point ⇒ resumes from the cursor on the next locked sync; a fresh install
|
|
1318
|
+
never sets the flag so this is a cheap no-op there."""
|
|
1319
|
+
if conn.execute(
|
|
1320
|
+
"SELECT 1 FROM cache_meta WHERE key='conversation_search_split_pending'"
|
|
1321
|
+
).fetchone() is None:
|
|
1322
|
+
return
|
|
1323
|
+
row = conn.execute(
|
|
1324
|
+
"SELECT value FROM cache_meta WHERE key='conversation_search_split_cursor'"
|
|
1325
|
+
).fetchone()
|
|
1326
|
+
last_id = int(row[0]) if row else 0
|
|
1327
|
+
while True:
|
|
1328
|
+
batch = conn.execute(
|
|
1329
|
+
"SELECT id, blocks_json FROM conversation_messages "
|
|
1330
|
+
"WHERE id > ? ORDER BY id LIMIT 500",
|
|
1331
|
+
(last_id,)).fetchall()
|
|
1332
|
+
if not batch:
|
|
1333
|
+
break
|
|
1334
|
+
ups = []
|
|
1335
|
+
for rid, bj in batch:
|
|
1336
|
+
try:
|
|
1337
|
+
blocks = json.loads(bj) if bj else []
|
|
1338
|
+
except (TypeError, ValueError):
|
|
1339
|
+
blocks = []
|
|
1340
|
+
st, sth = _lib_conversation._derive_search_columns(blocks)
|
|
1341
|
+
ups.append((st, sth, rid))
|
|
1342
|
+
last_id = rid
|
|
1343
|
+
conn.executemany(
|
|
1344
|
+
"UPDATE conversation_messages SET search_tool=?, search_thinking=? "
|
|
1345
|
+
"WHERE id=?", ups)
|
|
1346
|
+
_cctally_db_sib._set_cache_meta(
|
|
1347
|
+
conn, "conversation_search_split_cursor", str(last_id))
|
|
1348
|
+
conn.commit()
|
|
1349
|
+
fts_off = conn.execute(
|
|
1350
|
+
"SELECT 1 FROM cache_meta WHERE key='fts5_unavailable'"
|
|
1351
|
+
).fetchone() is not None
|
|
1352
|
+
if not fts_off:
|
|
1353
|
+
_cctally_db_sib._swap_conversation_fts_to_split(conn)
|
|
1354
|
+
conn.execute(
|
|
1355
|
+
"DELETE FROM cache_meta WHERE key IN "
|
|
1356
|
+
"('conversation_search_split_pending','conversation_search_split_cursor')")
|
|
1357
|
+
conn.commit()
|
|
1358
|
+
|
|
1359
|
+
|
|
1263
1360
|
def iter_entries(
|
|
1264
1361
|
conn: sqlite3.Connection,
|
|
1265
1362
|
range_start: dt.datetime,
|
|
@@ -5218,6 +5218,12 @@ def _qs_str(q: dict, key: str, default: str | None) -> str | None:
|
|
|
5218
5218
|
return vals[0] if vals else default
|
|
5219
5219
|
|
|
5220
5220
|
|
|
5221
|
+
# #177 S6: valid kind facets for the conversation search / find routes. Kept in
|
|
5222
|
+
# lockstep with ``_lib_conversation_query._SEARCH_KINDS`` (the kernel re-raises
|
|
5223
|
+
# ValueError on an unknown kind; the handlers reject with a 400 before the call).
|
|
5224
|
+
_CONV_SEARCH_KINDS = ("all", "prompts", "assistant", "tools", "thinking")
|
|
5225
|
+
|
|
5226
|
+
|
|
5221
5227
|
class DashboardHTTPHandler(BaseHTTPRequestHandler):
|
|
5222
5228
|
"""Routes:
|
|
5223
5229
|
GET / → dashboard.html
|
|
@@ -5348,6 +5354,14 @@ class DashboardHTTPHandler(BaseHTTPRequestHandler):
|
|
|
5348
5354
|
elif path.startswith("/api/conversation/") and path.endswith("/media"):
|
|
5349
5355
|
# #177 S4: on-demand media bytes. Matched BEFORE the <id> reader.
|
|
5350
5356
|
self._handle_get_conversation_media(path)
|
|
5357
|
+
elif path.startswith("/api/conversation/") and path.endswith("/outline"):
|
|
5358
|
+
# #177 S5: full-session outline skeleton + stats. Matched BEFORE
|
|
5359
|
+
# the <id> reader catch-all (Codex F2 — same precedence as /payload).
|
|
5360
|
+
self._handle_get_conversation_outline(path)
|
|
5361
|
+
elif path.startswith("/api/conversation/") and path.endswith("/find"):
|
|
5362
|
+
# #177 S6: in-conversation find → rendered-turn anchors. Matched
|
|
5363
|
+
# BEFORE the <id> reader catch-all (same precedence as /outline).
|
|
5364
|
+
self._handle_get_conversation_find(path)
|
|
5351
5365
|
elif path.startswith("/api/conversation/"):
|
|
5352
5366
|
self._handle_get_conversation_detail(path)
|
|
5353
5367
|
else:
|
|
@@ -7250,6 +7264,19 @@ class DashboardHTTPHandler(BaseHTTPRequestHandler):
|
|
|
7250
7264
|
"""Lazy-load the pure conversation query kernel (Plan 2, §3)."""
|
|
7251
7265
|
return sys.modules["cctally"]._load_sibling("_lib_conversation_query")
|
|
7252
7266
|
|
|
7267
|
+
def _parse_search_kind(self, q):
|
|
7268
|
+
"""Read + validate the ``kind`` facet shared by the conversation search
|
|
7269
|
+
and find routes (#177 S6). Returns the kind on success, or ``None`` after
|
|
7270
|
+
having ALREADY sent a 400 — callers just ``return`` on ``None``. Kept in
|
|
7271
|
+
lockstep with the kernel's ``_SEARCH_KINDS`` via ``_CONV_SEARCH_KINDS``
|
|
7272
|
+
(the kernel module is resolved lazily per-request, so the handler keeps a
|
|
7273
|
+
literal tuple rather than reaching across that import edge for a nit)."""
|
|
7274
|
+
kind = _qs_str(q, "kind", "all")
|
|
7275
|
+
if kind not in _CONV_SEARCH_KINDS:
|
|
7276
|
+
self._respond_json(400, {"error": f"unknown kind: {kind}"})
|
|
7277
|
+
return None
|
|
7278
|
+
return kind
|
|
7279
|
+
|
|
7253
7280
|
def _run_conversation_query(self, kernel_call, log_label):
|
|
7254
7281
|
"""Open cache.db, run ``kernel_call(conn)``, close — with the uniform
|
|
7255
7282
|
500 envelopes the three conversation routes share (#151).
|
|
@@ -7333,8 +7360,10 @@ class DashboardHTTPHandler(BaseHTTPRequestHandler):
|
|
|
7333
7360
|
self._respond_json(200, body)
|
|
7334
7361
|
|
|
7335
7362
|
def _handle_get_conversation_search(self) -> None:
|
|
7336
|
-
"""``GET /api/conversation/search?q=...`` — cross-session
|
|
7337
|
-
search (spec §3.3). Matched BEFORE the ``<id>`` reader in
|
|
7363
|
+
"""``GET /api/conversation/search?q=...&kind=...`` — cross-session
|
|
7364
|
+
FTS/LIKE search (spec §3.3). Matched BEFORE the ``<id>`` reader in
|
|
7365
|
+
``do_GET``. ``kind`` (#177 S6) is validated to ``_CONV_SEARCH_KINDS``
|
|
7366
|
+
(else 400) before the kernel call.
|
|
7338
7367
|
"""
|
|
7339
7368
|
if not self._require_transcripts_allowed():
|
|
7340
7369
|
return
|
|
@@ -7343,9 +7372,12 @@ class DashboardHTTPHandler(BaseHTTPRequestHandler):
|
|
|
7343
7372
|
query = _qs_str(q, "q", "")
|
|
7344
7373
|
limit = _qs_int(q, "limit", 50)
|
|
7345
7374
|
offset = _qs_int(q, "offset", 0)
|
|
7375
|
+
kind = self._parse_search_kind(q)
|
|
7376
|
+
if kind is None:
|
|
7377
|
+
return
|
|
7346
7378
|
ok, body = self._run_conversation_query(
|
|
7347
7379
|
lambda conn: self._conversation_query().search_conversations(
|
|
7348
|
-
conn, query, limit=limit, offset=offset),
|
|
7380
|
+
conn, query, limit=limit, offset=offset, kind=kind),
|
|
7349
7381
|
"/api/conversation/search")
|
|
7350
7382
|
if not ok:
|
|
7351
7383
|
return
|
|
@@ -7391,6 +7423,56 @@ class DashboardHTTPHandler(BaseHTTPRequestHandler):
|
|
|
7391
7423
|
return
|
|
7392
7424
|
self._respond_json(200, payload)
|
|
7393
7425
|
|
|
7426
|
+
def _handle_get_conversation_outline(self, path: str) -> None:
|
|
7427
|
+
"""``GET /api/conversation/<sid>/outline`` — full-session skeleton +
|
|
7428
|
+
session stats (#177 S5). Same fail-closed privacy gate; unknown id → 404.
|
|
7429
|
+
"""
|
|
7430
|
+
if not self._require_transcripts_allowed():
|
|
7431
|
+
return
|
|
7432
|
+
import urllib.parse as _u
|
|
7433
|
+
session_id = _u.unquote(path[len("/api/conversation/"):-len("/outline")])
|
|
7434
|
+
if not session_id:
|
|
7435
|
+
self.send_error(404, "conversation not found")
|
|
7436
|
+
return
|
|
7437
|
+
ok, body = self._run_conversation_query(
|
|
7438
|
+
lambda conn: self._conversation_query().get_conversation_outline(conn, session_id),
|
|
7439
|
+
"/api/conversation/outline")
|
|
7440
|
+
if not ok:
|
|
7441
|
+
return
|
|
7442
|
+
if body is None:
|
|
7443
|
+
self.send_error(404, "conversation not found")
|
|
7444
|
+
return
|
|
7445
|
+
self._respond_json(200, body)
|
|
7446
|
+
|
|
7447
|
+
def _handle_get_conversation_find(self, path: str) -> None:
|
|
7448
|
+
"""``GET /api/conversation/<sid>/find?q=...&kind=...`` — in-conversation
|
|
7449
|
+
find → document-ordered rendered-turn anchors (#177 S6). Same fail-closed
|
|
7450
|
+
privacy gate as the sibling routes; unknown id → 404; an invalid ``kind``
|
|
7451
|
+
→ 400. Matched BEFORE the ``<id>`` reader catch-all in ``do_GET``.
|
|
7452
|
+
"""
|
|
7453
|
+
if not self._require_transcripts_allowed():
|
|
7454
|
+
return
|
|
7455
|
+
import urllib.parse as _u
|
|
7456
|
+
session_id = _u.unquote(path[len("/api/conversation/"):-len("/find")])
|
|
7457
|
+
if not session_id:
|
|
7458
|
+
self.send_error(404, "conversation not found")
|
|
7459
|
+
return
|
|
7460
|
+
q = _u.parse_qs(self.path.partition("?")[2])
|
|
7461
|
+
query = _qs_str(q, "q", "")
|
|
7462
|
+
kind = self._parse_search_kind(q)
|
|
7463
|
+
if kind is None:
|
|
7464
|
+
return
|
|
7465
|
+
ok, body = self._run_conversation_query(
|
|
7466
|
+
lambda conn: self._conversation_query().find_in_conversation(
|
|
7467
|
+
conn, session_id, query, kind=kind),
|
|
7468
|
+
"/api/conversation/find")
|
|
7469
|
+
if not ok:
|
|
7470
|
+
return
|
|
7471
|
+
if body is None:
|
|
7472
|
+
self.send_error(404, "conversation not found")
|
|
7473
|
+
return
|
|
7474
|
+
self._respond_json(200, body)
|
|
7475
|
+
|
|
7394
7476
|
_MEDIA_FETCH_SITE_ALLOWED = ("same-origin", "same-site", "none")
|
|
7395
7477
|
|
|
7396
7478
|
def _handle_get_conversation_media(self, path: str) -> None:
|
package/bin/_cctally_db.py
CHANGED
|
@@ -2407,6 +2407,16 @@ def _apply_cache_schema(conn: sqlite3.Connection) -> None:
|
|
|
2407
2407
|
add_column_if_missing(conn, "conversation_messages", "attribution_plugin", "TEXT")
|
|
2408
2408
|
add_column_if_missing(
|
|
2409
2409
|
conn, "conversation_messages", "search_aux", "TEXT NOT NULL DEFAULT ''")
|
|
2410
|
+
# #177 S6: split the non-prose search index into two columns so kind facets
|
|
2411
|
+
# (Tools / Thinking) are exact in SQL. ``search_aux`` above is documented-dead
|
|
2412
|
+
# — the writer stops populating it (always ''); the consolidated multi-column
|
|
2413
|
+
# conversation_fts(text, search_tool, search_thinking) indexes these instead.
|
|
2414
|
+
# Idempotent column-adds (no marker, no version); migration 010 backfills the
|
|
2415
|
+
# values onto existing history from blocks_json under the cache.db.lock flock.
|
|
2416
|
+
add_column_if_missing(
|
|
2417
|
+
conn, "conversation_messages", "search_tool", "TEXT NOT NULL DEFAULT ''")
|
|
2418
|
+
add_column_if_missing(
|
|
2419
|
+
conn, "conversation_messages", "search_thinking", "TEXT NOT NULL DEFAULT ''")
|
|
2410
2420
|
conn.execute(
|
|
2411
2421
|
"CREATE INDEX IF NOT EXISTS idx_session_files_session_id "
|
|
2412
2422
|
"ON session_files(session_id)"
|
|
@@ -2417,6 +2427,23 @@ def _apply_cache_schema(conn: sqlite3.Connection) -> None:
|
|
|
2417
2427
|
# would itself error), set a persisted flag, and let search fall back to
|
|
2418
2428
|
# LIKE. Spec §1. Idempotent (IF NOT EXISTS).
|
|
2419
2429
|
if _fts5_available(conn):
|
|
2430
|
+
# #177 S6: if a LEGACY conversation_fts(text) shape is already present
|
|
2431
|
+
# (a pre-S6 install whose migration-010 backfill/swap has not yet run),
|
|
2432
|
+
# leave EVERYTHING untouched — the legacy table, the legacy aux table,
|
|
2433
|
+
# and the legacy triggers. The sync-side swap (_swap_conversation_fts_to_
|
|
2434
|
+
# split, under the cache.db.lock flock) owns the transition; re-applying
|
|
2435
|
+
# the schema must NOT race it or partially mutate the shape. A fresh
|
|
2436
|
+
# install (no conversation_fts) and an already-split shape both fall
|
|
2437
|
+
# through to the create/recover path below.
|
|
2438
|
+
legacy_present = (
|
|
2439
|
+
conn.execute(
|
|
2440
|
+
"SELECT 1 FROM sqlite_master "
|
|
2441
|
+
"WHERE type='table' AND name='conversation_fts'"
|
|
2442
|
+
).fetchone() is not None
|
|
2443
|
+
and not _conversation_fts_is_split(conn))
|
|
2444
|
+
if legacy_present:
|
|
2445
|
+
conn.commit()
|
|
2446
|
+
return
|
|
2420
2447
|
try:
|
|
2421
2448
|
# Recovery (spec §1/P2): if a PRIOR run marked FTS unavailable,
|
|
2422
2449
|
# conversation_messages rows were ingested (by sync_cache / the
|
|
@@ -2425,43 +2452,34 @@ def _apply_cache_schema(conn: sqlite3.Connection) -> None:
|
|
|
2425
2452
|
# rows. Detect that BEFORE clearing the flag so we can rebuild the
|
|
2426
2453
|
# external-content index from conversation_messages below. A fresh
|
|
2427
2454
|
# install never sets the flag, so this stays False and no rebuild
|
|
2428
|
-
# runs (the triggers index rows incrementally as they arrive).
|
|
2455
|
+
# runs (the triggers index rows incrementally as they arrive). This
|
|
2456
|
+
# is ALSO the FTS5-unavailable migration-010 recovery seam: the
|
|
2457
|
+
# backfill ran the base-column UPDATEs but skipped the vtable DDL, so
|
|
2458
|
+
# the first FTS-capable open lands the split shape + rebuilds here.
|
|
2429
2459
|
recovering = conn.execute(
|
|
2430
2460
|
"SELECT 1 FROM cache_meta WHERE key='fts5_unavailable'"
|
|
2431
2461
|
).fetchone() is not None
|
|
2432
|
-
|
|
2433
|
-
|
|
2434
|
-
|
|
2435
|
-
|
|
2436
|
-
#
|
|
2437
|
-
#
|
|
2438
|
-
# which
|
|
2439
|
-
#
|
|
2440
|
-
# tables + BOTH trigger sets (the except arm). A live aux trigger over
|
|
2441
|
-
# a missing conversation_fts_aux would roll back the shared
|
|
2442
|
-
# conversation_messages write transaction (which also carries cost
|
|
2443
|
-
# ingest into session_entries). It is NOT given its own guarded block
|
|
2444
|
-
# or its own flag.
|
|
2445
|
-
_create_conversation_fts_aux_table(conn)
|
|
2446
|
-
# Trigger DDL lives in ONE place (_CONV_FTS_TRIGGER_DDL +
|
|
2447
|
-
# _CONV_FTS_AUX_TRIGGER_DDL) so this initial create and the #138
|
|
2448
|
-
# storm-free full-clear (clear_conversation_messages, which drops +
|
|
2449
|
-
# recreates BOTH trigger sets) can never drift.
|
|
2462
|
+
# #177 S6: the single consolidated multi-column external-content
|
|
2463
|
+
# table replaces the old prose + aux pair. Column names match the
|
|
2464
|
+
# content table BY NAME (external-content rule).
|
|
2465
|
+
conn.execute(_CONV_FTS_SPLIT_DDL)
|
|
2466
|
+
# Trigger DDL lives in ONE place (_CONV_FTS_TRIGGER_DDL) so this
|
|
2467
|
+
# initial create and the #138 storm-free full-clear
|
|
2468
|
+
# (clear_conversation_messages, which drops + recreates the trigger
|
|
2469
|
+
# set) can never drift.
|
|
2450
2470
|
_create_conversation_fts_triggers(conn)
|
|
2451
2471
|
if recovering:
|
|
2452
|
-
# Repopulate the freshly-(re)created
|
|
2472
|
+
# Repopulate the freshly-(re)created index from the base table
|
|
2453
2473
|
# so pre-recovery history is searchable. Cheap no-op when
|
|
2454
2474
|
# conversation_messages is empty.
|
|
2455
2475
|
conn.execute(
|
|
2456
2476
|
"INSERT INTO conversation_fts(conversation_fts) VALUES('rebuild')")
|
|
2457
|
-
conn.execute(
|
|
2458
|
-
"INSERT INTO conversation_fts_aux(conversation_fts_aux) VALUES('rebuild')")
|
|
2459
2477
|
conn.execute("DELETE FROM cache_meta WHERE key='fts5_unavailable'")
|
|
2460
2478
|
except sqlite3.OperationalError:
|
|
2461
|
-
# partial create cleanup, then mark unavailable. _drop drops
|
|
2462
|
-
#
|
|
2463
|
-
#
|
|
2464
|
-
# table.
|
|
2479
|
+
# partial create cleanup, then mark unavailable. _drop drops the
|
|
2480
|
+
# split trigger set (and any legacy aux trigger names, harmlessly),
|
|
2481
|
+
# and we drop both possible vtables, so a failed create can't leave a
|
|
2482
|
+
# live trigger over a missing table.
|
|
2465
2483
|
_drop_conversation_fts_triggers(conn)
|
|
2466
2484
|
try:
|
|
2467
2485
|
conn.execute("DROP TABLE IF EXISTS conversation_fts")
|
|
@@ -2507,9 +2525,36 @@ def _set_cache_meta(conn: sqlite3.Connection, key: str, value: str) -> None:
|
|
|
2507
2525
|
"ON CONFLICT(key) DO UPDATE SET value=excluded.value", (key, value))
|
|
2508
2526
|
|
|
2509
2527
|
|
|
2528
|
+
# #177 S6: the consolidated multi-column external-content FTS5 table that
|
|
2529
|
+
# replaces the old conversation_fts(text) + conversation_fts_aux(search_aux)
|
|
2530
|
+
# pair. The three column names MUST match the conversation_messages columns BY
|
|
2531
|
+
# NAME (external-content FTS5 resolves columns through the content table by
|
|
2532
|
+
# name): a mismatch creates fine but breaks 'rebuild' + iterdump
|
|
2533
|
+
# ("no such column"). Fresh installs create this directly; existing installs
|
|
2534
|
+
# swap to it under the cache.db.lock flock via _swap_conversation_fts_to_split.
|
|
2535
|
+
_CONV_FTS_SPLIT_DDL = (
|
|
2536
|
+
"CREATE VIRTUAL TABLE IF NOT EXISTS conversation_fts USING fts5("
|
|
2537
|
+
"text, search_tool, search_thinking, "
|
|
2538
|
+
"content='conversation_messages', content_rowid='id')"
|
|
2539
|
+
)
|
|
2540
|
+
|
|
2541
|
+
|
|
2542
|
+
def _conversation_fts_is_split(conn: sqlite3.Connection) -> bool:
|
|
2543
|
+
"""True if conversation_fts is the #177 S6 multi-column shape (carries the
|
|
2544
|
+
``search_tool`` column), False for the legacy single-column ``text`` shape or
|
|
2545
|
+
an absent table."""
|
|
2546
|
+
cols = [r[1] for r in conn.execute("PRAGMA table_info(conversation_fts)")]
|
|
2547
|
+
return "search_tool" in cols
|
|
2548
|
+
|
|
2549
|
+
|
|
2510
2550
|
def _create_conversation_fts_aux_table(conn: sqlite3.Connection) -> None:
|
|
2511
2551
|
"""Create the #177 aux external-content FTS5 index over the ``search_aux``
|
|
2512
|
-
blob.
|
|
2552
|
+
blob. TEST/FIXTURE STANDUP ONLY — production code (``_apply_cache_schema``)
|
|
2553
|
+
builds the post-#177-S6 SPLIT index directly; this legacy single-aux-column
|
|
2554
|
+
shape is referenced solely by the migration-fixture builder + the swap tests
|
|
2555
|
+
(the migration's by-name DROP list still legitimately uses the trigger-name
|
|
2556
|
+
tuples, but never calls this helper). A standalone module-level seam (NOT
|
|
2557
|
+
inlined in ``_apply_cache_schema``)
|
|
2513
2558
|
so the all-or-nothing regression test can monkeypatch it to raise
|
|
2514
2559
|
``OperationalError`` AFTER the prose ``conversation_fts`` create succeeded —
|
|
2515
2560
|
proving the shared try-envelope drops BOTH indexes + BOTH trigger sets and a
|
|
@@ -2533,16 +2578,26 @@ def _create_conversation_fts_aux_table(conn: sqlite3.Connection) -> None:
|
|
|
2533
2578
|
# the initial create in _apply_cache_schema and the #138 storm-free full-clear
|
|
2534
2579
|
# in clear_conversation_messages (which drops + recreates them) can never drift.
|
|
2535
2580
|
# conv_fts_ad / conv_fts_au use the external-content `'delete'` idiom.
|
|
2581
|
+
# #177 S6: three-column trigger set for the consolidated conversation_fts. Same
|
|
2582
|
+
# trigger names (conv_fts_ai/ad/au) as the legacy single-column set so the swap
|
|
2583
|
+
# reuses them; the AU trigger now fires AFTER UPDATE OF text, search_tool,
|
|
2584
|
+
# search_thinking (the backfill UPDATEs search_tool/search_thinking, which the
|
|
2585
|
+
# OLD text-only AU did NOT fire — that invisibility is load-bearing, see
|
|
2586
|
+
# _consume_search_split, but it applies only while the legacy triggers are still
|
|
2587
|
+
# installed; once swapped, these fire on all three columns).
|
|
2536
2588
|
_CONV_FTS_TRIGGER_DDL = (
|
|
2537
2589
|
"CREATE TRIGGER IF NOT EXISTS conv_fts_ai AFTER INSERT ON conversation_messages "
|
|
2538
|
-
"BEGIN INSERT INTO conversation_fts(rowid, text
|
|
2590
|
+
"BEGIN INSERT INTO conversation_fts(rowid, text, search_tool, search_thinking) "
|
|
2591
|
+
"VALUES (new.id, new.text, new.search_tool, new.search_thinking); END",
|
|
2539
2592
|
"CREATE TRIGGER IF NOT EXISTS conv_fts_ad AFTER DELETE ON conversation_messages "
|
|
2540
|
-
"BEGIN INSERT INTO conversation_fts(conversation_fts, rowid, text) "
|
|
2541
|
-
"VALUES('delete', old.id, old.text); END",
|
|
2542
|
-
"CREATE TRIGGER IF NOT EXISTS conv_fts_au
|
|
2543
|
-
"
|
|
2544
|
-
"
|
|
2545
|
-
"
|
|
2593
|
+
"BEGIN INSERT INTO conversation_fts(conversation_fts, rowid, text, search_tool, search_thinking) "
|
|
2594
|
+
"VALUES('delete', old.id, old.text, old.search_tool, old.search_thinking); END",
|
|
2595
|
+
"CREATE TRIGGER IF NOT EXISTS conv_fts_au "
|
|
2596
|
+
"AFTER UPDATE OF text, search_tool, search_thinking ON conversation_messages "
|
|
2597
|
+
"BEGIN INSERT INTO conversation_fts(conversation_fts, rowid, text, search_tool, search_thinking) "
|
|
2598
|
+
"VALUES('delete', old.id, old.text, old.search_tool, old.search_thinking); "
|
|
2599
|
+
"INSERT INTO conversation_fts(rowid, text, search_tool, search_thinking) "
|
|
2600
|
+
"VALUES (new.id, new.text, new.search_tool, new.search_thinking); END",
|
|
2546
2601
|
)
|
|
2547
2602
|
# Drop by name (the body is irrelevant to DROP TRIGGER); reverse order is
|
|
2548
2603
|
# cosmetic — order doesn't matter for independent triggers.
|
|
@@ -2567,17 +2622,46 @@ _CONV_FTS_AUX_TRIGGER_DDL = (
|
|
|
2567
2622
|
)
|
|
2568
2623
|
_CONV_FTS_AUX_TRIGGER_NAMES = ("conv_fts_aux_au", "conv_fts_aux_ad", "conv_fts_aux_ai")
|
|
2569
2624
|
|
|
2625
|
+
# #177 S6: the LEGACY single-column prose trigger set, retained ONLY so a
|
|
2626
|
+
# pre-swap install (and the legacy-shape test fixture) can stand up the old
|
|
2627
|
+
# conversation_fts(text) shape. Fresh installs + the post-swap shape use the
|
|
2628
|
+
# three-column _CONV_FTS_TRIGGER_DDL above. Same trigger names, so the swap's
|
|
2629
|
+
# DROP-by-name (_CONV_FTS_TRIGGER_NAMES) covers either generation.
|
|
2630
|
+
_CONV_FTS_LEGACY_TRIGGER_DDL = (
|
|
2631
|
+
"CREATE TRIGGER IF NOT EXISTS conv_fts_ai AFTER INSERT ON conversation_messages "
|
|
2632
|
+
"BEGIN INSERT INTO conversation_fts(rowid, text) VALUES (new.id, new.text); END",
|
|
2633
|
+
"CREATE TRIGGER IF NOT EXISTS conv_fts_ad AFTER DELETE ON conversation_messages "
|
|
2634
|
+
"BEGIN INSERT INTO conversation_fts(conversation_fts, rowid, text) "
|
|
2635
|
+
"VALUES('delete', old.id, old.text); END",
|
|
2636
|
+
"CREATE TRIGGER IF NOT EXISTS conv_fts_au AFTER UPDATE OF text ON conversation_messages "
|
|
2637
|
+
"BEGIN INSERT INTO conversation_fts(conversation_fts, rowid, text) "
|
|
2638
|
+
"VALUES('delete', old.id, old.text); "
|
|
2639
|
+
"INSERT INTO conversation_fts(rowid, text) VALUES (new.id, new.text); END",
|
|
2640
|
+
)
|
|
2641
|
+
|
|
2570
2642
|
|
|
2571
2643
|
def _create_conversation_fts_triggers(conn: sqlite3.Connection) -> None:
|
|
2572
|
-
"""Create
|
|
2573
|
-
|
|
2574
|
-
|
|
2575
|
-
|
|
2576
|
-
|
|
2577
|
-
|
|
2578
|
-
|
|
2644
|
+
"""Create the #177 S6 split conversation_fts(text, search_tool,
|
|
2645
|
+
search_thinking) sync trigger set — idempotent (each is ``IF NOT EXISTS``).
|
|
2646
|
+
Single source of truth for the trigger DDL, shared by ``_apply_cache_schema``
|
|
2647
|
+
and ``clear_conversation_messages`` (#138). The caller must have already
|
|
2648
|
+
created ``conversation_fts`` in the split shape (the consolidated table
|
|
2649
|
+
replaced the old prose+aux pair, so there is no separate aux trigger set to
|
|
2650
|
+
create)."""
|
|
2579
2651
|
for stmt in _CONV_FTS_TRIGGER_DDL:
|
|
2580
2652
|
conn.execute(stmt)
|
|
2653
|
+
|
|
2654
|
+
|
|
2655
|
+
def _create_conversation_fts_legacy_triggers(conn: sqlite3.Connection) -> None:
|
|
2656
|
+
"""Create the LEGACY single-column prose + aux trigger sets over the
|
|
2657
|
+
pre-#177-S6 conversation_fts(text) + conversation_fts_aux(search_aux) pair.
|
|
2658
|
+
TEST/FIXTURE STANDUP ONLY — referenced solely by the migration-fixture
|
|
2659
|
+
builder + the swap tests to materialize the legacy shape the migration then
|
|
2660
|
+
upgrades. Production never calls this (the swap drops these trigger sets BY
|
|
2661
|
+
NAME via the ``_CONV_FTS_*_TRIGGER_NAMES`` tuples and recreates the split
|
|
2662
|
+
set). The caller must have created both legacy tables."""
|
|
2663
|
+
for stmt in _CONV_FTS_LEGACY_TRIGGER_DDL:
|
|
2664
|
+
conn.execute(stmt)
|
|
2581
2665
|
for stmt in _CONV_FTS_AUX_TRIGGER_DDL:
|
|
2582
2666
|
conn.execute(stmt)
|
|
2583
2667
|
|
|
@@ -2639,10 +2723,16 @@ def clear_conversation_messages(conn: sqlite3.Connection) -> None:
|
|
|
2639
2723
|
_drop_conversation_fts_triggers(conn)
|
|
2640
2724
|
conn.execute("DELETE FROM conversation_messages")
|
|
2641
2725
|
conn.execute("INSERT INTO conversation_fts(conversation_fts) VALUES('delete-all')")
|
|
2642
|
-
# #177: the
|
|
2643
|
-
#
|
|
2644
|
-
#
|
|
2645
|
-
|
|
2726
|
+
# #177 S6: the consolidated split table is the only FTS vtable on a swapped /
|
|
2727
|
+
# fresh DB. A pre-swap legacy install still carries conversation_fts_aux, so
|
|
2728
|
+
# reset it the same storm-free way ONLY when it physically exists (the swap
|
|
2729
|
+
# drops it; a fresh/split DB never had it). Triggers were dropped above, so
|
|
2730
|
+
# the base DELETE touched neither index.
|
|
2731
|
+
if conn.execute(
|
|
2732
|
+
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='conversation_fts_aux'"
|
|
2733
|
+
).fetchone() is not None:
|
|
2734
|
+
conn.execute(
|
|
2735
|
+
"INSERT INTO conversation_fts_aux(conversation_fts_aux) VALUES('delete-all')")
|
|
2646
2736
|
_create_conversation_fts_triggers(conn)
|
|
2647
2737
|
|
|
2648
2738
|
|
|
@@ -3225,6 +3315,50 @@ def _009_conversation_media_reingest(conn: sqlite3.Connection) -> None:
|
|
|
3225
3315
|
conn.commit()
|
|
3226
3316
|
|
|
3227
3317
|
|
|
3318
|
+
@cache_migration("010_conversation_search_split")
|
|
3319
|
+
def _010_conversation_search_split(conn: sqlite3.Connection) -> None:
|
|
3320
|
+
"""Flag-only arm for the #177 S6 search-column split. Sets
|
|
3321
|
+
``conversation_search_split_pending``; sync_cache consumes it under the
|
|
3322
|
+
cache.db.lock flock (cursor-resumable blocks_json backfill of
|
|
3323
|
+
search_tool/search_thinking, then an atomic legacy->split FTS swap+rebuild —
|
|
3324
|
+
see _cctally_cache._consume_search_split). The handler does NO data work so
|
|
3325
|
+
the dispatcher's central stamp (#140) marks a genuinely-complete handler; a
|
|
3326
|
+
fresh install stamps it WITHOUT running (its schema is already split via
|
|
3327
|
+
_apply_cache_schema, so the consumer finds no flag and no-ops). Mirrors the
|
|
3328
|
+
flag-only reingest pattern of 002/003/007/009."""
|
|
3329
|
+
_set_cache_meta(conn, "conversation_search_split_pending", "1")
|
|
3330
|
+
conn.commit()
|
|
3331
|
+
|
|
3332
|
+
|
|
3333
|
+
def _swap_conversation_fts_to_split(conn: sqlite3.Connection) -> None:
|
|
3334
|
+
"""Legacy two-table shape -> split multi-column shape, in one transaction.
|
|
3335
|
+
Explicit DROPs of BOTH old tables + triggers (an ``IF NOT EXISTS`` create
|
|
3336
|
+
would silently keep the old same-name single-column conversation_fts — spec
|
|
3337
|
+
F9), then create the split DDL + new triggers and rebuild the index from the
|
|
3338
|
+
base table. Runs under the caller's held cache.db.lock flock (the
|
|
3339
|
+
_consume_search_split end-transaction)."""
|
|
3340
|
+
_drop_conversation_fts_triggers(conn)
|
|
3341
|
+
conn.execute("DROP TABLE IF EXISTS conversation_fts")
|
|
3342
|
+
conn.execute("DROP TABLE IF EXISTS conversation_fts_aux")
|
|
3343
|
+
conn.execute(_CONV_FTS_SPLIT_DDL)
|
|
3344
|
+
_create_conversation_fts_triggers(conn)
|
|
3345
|
+
conn.execute("INSERT INTO conversation_fts(conversation_fts) VALUES('rebuild')")
|
|
3346
|
+
|
|
3347
|
+
|
|
3348
|
+
def conversation_search_depth(conn: sqlite3.Connection) -> str:
|
|
3349
|
+
"""'prose-only' while migration 010's backfill is pending, else 'full'.
|
|
3350
|
+
Surfaced additively on the search/find responses so the client can degrade
|
|
3351
|
+
the Tools/Thinking facets during the one-time index split. An
|
|
3352
|
+
OperationalError (no cache_meta table yet) degrades to 'full'."""
|
|
3353
|
+
try:
|
|
3354
|
+
pending = conn.execute(
|
|
3355
|
+
"SELECT 1 FROM cache_meta WHERE key='conversation_search_split_pending'"
|
|
3356
|
+
).fetchone()
|
|
3357
|
+
except sqlite3.OperationalError:
|
|
3358
|
+
return "full"
|
|
3359
|
+
return "prose-only" if pending else "full"
|
|
3360
|
+
|
|
3361
|
+
|
|
3228
3362
|
# === Region 7d: Stats migration 008_recompute_weekly_cost_snapshots_dedup_fix ===
|
|
3229
3363
|
|
|
3230
3364
|
@stats_migration("008_recompute_weekly_cost_snapshots_dedup_fix")
|