cctally 1.40.0 → 1.42.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/bin/_cctally_cache.py +103 -6
- package/bin/_cctally_dashboard.py +60 -3
- package/bin/_cctally_db.py +179 -45
- package/bin/_lib_conversation.py +142 -23
- package/bin/_lib_conversation_query.py +447 -60
- package/dashboard/static/assets/{index-BXm_vBKA.css → index-3yVZt8cV.css} +1 -1
- package/dashboard/static/assets/{index-BtbVKnkm.js → index-DDuPO7EI.js} +34 -34
- package/dashboard/static/dashboard.html +2 -2
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -5,6 +5,21 @@ based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
|
|
|
5
5
|
|
|
6
6
|
## [Unreleased]
|
|
7
7
|
|
|
8
|
+
## [1.42.0] - 2026-06-13
|
|
9
|
+
|
|
10
|
+
### Fixed
|
|
11
|
+
- Conversation viewer: system and slash-command messages (`/clear`, `/model`, the `local-command-stdout`/`local-command-stderr` echoes, and the caveats) were being attributed to you as "You" turns, and the first such line became the whole conversation's title; they now fold into collapsed "System marker" pills and the title falls back to your real first prompt. Such rows are recognized at ingest (stored as meta and kept out of search and the prompts facet) with a read-time fallback that also cleans already-recorded history, plus a belt-and-suspenders title skip that degrades unknown `command-*`/`local-command-*` tags to "skip" rather than letting them poison the title (#186).
|
|
12
|
+
- Conversation viewer: terminal ANSI escape codes (for example the bold `^[[1m…^[[22m` wrapping "Fable 5") no longer leak into titles, outline labels, or message and command body text — they are stripped at ingest (which also keeps the search index clean for new rows) and again at the render chokepoints (which cleans already-recorded history without a re-ingest), while Bash tool output keeps its intentional terminal colors (#186).
|
|
13
|
+
|
|
14
|
+
### Changed
|
|
15
|
+
- Conversation viewer: the outline rail is redesigned from a flat wall of ~200 truncated prose lines into a bold prompt spine (your prompts, each with a divider above it) with curated landmarks demoted beneath each prompt — section headings, plans, questions, errors (red), and subagent spawns (magenta) — where a turn's thinking blocks collapse to a single `🧠 ×N` badge on its prompt instead of doubling the list, session-start plumbing rows are gone, role glyphs are color-coded for at-a-glance scanning, and the scroll-sync highlight tracks the enclosing prompt (#186).
|
|
16
|
+
- Conversation viewer: the outline header is merged into one "session at a glance" card — time, tokens, and cost become labeled stat tiles, models/tools/errors become labeled rows, the formerly cryptic unlabeled glyph jump-chips gain text labels and move into the card under a "Jump to" row, and the error count reconciles to one phrase ("14 errors in 13 turns") so the previously-disagreeing 13 and 14 no longer read as two competing numbers (#186).
|
|
17
|
+
|
|
18
|
+
## [1.41.0] - 2026-06-13
|
|
19
|
+
|
|
20
|
+
### Added
|
|
21
|
+
- Conversation viewer: search now reaches past prose into commands, file paths, error strings, and the assistant's thinking, with exact kind facets and a browser-style in-conversation find bar — the rail gains an `All · Prompts · Assistant · Tools · Thinking` chip row, a result count line, match badges (`tool`, `thinking`), and a `Load N more` button, while pressing `/` over an open conversation opens a floating find pill (`Enter`/`Shift+Enter` and `n`/`N` step matches, `Esc` closes, wrap-around, exact `k / N` counter, `<mark>` term highlighting in prose) backed by a new `GET /api/conversation/<id>/find` endpoint and a `kind` param on `/api/conversation/search`; the tool/thinking facets light up after a one-time, lossless search-index split (cache migration `010`) runs automatically under the cache lock on the next sync, reporting `search_depth: "prose-only"` (with the Tools/Thinking chips disabled and an `indexing…` hint) until it completes, and degrading to a basic substring scan on hosts without FTS5 (#177 S6).
|
|
22
|
+
|
|
8
23
|
## [1.40.0] - 2026-06-12
|
|
9
24
|
|
|
10
25
|
### Added
|
package/bin/_cctally_cache.py
CHANGED
|
@@ -184,8 +184,9 @@ _CONV_INSERT_SQL = (
|
|
|
184
184
|
"(session_id,uuid,parent_uuid,source_path,byte_offset,"
|
|
185
185
|
" timestamp_utc,entry_type,text,blocks_json,model,msg_id,"
|
|
186
186
|
" req_id,cwd,git_branch,is_sidechain,source_tool_use_id,"
|
|
187
|
-
" stop_reason,attribution_skill,attribution_plugin,
|
|
188
|
-
"
|
|
187
|
+
" stop_reason,attribution_skill,attribution_plugin,"
|
|
188
|
+
" search_tool,search_thinking,search_aux)"
|
|
189
|
+
" VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
|
|
189
190
|
)
|
|
190
191
|
|
|
191
192
|
|
|
@@ -193,15 +194,19 @@ def _conv_row_tuple(m, path_str):
|
|
|
193
194
|
"""Flatten a ``MessageRow`` into the ``_CONV_INSERT_SQL`` column order.
|
|
194
195
|
|
|
195
196
|
The #177 enrichment fields (stop_reason / attribution_skill /
|
|
196
|
-
attribution_plugin / search_aux) are
|
|
197
|
-
— same order as the SQL column list —
|
|
198
|
-
|
|
197
|
+
attribution_plugin / search_tool / search_thinking / search_aux) are
|
|
198
|
+
TAIL-APPENDED after source_tool_use_id — same order as the SQL column list —
|
|
199
|
+
so both ingest paths (fused per-file walk + backfill_conversation_messages)
|
|
200
|
+
carry them through this one tuple. #177 S6: ``search_aux`` is always written
|
|
201
|
+
as ``''`` (documented-dead); the split ``search_tool``/``search_thinking``
|
|
202
|
+
columns carry the non-prose index."""
|
|
199
203
|
return (
|
|
200
204
|
m.session_id, m.uuid, m.parent_uuid, path_str, m.byte_offset,
|
|
201
205
|
m.timestamp_utc, m.entry_type, m.text, m.blocks_json, m.model,
|
|
202
206
|
m.msg_id, m.req_id, m.cwd, m.git_branch, m.is_sidechain,
|
|
203
207
|
m.source_tool_use_id,
|
|
204
|
-
m.stop_reason, m.attribution_skill, m.attribution_plugin,
|
|
208
|
+
m.stop_reason, m.attribution_skill, m.attribution_plugin,
|
|
209
|
+
m.search_tool, m.search_thinking, m.search_aux,
|
|
205
210
|
)
|
|
206
211
|
|
|
207
212
|
|
|
@@ -651,6 +656,31 @@ def sync_cache(
|
|
|
651
656
|
" 'conversation_media_reingest_pending',"
|
|
652
657
|
" 'conversation_reingest_cursor',"
|
|
653
658
|
" 'conversation_reingest_cursor_gen')")
|
|
659
|
+
# #177 S6: a rebuild repopulates search_tool/search_thinking via the
|
|
660
|
+
# offset-0 walk (the parser derives them), so the migration-010
|
|
661
|
+
# backfill is redundant. But a LEGACY-shape DB still carries the old
|
|
662
|
+
# prose+aux FTS pair that the split triggers can't write — swap to the
|
|
663
|
+
# split shape NOW (the table is empty post-clear, so the walk below
|
|
664
|
+
# populates it through the new triggers), then drop the pending flag +
|
|
665
|
+
# cursor so the post-rebuild sync runs no redundant backfill/swap.
|
|
666
|
+
# MISSING this site re-arms the flag on every cache-sync --rebuild.
|
|
667
|
+
try:
|
|
668
|
+
_split_pending = conn.execute(
|
|
669
|
+
"SELECT 1 FROM cache_meta "
|
|
670
|
+
"WHERE key='conversation_search_split_pending'"
|
|
671
|
+
).fetchone() is not None
|
|
672
|
+
except sqlite3.OperationalError:
|
|
673
|
+
_split_pending = False
|
|
674
|
+
if _split_pending:
|
|
675
|
+
_fts_off = conn.execute(
|
|
676
|
+
"SELECT 1 FROM cache_meta WHERE key='fts5_unavailable'"
|
|
677
|
+
).fetchone() is not None
|
|
678
|
+
if not _fts_off and not _cctally_db_sib._conversation_fts_is_split(conn):
|
|
679
|
+
_cctally_db_sib._swap_conversation_fts_to_split(conn)
|
|
680
|
+
conn.execute(
|
|
681
|
+
"DELETE FROM cache_meta WHERE key IN "
|
|
682
|
+
"('conversation_search_split_pending',"
|
|
683
|
+
" 'conversation_search_split_cursor')")
|
|
654
684
|
conn.commit()
|
|
655
685
|
eprint("[cache-sync] rebuild: cleared Claude cached entries")
|
|
656
686
|
|
|
@@ -742,6 +772,14 @@ def sync_cache(
|
|
|
742
772
|
# path (which already wipes + repopulates id-aware via the normal walk).
|
|
743
773
|
_resumable_reingest_conversation_messages(conn)
|
|
744
774
|
|
|
775
|
+
# #177 S6: consume the migration-010 search-column split under the
|
|
776
|
+
# SAME held flock, AFTER the reingest so any just-re-ingested rows
|
|
777
|
+
# already carry fresh search_tool/search_thinking before the backfill
|
|
778
|
+
# touches the tail. Cursor-resumable; the legacy triggers are blind to
|
|
779
|
+
# the search_tool/search_thinking UPDATEs (they fire on text only), so
|
|
780
|
+
# old search keeps working until the final swap.
|
|
781
|
+
_consume_search_split(conn)
|
|
782
|
+
|
|
745
783
|
paths: list[pathlib.Path] = list(_iter_claude_jsonl_files())
|
|
746
784
|
stats.files_total = len(paths)
|
|
747
785
|
|
|
@@ -1260,6 +1298,65 @@ def _resumable_reingest_conversation_messages(conn):
|
|
|
1260
1298
|
conn.commit()
|
|
1261
1299
|
|
|
1262
1300
|
|
|
1301
|
+
def _consume_search_split(conn) -> None:
|
|
1302
|
+
"""#177 S6: flock-held consumer for ``conversation_search_split_pending``
|
|
1303
|
+
(set by cache migration 010). Cursor-resumable: backfills
|
|
1304
|
+
search_tool/search_thinking from each row's ``blocks_json`` via the SHARED
|
|
1305
|
+
``_lib_conversation._derive_search_columns`` chokepoint (so the values are
|
|
1306
|
+
byte-identical to live ingest), checkpointing
|
|
1307
|
+
``conversation_search_split_cursor`` per 500-row batch. These UPDATEs are
|
|
1308
|
+
INVISIBLE to the LEGACY triggers (which fire on text/search_aux only), so the
|
|
1309
|
+
old prose search keeps working untouched until the final swap (spec F5).
|
|
1310
|
+
|
|
1311
|
+
When the cursor completes, swap the legacy two-table FTS to the consolidated
|
|
1312
|
+
split shape + rebuild (one short transaction), then delete the pending +
|
|
1313
|
+
cursor meta keys. FTS5-unavailable (``fts5_unavailable`` set): the base-column
|
|
1314
|
+
backfill still runs (it is FTS-independent), the vtable swap is SKIPPED, the
|
|
1315
|
+
flag still clears, and the rebuild-on-availability recovery path
|
|
1316
|
+
(_apply_cache_schema) lands the split shape later (spec F6). Interrupted at
|
|
1317
|
+
any point ⇒ resumes from the cursor on the next locked sync; a fresh install
|
|
1318
|
+
never sets the flag so this is a cheap no-op there."""
|
|
1319
|
+
if conn.execute(
|
|
1320
|
+
"SELECT 1 FROM cache_meta WHERE key='conversation_search_split_pending'"
|
|
1321
|
+
).fetchone() is None:
|
|
1322
|
+
return
|
|
1323
|
+
row = conn.execute(
|
|
1324
|
+
"SELECT value FROM cache_meta WHERE key='conversation_search_split_cursor'"
|
|
1325
|
+
).fetchone()
|
|
1326
|
+
last_id = int(row[0]) if row else 0
|
|
1327
|
+
while True:
|
|
1328
|
+
batch = conn.execute(
|
|
1329
|
+
"SELECT id, blocks_json FROM conversation_messages "
|
|
1330
|
+
"WHERE id > ? ORDER BY id LIMIT 500",
|
|
1331
|
+
(last_id,)).fetchall()
|
|
1332
|
+
if not batch:
|
|
1333
|
+
break
|
|
1334
|
+
ups = []
|
|
1335
|
+
for rid, bj in batch:
|
|
1336
|
+
try:
|
|
1337
|
+
blocks = json.loads(bj) if bj else []
|
|
1338
|
+
except (TypeError, ValueError):
|
|
1339
|
+
blocks = []
|
|
1340
|
+
st, sth = _lib_conversation._derive_search_columns(blocks)
|
|
1341
|
+
ups.append((st, sth, rid))
|
|
1342
|
+
last_id = rid
|
|
1343
|
+
conn.executemany(
|
|
1344
|
+
"UPDATE conversation_messages SET search_tool=?, search_thinking=? "
|
|
1345
|
+
"WHERE id=?", ups)
|
|
1346
|
+
_cctally_db_sib._set_cache_meta(
|
|
1347
|
+
conn, "conversation_search_split_cursor", str(last_id))
|
|
1348
|
+
conn.commit()
|
|
1349
|
+
fts_off = conn.execute(
|
|
1350
|
+
"SELECT 1 FROM cache_meta WHERE key='fts5_unavailable'"
|
|
1351
|
+
).fetchone() is not None
|
|
1352
|
+
if not fts_off:
|
|
1353
|
+
_cctally_db_sib._swap_conversation_fts_to_split(conn)
|
|
1354
|
+
conn.execute(
|
|
1355
|
+
"DELETE FROM cache_meta WHERE key IN "
|
|
1356
|
+
"('conversation_search_split_pending','conversation_search_split_cursor')")
|
|
1357
|
+
conn.commit()
|
|
1358
|
+
|
|
1359
|
+
|
|
1263
1360
|
def iter_entries(
|
|
1264
1361
|
conn: sqlite3.Connection,
|
|
1265
1362
|
range_start: dt.datetime,
|
|
@@ -5218,6 +5218,12 @@ def _qs_str(q: dict, key: str, default: str | None) -> str | None:
|
|
|
5218
5218
|
return vals[0] if vals else default
|
|
5219
5219
|
|
|
5220
5220
|
|
|
5221
|
+
# #177 S6: valid kind facets for the conversation search / find routes. Kept in
|
|
5222
|
+
# lockstep with ``_lib_conversation_query._SEARCH_KINDS`` (the kernel re-raises
|
|
5223
|
+
# ValueError on an unknown kind; the handlers reject with a 400 before the call).
|
|
5224
|
+
_CONV_SEARCH_KINDS = ("all", "prompts", "assistant", "tools", "thinking")
|
|
5225
|
+
|
|
5226
|
+
|
|
5221
5227
|
class DashboardHTTPHandler(BaseHTTPRequestHandler):
|
|
5222
5228
|
"""Routes:
|
|
5223
5229
|
GET / → dashboard.html
|
|
@@ -5352,6 +5358,10 @@ class DashboardHTTPHandler(BaseHTTPRequestHandler):
|
|
|
5352
5358
|
# #177 S5: full-session outline skeleton + stats. Matched BEFORE
|
|
5353
5359
|
# the <id> reader catch-all (Codex F2 — same precedence as /payload).
|
|
5354
5360
|
self._handle_get_conversation_outline(path)
|
|
5361
|
+
elif path.startswith("/api/conversation/") and path.endswith("/find"):
|
|
5362
|
+
# #177 S6: in-conversation find → rendered-turn anchors. Matched
|
|
5363
|
+
# BEFORE the <id> reader catch-all (same precedence as /outline).
|
|
5364
|
+
self._handle_get_conversation_find(path)
|
|
5355
5365
|
elif path.startswith("/api/conversation/"):
|
|
5356
5366
|
self._handle_get_conversation_detail(path)
|
|
5357
5367
|
else:
|
|
@@ -7254,6 +7264,19 @@ class DashboardHTTPHandler(BaseHTTPRequestHandler):
|
|
|
7254
7264
|
"""Lazy-load the pure conversation query kernel (Plan 2, §3)."""
|
|
7255
7265
|
return sys.modules["cctally"]._load_sibling("_lib_conversation_query")
|
|
7256
7266
|
|
|
7267
|
+
def _parse_search_kind(self, q):
|
|
7268
|
+
"""Read + validate the ``kind`` facet shared by the conversation search
|
|
7269
|
+
and find routes (#177 S6). Returns the kind on success, or ``None`` after
|
|
7270
|
+
having ALREADY sent a 400 — callers just ``return`` on ``None``. Kept in
|
|
7271
|
+
lockstep with the kernel's ``_SEARCH_KINDS`` via ``_CONV_SEARCH_KINDS``
|
|
7272
|
+
(the kernel module is resolved lazily per-request, so the handler keeps a
|
|
7273
|
+
literal tuple rather than reaching across that import edge for a nit)."""
|
|
7274
|
+
kind = _qs_str(q, "kind", "all")
|
|
7275
|
+
if kind not in _CONV_SEARCH_KINDS:
|
|
7276
|
+
self._respond_json(400, {"error": f"unknown kind: {kind}"})
|
|
7277
|
+
return None
|
|
7278
|
+
return kind
|
|
7279
|
+
|
|
7257
7280
|
def _run_conversation_query(self, kernel_call, log_label):
|
|
7258
7281
|
"""Open cache.db, run ``kernel_call(conn)``, close — with the uniform
|
|
7259
7282
|
500 envelopes the three conversation routes share (#151).
|
|
@@ -7337,8 +7360,10 @@ class DashboardHTTPHandler(BaseHTTPRequestHandler):
|
|
|
7337
7360
|
self._respond_json(200, body)
|
|
7338
7361
|
|
|
7339
7362
|
def _handle_get_conversation_search(self) -> None:
|
|
7340
|
-
"""``GET /api/conversation/search?q=...`` — cross-session
|
|
7341
|
-
search (spec §3.3). Matched BEFORE the ``<id>`` reader in
|
|
7363
|
+
"""``GET /api/conversation/search?q=...&kind=...`` — cross-session
|
|
7364
|
+
FTS/LIKE search (spec §3.3). Matched BEFORE the ``<id>`` reader in
|
|
7365
|
+
``do_GET``. ``kind`` (#177 S6) is validated to ``_CONV_SEARCH_KINDS``
|
|
7366
|
+
(else 400) before the kernel call.
|
|
7342
7367
|
"""
|
|
7343
7368
|
if not self._require_transcripts_allowed():
|
|
7344
7369
|
return
|
|
@@ -7347,9 +7372,12 @@ class DashboardHTTPHandler(BaseHTTPRequestHandler):
|
|
|
7347
7372
|
query = _qs_str(q, "q", "")
|
|
7348
7373
|
limit = _qs_int(q, "limit", 50)
|
|
7349
7374
|
offset = _qs_int(q, "offset", 0)
|
|
7375
|
+
kind = self._parse_search_kind(q)
|
|
7376
|
+
if kind is None:
|
|
7377
|
+
return
|
|
7350
7378
|
ok, body = self._run_conversation_query(
|
|
7351
7379
|
lambda conn: self._conversation_query().search_conversations(
|
|
7352
|
-
conn, query, limit=limit, offset=offset),
|
|
7380
|
+
conn, query, limit=limit, offset=offset, kind=kind),
|
|
7353
7381
|
"/api/conversation/search")
|
|
7354
7382
|
if not ok:
|
|
7355
7383
|
return
|
|
@@ -7416,6 +7444,35 @@ class DashboardHTTPHandler(BaseHTTPRequestHandler):
|
|
|
7416
7444
|
return
|
|
7417
7445
|
self._respond_json(200, body)
|
|
7418
7446
|
|
|
7447
|
+
def _handle_get_conversation_find(self, path: str) -> None:
|
|
7448
|
+
"""``GET /api/conversation/<sid>/find?q=...&kind=...`` — in-conversation
|
|
7449
|
+
find → document-ordered rendered-turn anchors (#177 S6). Same fail-closed
|
|
7450
|
+
privacy gate as the sibling routes; unknown id → 404; an invalid ``kind``
|
|
7451
|
+
→ 400. Matched BEFORE the ``<id>`` reader catch-all in ``do_GET``.
|
|
7452
|
+
"""
|
|
7453
|
+
if not self._require_transcripts_allowed():
|
|
7454
|
+
return
|
|
7455
|
+
import urllib.parse as _u
|
|
7456
|
+
session_id = _u.unquote(path[len("/api/conversation/"):-len("/find")])
|
|
7457
|
+
if not session_id:
|
|
7458
|
+
self.send_error(404, "conversation not found")
|
|
7459
|
+
return
|
|
7460
|
+
q = _u.parse_qs(self.path.partition("?")[2])
|
|
7461
|
+
query = _qs_str(q, "q", "")
|
|
7462
|
+
kind = self._parse_search_kind(q)
|
|
7463
|
+
if kind is None:
|
|
7464
|
+
return
|
|
7465
|
+
ok, body = self._run_conversation_query(
|
|
7466
|
+
lambda conn: self._conversation_query().find_in_conversation(
|
|
7467
|
+
conn, session_id, query, kind=kind),
|
|
7468
|
+
"/api/conversation/find")
|
|
7469
|
+
if not ok:
|
|
7470
|
+
return
|
|
7471
|
+
if body is None:
|
|
7472
|
+
self.send_error(404, "conversation not found")
|
|
7473
|
+
return
|
|
7474
|
+
self._respond_json(200, body)
|
|
7475
|
+
|
|
7419
7476
|
_MEDIA_FETCH_SITE_ALLOWED = ("same-origin", "same-site", "none")
|
|
7420
7477
|
|
|
7421
7478
|
def _handle_get_conversation_media(self, path: str) -> None:
|
package/bin/_cctally_db.py
CHANGED
|
@@ -2407,6 +2407,16 @@ def _apply_cache_schema(conn: sqlite3.Connection) -> None:
|
|
|
2407
2407
|
add_column_if_missing(conn, "conversation_messages", "attribution_plugin", "TEXT")
|
|
2408
2408
|
add_column_if_missing(
|
|
2409
2409
|
conn, "conversation_messages", "search_aux", "TEXT NOT NULL DEFAULT ''")
|
|
2410
|
+
# #177 S6: split the non-prose search index into two columns so kind facets
|
|
2411
|
+
# (Tools / Thinking) are exact in SQL. ``search_aux`` above is documented-dead
|
|
2412
|
+
# — the writer stops populating it (always ''); the consolidated multi-column
|
|
2413
|
+
# conversation_fts(text, search_tool, search_thinking) indexes these instead.
|
|
2414
|
+
# Idempotent column-adds (no marker, no version); migration 010 backfills the
|
|
2415
|
+
# values onto existing history from blocks_json under the cache.db.lock flock.
|
|
2416
|
+
add_column_if_missing(
|
|
2417
|
+
conn, "conversation_messages", "search_tool", "TEXT NOT NULL DEFAULT ''")
|
|
2418
|
+
add_column_if_missing(
|
|
2419
|
+
conn, "conversation_messages", "search_thinking", "TEXT NOT NULL DEFAULT ''")
|
|
2410
2420
|
conn.execute(
|
|
2411
2421
|
"CREATE INDEX IF NOT EXISTS idx_session_files_session_id "
|
|
2412
2422
|
"ON session_files(session_id)"
|
|
@@ -2417,6 +2427,23 @@ def _apply_cache_schema(conn: sqlite3.Connection) -> None:
|
|
|
2417
2427
|
# would itself error), set a persisted flag, and let search fall back to
|
|
2418
2428
|
# LIKE. Spec §1. Idempotent (IF NOT EXISTS).
|
|
2419
2429
|
if _fts5_available(conn):
|
|
2430
|
+
# #177 S6: if a LEGACY conversation_fts(text) shape is already present
|
|
2431
|
+
# (a pre-S6 install whose migration-010 backfill/swap has not yet run),
|
|
2432
|
+
# leave EVERYTHING untouched — the legacy table, the legacy aux table,
|
|
2433
|
+
# and the legacy triggers. The sync-side swap (_swap_conversation_fts_to_
|
|
2434
|
+
# split, under the cache.db.lock flock) owns the transition; re-applying
|
|
2435
|
+
# the schema must NOT race it or partially mutate the shape. A fresh
|
|
2436
|
+
# install (no conversation_fts) and an already-split shape both fall
|
|
2437
|
+
# through to the create/recover path below.
|
|
2438
|
+
legacy_present = (
|
|
2439
|
+
conn.execute(
|
|
2440
|
+
"SELECT 1 FROM sqlite_master "
|
|
2441
|
+
"WHERE type='table' AND name='conversation_fts'"
|
|
2442
|
+
).fetchone() is not None
|
|
2443
|
+
and not _conversation_fts_is_split(conn))
|
|
2444
|
+
if legacy_present:
|
|
2445
|
+
conn.commit()
|
|
2446
|
+
return
|
|
2420
2447
|
try:
|
|
2421
2448
|
# Recovery (spec §1/P2): if a PRIOR run marked FTS unavailable,
|
|
2422
2449
|
# conversation_messages rows were ingested (by sync_cache / the
|
|
@@ -2425,43 +2452,34 @@ def _apply_cache_schema(conn: sqlite3.Connection) -> None:
|
|
|
2425
2452
|
# rows. Detect that BEFORE clearing the flag so we can rebuild the
|
|
2426
2453
|
# external-content index from conversation_messages below. A fresh
|
|
2427
2454
|
# install never sets the flag, so this stays False and no rebuild
|
|
2428
|
-
# runs (the triggers index rows incrementally as they arrive).
|
|
2455
|
+
# runs (the triggers index rows incrementally as they arrive). This
|
|
2456
|
+
# is ALSO the FTS5-unavailable migration-010 recovery seam: the
|
|
2457
|
+
# backfill ran the base-column UPDATEs but skipped the vtable DDL, so
|
|
2458
|
+
# the first FTS-capable open lands the split shape + rebuilds here.
|
|
2429
2459
|
recovering = conn.execute(
|
|
2430
2460
|
"SELECT 1 FROM cache_meta WHERE key='fts5_unavailable'"
|
|
2431
2461
|
).fetchone() is not None
|
|
2432
|
-
|
|
2433
|
-
|
|
2434
|
-
|
|
2435
|
-
|
|
2436
|
-
#
|
|
2437
|
-
#
|
|
2438
|
-
# which
|
|
2439
|
-
#
|
|
2440
|
-
# tables + BOTH trigger sets (the except arm). A live aux trigger over
|
|
2441
|
-
# a missing conversation_fts_aux would roll back the shared
|
|
2442
|
-
# conversation_messages write transaction (which also carries cost
|
|
2443
|
-
# ingest into session_entries). It is NOT given its own guarded block
|
|
2444
|
-
# or its own flag.
|
|
2445
|
-
_create_conversation_fts_aux_table(conn)
|
|
2446
|
-
# Trigger DDL lives in ONE place (_CONV_FTS_TRIGGER_DDL +
|
|
2447
|
-
# _CONV_FTS_AUX_TRIGGER_DDL) so this initial create and the #138
|
|
2448
|
-
# storm-free full-clear (clear_conversation_messages, which drops +
|
|
2449
|
-
# recreates BOTH trigger sets) can never drift.
|
|
2462
|
+
# #177 S6: the single consolidated multi-column external-content
|
|
2463
|
+
# table replaces the old prose + aux pair. Column names match the
|
|
2464
|
+
# content table BY NAME (external-content rule).
|
|
2465
|
+
conn.execute(_CONV_FTS_SPLIT_DDL)
|
|
2466
|
+
# Trigger DDL lives in ONE place (_CONV_FTS_TRIGGER_DDL) so this
|
|
2467
|
+
# initial create and the #138 storm-free full-clear
|
|
2468
|
+
# (clear_conversation_messages, which drops + recreates the trigger
|
|
2469
|
+
# set) can never drift.
|
|
2450
2470
|
_create_conversation_fts_triggers(conn)
|
|
2451
2471
|
if recovering:
|
|
2452
|
-
# Repopulate the freshly-(re)created
|
|
2472
|
+
# Repopulate the freshly-(re)created index from the base table
|
|
2453
2473
|
# so pre-recovery history is searchable. Cheap no-op when
|
|
2454
2474
|
# conversation_messages is empty.
|
|
2455
2475
|
conn.execute(
|
|
2456
2476
|
"INSERT INTO conversation_fts(conversation_fts) VALUES('rebuild')")
|
|
2457
|
-
conn.execute(
|
|
2458
|
-
"INSERT INTO conversation_fts_aux(conversation_fts_aux) VALUES('rebuild')")
|
|
2459
2477
|
conn.execute("DELETE FROM cache_meta WHERE key='fts5_unavailable'")
|
|
2460
2478
|
except sqlite3.OperationalError:
|
|
2461
|
-
# partial create cleanup, then mark unavailable. _drop drops
|
|
2462
|
-
#
|
|
2463
|
-
#
|
|
2464
|
-
# table.
|
|
2479
|
+
# partial create cleanup, then mark unavailable. _drop drops the
|
|
2480
|
+
# split trigger set (and any legacy aux trigger names, harmlessly),
|
|
2481
|
+
# and we drop both possible vtables, so a failed create can't leave a
|
|
2482
|
+
# live trigger over a missing table.
|
|
2465
2483
|
_drop_conversation_fts_triggers(conn)
|
|
2466
2484
|
try:
|
|
2467
2485
|
conn.execute("DROP TABLE IF EXISTS conversation_fts")
|
|
@@ -2507,9 +2525,36 @@ def _set_cache_meta(conn: sqlite3.Connection, key: str, value: str) -> None:
|
|
|
2507
2525
|
"ON CONFLICT(key) DO UPDATE SET value=excluded.value", (key, value))
|
|
2508
2526
|
|
|
2509
2527
|
|
|
2528
|
+
# #177 S6: the consolidated multi-column external-content FTS5 table that
|
|
2529
|
+
# replaces the old conversation_fts(text) + conversation_fts_aux(search_aux)
|
|
2530
|
+
# pair. The three column names MUST match the conversation_messages columns BY
|
|
2531
|
+
# NAME (external-content FTS5 resolves columns through the content table by
|
|
2532
|
+
# name): a mismatch creates fine but breaks 'rebuild' + iterdump
|
|
2533
|
+
# ("no such column"). Fresh installs create this directly; existing installs
|
|
2534
|
+
# swap to it under the cache.db.lock flock via _swap_conversation_fts_to_split.
|
|
2535
|
+
_CONV_FTS_SPLIT_DDL = (
|
|
2536
|
+
"CREATE VIRTUAL TABLE IF NOT EXISTS conversation_fts USING fts5("
|
|
2537
|
+
"text, search_tool, search_thinking, "
|
|
2538
|
+
"content='conversation_messages', content_rowid='id')"
|
|
2539
|
+
)
|
|
2540
|
+
|
|
2541
|
+
|
|
2542
|
+
def _conversation_fts_is_split(conn: sqlite3.Connection) -> bool:
|
|
2543
|
+
"""True if conversation_fts is the #177 S6 multi-column shape (carries the
|
|
2544
|
+
``search_tool`` column), False for the legacy single-column ``text`` shape or
|
|
2545
|
+
an absent table."""
|
|
2546
|
+
cols = [r[1] for r in conn.execute("PRAGMA table_info(conversation_fts)")]
|
|
2547
|
+
return "search_tool" in cols
|
|
2548
|
+
|
|
2549
|
+
|
|
2510
2550
|
def _create_conversation_fts_aux_table(conn: sqlite3.Connection) -> None:
|
|
2511
2551
|
"""Create the #177 aux external-content FTS5 index over the ``search_aux``
|
|
2512
|
-
blob.
|
|
2552
|
+
blob. TEST/FIXTURE STANDUP ONLY — production code (``_apply_cache_schema``)
|
|
2553
|
+
builds the post-#177-S6 SPLIT index directly; this legacy single-aux-column
|
|
2554
|
+
shape is referenced solely by the migration-fixture builder + the swap tests
|
|
2555
|
+
(the migration's by-name DROP list still legitimately uses the trigger-name
|
|
2556
|
+
tuples, but never calls this helper). A standalone module-level seam (NOT
|
|
2557
|
+
inlined in ``_apply_cache_schema``)
|
|
2513
2558
|
so the all-or-nothing regression test can monkeypatch it to raise
|
|
2514
2559
|
``OperationalError`` AFTER the prose ``conversation_fts`` create succeeded —
|
|
2515
2560
|
proving the shared try-envelope drops BOTH indexes + BOTH trigger sets and a
|
|
@@ -2533,16 +2578,26 @@ def _create_conversation_fts_aux_table(conn: sqlite3.Connection) -> None:
|
|
|
2533
2578
|
# the initial create in _apply_cache_schema and the #138 storm-free full-clear
|
|
2534
2579
|
# in clear_conversation_messages (which drops + recreates them) can never drift.
|
|
2535
2580
|
# conv_fts_ad / conv_fts_au use the external-content `'delete'` idiom.
|
|
2581
|
+
# #177 S6: three-column trigger set for the consolidated conversation_fts. Same
|
|
2582
|
+
# trigger names (conv_fts_ai/ad/au) as the legacy single-column set so the swap
|
|
2583
|
+
# reuses them; the AU trigger now fires AFTER UPDATE OF text, search_tool,
|
|
2584
|
+
# search_thinking (the backfill UPDATEs search_tool/search_thinking, which the
|
|
2585
|
+
# OLD text-only AU did NOT fire — that invisibility is load-bearing, see
|
|
2586
|
+
# _consume_search_split, but it applies only while the legacy triggers are still
|
|
2587
|
+
# installed; once swapped, these fire on all three columns).
|
|
2536
2588
|
_CONV_FTS_TRIGGER_DDL = (
|
|
2537
2589
|
"CREATE TRIGGER IF NOT EXISTS conv_fts_ai AFTER INSERT ON conversation_messages "
|
|
2538
|
-
"BEGIN INSERT INTO conversation_fts(rowid, text
|
|
2590
|
+
"BEGIN INSERT INTO conversation_fts(rowid, text, search_tool, search_thinking) "
|
|
2591
|
+
"VALUES (new.id, new.text, new.search_tool, new.search_thinking); END",
|
|
2539
2592
|
"CREATE TRIGGER IF NOT EXISTS conv_fts_ad AFTER DELETE ON conversation_messages "
|
|
2540
|
-
"BEGIN INSERT INTO conversation_fts(conversation_fts, rowid, text) "
|
|
2541
|
-
"VALUES('delete', old.id, old.text); END",
|
|
2542
|
-
"CREATE TRIGGER IF NOT EXISTS conv_fts_au
|
|
2543
|
-
"
|
|
2544
|
-
"
|
|
2545
|
-
"
|
|
2593
|
+
"BEGIN INSERT INTO conversation_fts(conversation_fts, rowid, text, search_tool, search_thinking) "
|
|
2594
|
+
"VALUES('delete', old.id, old.text, old.search_tool, old.search_thinking); END",
|
|
2595
|
+
"CREATE TRIGGER IF NOT EXISTS conv_fts_au "
|
|
2596
|
+
"AFTER UPDATE OF text, search_tool, search_thinking ON conversation_messages "
|
|
2597
|
+
"BEGIN INSERT INTO conversation_fts(conversation_fts, rowid, text, search_tool, search_thinking) "
|
|
2598
|
+
"VALUES('delete', old.id, old.text, old.search_tool, old.search_thinking); "
|
|
2599
|
+
"INSERT INTO conversation_fts(rowid, text, search_tool, search_thinking) "
|
|
2600
|
+
"VALUES (new.id, new.text, new.search_tool, new.search_thinking); END",
|
|
2546
2601
|
)
|
|
2547
2602
|
# Drop by name (the body is irrelevant to DROP TRIGGER); reverse order is
|
|
2548
2603
|
# cosmetic — order doesn't matter for independent triggers.
|
|
@@ -2567,17 +2622,46 @@ _CONV_FTS_AUX_TRIGGER_DDL = (
|
|
|
2567
2622
|
)
|
|
2568
2623
|
_CONV_FTS_AUX_TRIGGER_NAMES = ("conv_fts_aux_au", "conv_fts_aux_ad", "conv_fts_aux_ai")
|
|
2569
2624
|
|
|
2625
|
+
# #177 S6: the LEGACY single-column prose trigger set, retained ONLY so a
|
|
2626
|
+
# pre-swap install (and the legacy-shape test fixture) can stand up the old
|
|
2627
|
+
# conversation_fts(text) shape. Fresh installs + the post-swap shape use the
|
|
2628
|
+
# three-column _CONV_FTS_TRIGGER_DDL above. Same trigger names, so the swap's
|
|
2629
|
+
# DROP-by-name (_CONV_FTS_TRIGGER_NAMES) covers either generation.
|
|
2630
|
+
_CONV_FTS_LEGACY_TRIGGER_DDL = (
|
|
2631
|
+
"CREATE TRIGGER IF NOT EXISTS conv_fts_ai AFTER INSERT ON conversation_messages "
|
|
2632
|
+
"BEGIN INSERT INTO conversation_fts(rowid, text) VALUES (new.id, new.text); END",
|
|
2633
|
+
"CREATE TRIGGER IF NOT EXISTS conv_fts_ad AFTER DELETE ON conversation_messages "
|
|
2634
|
+
"BEGIN INSERT INTO conversation_fts(conversation_fts, rowid, text) "
|
|
2635
|
+
"VALUES('delete', old.id, old.text); END",
|
|
2636
|
+
"CREATE TRIGGER IF NOT EXISTS conv_fts_au AFTER UPDATE OF text ON conversation_messages "
|
|
2637
|
+
"BEGIN INSERT INTO conversation_fts(conversation_fts, rowid, text) "
|
|
2638
|
+
"VALUES('delete', old.id, old.text); "
|
|
2639
|
+
"INSERT INTO conversation_fts(rowid, text) VALUES (new.id, new.text); END",
|
|
2640
|
+
)
|
|
2641
|
+
|
|
2570
2642
|
|
|
2571
2643
|
def _create_conversation_fts_triggers(conn: sqlite3.Connection) -> None:
|
|
2572
|
-
"""Create
|
|
2573
|
-
|
|
2574
|
-
|
|
2575
|
-
|
|
2576
|
-
|
|
2577
|
-
|
|
2578
|
-
|
|
2644
|
+
"""Create the #177 S6 split conversation_fts(text, search_tool,
|
|
2645
|
+
search_thinking) sync trigger set — idempotent (each is ``IF NOT EXISTS``).
|
|
2646
|
+
Single source of truth for the trigger DDL, shared by ``_apply_cache_schema``
|
|
2647
|
+
and ``clear_conversation_messages`` (#138). The caller must have already
|
|
2648
|
+
created ``conversation_fts`` in the split shape (the consolidated table
|
|
2649
|
+
replaced the old prose+aux pair, so there is no separate aux trigger set to
|
|
2650
|
+
create)."""
|
|
2579
2651
|
for stmt in _CONV_FTS_TRIGGER_DDL:
|
|
2580
2652
|
conn.execute(stmt)
|
|
2653
|
+
|
|
2654
|
+
|
|
2655
|
+
def _create_conversation_fts_legacy_triggers(conn: sqlite3.Connection) -> None:
|
|
2656
|
+
"""Create the LEGACY single-column prose + aux trigger sets over the
|
|
2657
|
+
pre-#177-S6 conversation_fts(text) + conversation_fts_aux(search_aux) pair.
|
|
2658
|
+
TEST/FIXTURE STANDUP ONLY — referenced solely by the migration-fixture
|
|
2659
|
+
builder + the swap tests to materialize the legacy shape the migration then
|
|
2660
|
+
upgrades. Production never calls this (the swap drops these trigger sets BY
|
|
2661
|
+
NAME via the ``_CONV_FTS_*_TRIGGER_NAMES`` tuples and recreates the split
|
|
2662
|
+
set). The caller must have created both legacy tables."""
|
|
2663
|
+
for stmt in _CONV_FTS_LEGACY_TRIGGER_DDL:
|
|
2664
|
+
conn.execute(stmt)
|
|
2581
2665
|
for stmt in _CONV_FTS_AUX_TRIGGER_DDL:
|
|
2582
2666
|
conn.execute(stmt)
|
|
2583
2667
|
|
|
@@ -2639,10 +2723,16 @@ def clear_conversation_messages(conn: sqlite3.Connection) -> None:
|
|
|
2639
2723
|
_drop_conversation_fts_triggers(conn)
|
|
2640
2724
|
conn.execute("DELETE FROM conversation_messages")
|
|
2641
2725
|
conn.execute("INSERT INTO conversation_fts(conversation_fts) VALUES('delete-all')")
|
|
2642
|
-
# #177: the
|
|
2643
|
-
#
|
|
2644
|
-
#
|
|
2645
|
-
|
|
2726
|
+
# #177 S6: the consolidated split table is the only FTS vtable on a swapped /
|
|
2727
|
+
# fresh DB. A pre-swap legacy install still carries conversation_fts_aux, so
|
|
2728
|
+
# reset it the same storm-free way ONLY when it physically exists (the swap
|
|
2729
|
+
# drops it; a fresh/split DB never had it). Triggers were dropped above, so
|
|
2730
|
+
# the base DELETE touched neither index.
|
|
2731
|
+
if conn.execute(
|
|
2732
|
+
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='conversation_fts_aux'"
|
|
2733
|
+
).fetchone() is not None:
|
|
2734
|
+
conn.execute(
|
|
2735
|
+
"INSERT INTO conversation_fts_aux(conversation_fts_aux) VALUES('delete-all')")
|
|
2646
2736
|
_create_conversation_fts_triggers(conn)
|
|
2647
2737
|
|
|
2648
2738
|
|
|
@@ -3225,6 +3315,50 @@ def _009_conversation_media_reingest(conn: sqlite3.Connection) -> None:
|
|
|
3225
3315
|
conn.commit()
|
|
3226
3316
|
|
|
3227
3317
|
|
|
3318
|
+
@cache_migration("010_conversation_search_split")
|
|
3319
|
+
def _010_conversation_search_split(conn: sqlite3.Connection) -> None:
|
|
3320
|
+
"""Flag-only arm for the #177 S6 search-column split. Sets
|
|
3321
|
+
``conversation_search_split_pending``; sync_cache consumes it under the
|
|
3322
|
+
cache.db.lock flock (cursor-resumable blocks_json backfill of
|
|
3323
|
+
search_tool/search_thinking, then an atomic legacy->split FTS swap+rebuild —
|
|
3324
|
+
see _cctally_cache._consume_search_split). The handler does NO data work so
|
|
3325
|
+
the dispatcher's central stamp (#140) marks a genuinely-complete handler; a
|
|
3326
|
+
fresh install stamps it WITHOUT running (its schema is already split via
|
|
3327
|
+
_apply_cache_schema, so the consumer finds no flag and no-ops). Mirrors the
|
|
3328
|
+
flag-only reingest pattern of 002/003/007/009."""
|
|
3329
|
+
_set_cache_meta(conn, "conversation_search_split_pending", "1")
|
|
3330
|
+
conn.commit()
|
|
3331
|
+
|
|
3332
|
+
|
|
3333
|
+
def _swap_conversation_fts_to_split(conn: sqlite3.Connection) -> None:
|
|
3334
|
+
"""Legacy two-table shape -> split multi-column shape, in one transaction.
|
|
3335
|
+
Explicit DROPs of BOTH old tables + triggers (an ``IF NOT EXISTS`` create
|
|
3336
|
+
would silently keep the old same-name single-column conversation_fts — spec
|
|
3337
|
+
F9), then create the split DDL + new triggers and rebuild the index from the
|
|
3338
|
+
base table. Runs under the caller's held cache.db.lock flock (the
|
|
3339
|
+
_consume_search_split end-transaction)."""
|
|
3340
|
+
_drop_conversation_fts_triggers(conn)
|
|
3341
|
+
conn.execute("DROP TABLE IF EXISTS conversation_fts")
|
|
3342
|
+
conn.execute("DROP TABLE IF EXISTS conversation_fts_aux")
|
|
3343
|
+
conn.execute(_CONV_FTS_SPLIT_DDL)
|
|
3344
|
+
_create_conversation_fts_triggers(conn)
|
|
3345
|
+
conn.execute("INSERT INTO conversation_fts(conversation_fts) VALUES('rebuild')")
|
|
3346
|
+
|
|
3347
|
+
|
|
3348
|
+
def conversation_search_depth(conn: sqlite3.Connection) -> str:
|
|
3349
|
+
"""'prose-only' while migration 010's backfill is pending, else 'full'.
|
|
3350
|
+
Surfaced additively on the search/find responses so the client can degrade
|
|
3351
|
+
the Tools/Thinking facets during the one-time index split. An
|
|
3352
|
+
OperationalError (no cache_meta table yet) degrades to 'full'."""
|
|
3353
|
+
try:
|
|
3354
|
+
pending = conn.execute(
|
|
3355
|
+
"SELECT 1 FROM cache_meta WHERE key='conversation_search_split_pending'"
|
|
3356
|
+
).fetchone()
|
|
3357
|
+
except sqlite3.OperationalError:
|
|
3358
|
+
return "full"
|
|
3359
|
+
return "prose-only" if pending else "full"
|
|
3360
|
+
|
|
3361
|
+
|
|
3228
3362
|
# === Region 7d: Stats migration 008_recompute_weekly_cost_snapshots_dedup_fix ===
|
|
3229
3363
|
|
|
3230
3364
|
@stats_migration("008_recompute_weekly_cost_snapshots_dedup_fix")
|