cctally 1.39.0 → 1.41.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -5,6 +5,17 @@ based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
5
5
 
6
6
  ## [Unreleased]
7
7
 
8
+ ## [1.41.0] - 2026-06-13
9
+
10
+ ### Added
11
+ - Conversation viewer: search now reaches past prose into commands, file paths, error strings, and the assistant's thinking, with exact kind facets and a browser-style in-conversation find bar — the rail gains an `All · Prompts · Assistant · Tools · Thinking` chip row, a result count line, match badges (`tool`, `thinking`), and a `Load N more` button, while pressing `/` over an open conversation opens a floating find pill (`Enter`/`Shift+Enter` and `n`/`N` step matches, `Esc` closes, wrap-around, exact `k / N` counter, `<mark>` term highlighting in prose) backed by a new `GET /api/conversation/<id>/find` endpoint and a `kind` param on `/api/conversation/search`; the tool/thinking facets light up after a one-time, lossless search-index split (cache migration `010`) runs automatically under the cache lock on the next sync, reporting `search_depth: "prose-only"` (with the Tools/Thinking chips disabled and an `indexing…` hint) until it completes, and degrading to a basic substring scan on hosts without FTS5 (#177 S6).
12
+
13
+ ## [1.40.0] - 2026-06-12
14
+
15
+ ### Added
16
+ - Conversation viewer: a full-session navigation layer — a collapsible outline sidebar (toggle `o`) listing every turn as a landmark with nested thinking entries and scroll-sync highlighting, a stats overview (turn counts, duration, tokens, cost, models, tool histogram, and an error row that jumps to the first error), jump-to-next keys with no wrap-around (`e`/`E` errors, `u`/`U` prompts, `b`/`B` subagents, `p`/`P` plans/questions) mirrored by a clickable glyph cluster with counts, and focus modes (`v` cycles All → Chat → Prompts → Errors; hidden turns coalesce into a clickable `· N hidden ·` marker, and a jump whose target the mode would hide resets to All before landing) (#177 Session 5).
17
+ - Conversation viewer: per-turn timestamps and inter-turn markers — each turn header shows a quiet `· HH:mm` in your `display.tz` (precise instant in a tooltip), with a `⏸ 42 min later` gap rule between turns ten or more minutes apart, a `— Jun 13 —` rule when the calendar day changes, and a combined `⏸ 9.5 h later · Jun 13` when both apply — plus per-turn token footers that extend the assistant cost line to `$0.0214 · in 1.2k · out 4.8k · cache 310k` (tokens-only when a turn has usage but no attributable cost, cost-only for un-reingested turns) (#177 Session 5).
18
+
8
19
  ## [1.39.0] - 2026-06-12
9
20
 
10
21
  ### Added
@@ -184,8 +184,9 @@ _CONV_INSERT_SQL = (
184
184
  "(session_id,uuid,parent_uuid,source_path,byte_offset,"
185
185
  " timestamp_utc,entry_type,text,blocks_json,model,msg_id,"
186
186
  " req_id,cwd,git_branch,is_sidechain,source_tool_use_id,"
187
- " stop_reason,attribution_skill,attribution_plugin,search_aux)"
188
- " VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
187
+ " stop_reason,attribution_skill,attribution_plugin,"
188
+ " search_tool,search_thinking,search_aux)"
189
+ " VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
189
190
  )
190
191
 
191
192
 
@@ -193,15 +194,19 @@ def _conv_row_tuple(m, path_str):
193
194
  """Flatten a ``MessageRow`` into the ``_CONV_INSERT_SQL`` column order.
194
195
 
195
196
  The #177 enrichment fields (stop_reason / attribution_skill /
196
- attribution_plugin / search_aux) are TAIL-APPENDED after source_tool_use_id
197
- — same order as the SQL column list — so both ingest paths (fused per-file
198
- walk + backfill_conversation_messages) carry them through this one tuple."""
197
+ attribution_plugin / search_tool / search_thinking / search_aux) are
198
+ TAIL-APPENDED after source_tool_use_id — same order as the SQL column list —
199
+ so both ingest paths (fused per-file walk + backfill_conversation_messages)
200
+ carry them through this one tuple. #177 S6: ``search_aux`` is always written
201
+ as ``''`` (documented-dead); the split ``search_tool``/``search_thinking``
202
+ columns carry the non-prose index."""
199
203
  return (
200
204
  m.session_id, m.uuid, m.parent_uuid, path_str, m.byte_offset,
201
205
  m.timestamp_utc, m.entry_type, m.text, m.blocks_json, m.model,
202
206
  m.msg_id, m.req_id, m.cwd, m.git_branch, m.is_sidechain,
203
207
  m.source_tool_use_id,
204
- m.stop_reason, m.attribution_skill, m.attribution_plugin, m.search_aux,
208
+ m.stop_reason, m.attribution_skill, m.attribution_plugin,
209
+ m.search_tool, m.search_thinking, m.search_aux,
205
210
  )
206
211
 
207
212
 
@@ -651,6 +656,31 @@ def sync_cache(
651
656
  " 'conversation_media_reingest_pending',"
652
657
  " 'conversation_reingest_cursor',"
653
658
  " 'conversation_reingest_cursor_gen')")
659
+ # #177 S6: a rebuild repopulates search_tool/search_thinking via the
660
+ # offset-0 walk (the parser derives them), so the migration-010
661
+ # backfill is redundant. But a LEGACY-shape DB still carries the old
662
+ # prose+aux FTS pair that the split triggers can't write — swap to the
663
+ # split shape NOW (the table is empty post-clear, so the walk below
664
+ # populates it through the new triggers), then drop the pending flag +
665
+ # cursor so the post-rebuild sync runs no redundant backfill/swap.
666
+ # MISSING this site re-arms the flag on every cache-sync --rebuild.
667
+ try:
668
+ _split_pending = conn.execute(
669
+ "SELECT 1 FROM cache_meta "
670
+ "WHERE key='conversation_search_split_pending'"
671
+ ).fetchone() is not None
672
+ except sqlite3.OperationalError:
673
+ _split_pending = False
674
+ if _split_pending:
675
+ _fts_off = conn.execute(
676
+ "SELECT 1 FROM cache_meta WHERE key='fts5_unavailable'"
677
+ ).fetchone() is not None
678
+ if not _fts_off and not _cctally_db_sib._conversation_fts_is_split(conn):
679
+ _cctally_db_sib._swap_conversation_fts_to_split(conn)
680
+ conn.execute(
681
+ "DELETE FROM cache_meta WHERE key IN "
682
+ "('conversation_search_split_pending',"
683
+ " 'conversation_search_split_cursor')")
654
684
  conn.commit()
655
685
  eprint("[cache-sync] rebuild: cleared Claude cached entries")
656
686
 
@@ -742,6 +772,14 @@ def sync_cache(
742
772
  # path (which already wipes + repopulates id-aware via the normal walk).
743
773
  _resumable_reingest_conversation_messages(conn)
744
774
 
775
+ # #177 S6: consume the migration-010 search-column split under the
776
+ # SAME held flock, AFTER the reingest so any just-re-ingested rows
777
+ # already carry fresh search_tool/search_thinking before the backfill
778
+ # touches the tail. Cursor-resumable; the legacy triggers are blind to
779
+ # the search_tool/search_thinking UPDATEs (they fire on text only), so
780
+ # old search keeps working until the final swap.
781
+ _consume_search_split(conn)
782
+
745
783
  paths: list[pathlib.Path] = list(_iter_claude_jsonl_files())
746
784
  stats.files_total = len(paths)
747
785
 
@@ -1260,6 +1298,65 @@ def _resumable_reingest_conversation_messages(conn):
1260
1298
  conn.commit()
1261
1299
 
1262
1300
 
1301
+ def _consume_search_split(conn) -> None:
1302
+ """#177 S6: flock-held consumer for ``conversation_search_split_pending``
1303
+ (set by cache migration 010). Cursor-resumable: backfills
1304
+ search_tool/search_thinking from each row's ``blocks_json`` via the SHARED
1305
+ ``_lib_conversation._derive_search_columns`` chokepoint (so the values are
1306
+ byte-identical to live ingest), checkpointing
1307
+ ``conversation_search_split_cursor`` per 500-row batch. These UPDATEs are
1308
+ INVISIBLE to the LEGACY triggers (which fire on text/search_aux only), so the
1309
+ old prose search keeps working untouched until the final swap (spec F5).
1310
+
1311
+ When the cursor completes, swap the legacy two-table FTS to the consolidated
1312
+ split shape + rebuild (one short transaction), then delete the pending +
1313
+ cursor meta keys. FTS5-unavailable (``fts5_unavailable`` set): the base-column
1314
+ backfill still runs (it is FTS-independent), the vtable swap is SKIPPED, the
1315
+ flag still clears, and the rebuild-on-availability recovery path
1316
+ (_apply_cache_schema) lands the split shape later (spec F6). Interrupted at
1317
+ any point ⇒ resumes from the cursor on the next locked sync; a fresh install
1318
+ never sets the flag so this is a cheap no-op there."""
1319
+ if conn.execute(
1320
+ "SELECT 1 FROM cache_meta WHERE key='conversation_search_split_pending'"
1321
+ ).fetchone() is None:
1322
+ return
1323
+ row = conn.execute(
1324
+ "SELECT value FROM cache_meta WHERE key='conversation_search_split_cursor'"
1325
+ ).fetchone()
1326
+ last_id = int(row[0]) if row else 0
1327
+ while True:
1328
+ batch = conn.execute(
1329
+ "SELECT id, blocks_json FROM conversation_messages "
1330
+ "WHERE id > ? ORDER BY id LIMIT 500",
1331
+ (last_id,)).fetchall()
1332
+ if not batch:
1333
+ break
1334
+ ups = []
1335
+ for rid, bj in batch:
1336
+ try:
1337
+ blocks = json.loads(bj) if bj else []
1338
+ except (TypeError, ValueError):
1339
+ blocks = []
1340
+ st, sth = _lib_conversation._derive_search_columns(blocks)
1341
+ ups.append((st, sth, rid))
1342
+ last_id = rid
1343
+ conn.executemany(
1344
+ "UPDATE conversation_messages SET search_tool=?, search_thinking=? "
1345
+ "WHERE id=?", ups)
1346
+ _cctally_db_sib._set_cache_meta(
1347
+ conn, "conversation_search_split_cursor", str(last_id))
1348
+ conn.commit()
1349
+ fts_off = conn.execute(
1350
+ "SELECT 1 FROM cache_meta WHERE key='fts5_unavailable'"
1351
+ ).fetchone() is not None
1352
+ if not fts_off:
1353
+ _cctally_db_sib._swap_conversation_fts_to_split(conn)
1354
+ conn.execute(
1355
+ "DELETE FROM cache_meta WHERE key IN "
1356
+ "('conversation_search_split_pending','conversation_search_split_cursor')")
1357
+ conn.commit()
1358
+
1359
+
1263
1360
  def iter_entries(
1264
1361
  conn: sqlite3.Connection,
1265
1362
  range_start: dt.datetime,
@@ -5218,6 +5218,12 @@ def _qs_str(q: dict, key: str, default: str | None) -> str | None:
5218
5218
  return vals[0] if vals else default
5219
5219
 
5220
5220
 
5221
+ # #177 S6: valid kind facets for the conversation search / find routes. Kept in
5222
+ # lockstep with ``_lib_conversation_query._SEARCH_KINDS`` (the kernel re-raises
5223
+ # ValueError on an unknown kind; the handlers reject with a 400 before the call).
5224
+ _CONV_SEARCH_KINDS = ("all", "prompts", "assistant", "tools", "thinking")
5225
+
5226
+
5221
5227
  class DashboardHTTPHandler(BaseHTTPRequestHandler):
5222
5228
  """Routes:
5223
5229
  GET / → dashboard.html
@@ -5348,6 +5354,14 @@ class DashboardHTTPHandler(BaseHTTPRequestHandler):
5348
5354
  elif path.startswith("/api/conversation/") and path.endswith("/media"):
5349
5355
  # #177 S4: on-demand media bytes. Matched BEFORE the <id> reader.
5350
5356
  self._handle_get_conversation_media(path)
5357
+ elif path.startswith("/api/conversation/") and path.endswith("/outline"):
5358
+ # #177 S5: full-session outline skeleton + stats. Matched BEFORE
5359
+ # the <id> reader catch-all (Codex F2 — same precedence as /payload).
5360
+ self._handle_get_conversation_outline(path)
5361
+ elif path.startswith("/api/conversation/") and path.endswith("/find"):
5362
+ # #177 S6: in-conversation find → rendered-turn anchors. Matched
5363
+ # BEFORE the <id> reader catch-all (same precedence as /outline).
5364
+ self._handle_get_conversation_find(path)
5351
5365
  elif path.startswith("/api/conversation/"):
5352
5366
  self._handle_get_conversation_detail(path)
5353
5367
  else:
@@ -7250,6 +7264,19 @@ class DashboardHTTPHandler(BaseHTTPRequestHandler):
7250
7264
  """Lazy-load the pure conversation query kernel (Plan 2, §3)."""
7251
7265
  return sys.modules["cctally"]._load_sibling("_lib_conversation_query")
7252
7266
 
7267
+ def _parse_search_kind(self, q):
7268
+ """Read + validate the ``kind`` facet shared by the conversation search
7269
+ and find routes (#177 S6). Returns the kind on success, or ``None`` after
7270
+ having ALREADY sent a 400 — callers just ``return`` on ``None``. Kept in
7271
+ lockstep with the kernel's ``_SEARCH_KINDS`` via ``_CONV_SEARCH_KINDS``
7272
+ (the kernel module is resolved lazily per-request, so the handler keeps a
7273
+ literal tuple rather than reaching across that import edge for a nit)."""
7274
+ kind = _qs_str(q, "kind", "all")
7275
+ if kind not in _CONV_SEARCH_KINDS:
7276
+ self._respond_json(400, {"error": f"unknown kind: {kind}"})
7277
+ return None
7278
+ return kind
7279
+
7253
7280
  def _run_conversation_query(self, kernel_call, log_label):
7254
7281
  """Open cache.db, run ``kernel_call(conn)``, close — with the uniform
7255
7282
  500 envelopes the three conversation routes share (#151).
@@ -7333,8 +7360,10 @@ class DashboardHTTPHandler(BaseHTTPRequestHandler):
7333
7360
  self._respond_json(200, body)
7334
7361
 
7335
7362
  def _handle_get_conversation_search(self) -> None:
7336
- """``GET /api/conversation/search?q=...`` — cross-session FTS/LIKE
7337
- search (spec §3.3). Matched BEFORE the ``<id>`` reader in ``do_GET``.
7363
+ """``GET /api/conversation/search?q=...&kind=...`` — cross-session
7364
+ FTS/LIKE search (spec §3.3). Matched BEFORE the ``<id>`` reader in
7365
+ ``do_GET``. ``kind`` (#177 S6) is validated to ``_CONV_SEARCH_KINDS``
7366
+ (else 400) before the kernel call.
7338
7367
  """
7339
7368
  if not self._require_transcripts_allowed():
7340
7369
  return
@@ -7343,9 +7372,12 @@ class DashboardHTTPHandler(BaseHTTPRequestHandler):
7343
7372
  query = _qs_str(q, "q", "")
7344
7373
  limit = _qs_int(q, "limit", 50)
7345
7374
  offset = _qs_int(q, "offset", 0)
7375
+ kind = self._parse_search_kind(q)
7376
+ if kind is None:
7377
+ return
7346
7378
  ok, body = self._run_conversation_query(
7347
7379
  lambda conn: self._conversation_query().search_conversations(
7348
- conn, query, limit=limit, offset=offset),
7380
+ conn, query, limit=limit, offset=offset, kind=kind),
7349
7381
  "/api/conversation/search")
7350
7382
  if not ok:
7351
7383
  return
@@ -7391,6 +7423,56 @@ class DashboardHTTPHandler(BaseHTTPRequestHandler):
7391
7423
  return
7392
7424
  self._respond_json(200, payload)
7393
7425
 
7426
+ def _handle_get_conversation_outline(self, path: str) -> None:
7427
+ """``GET /api/conversation/<sid>/outline`` — full-session skeleton +
7428
+ session stats (#177 S5). Same fail-closed privacy gate; unknown id → 404.
7429
+ """
7430
+ if not self._require_transcripts_allowed():
7431
+ return
7432
+ import urllib.parse as _u
7433
+ session_id = _u.unquote(path[len("/api/conversation/"):-len("/outline")])
7434
+ if not session_id:
7435
+ self.send_error(404, "conversation not found")
7436
+ return
7437
+ ok, body = self._run_conversation_query(
7438
+ lambda conn: self._conversation_query().get_conversation_outline(conn, session_id),
7439
+ "/api/conversation/outline")
7440
+ if not ok:
7441
+ return
7442
+ if body is None:
7443
+ self.send_error(404, "conversation not found")
7444
+ return
7445
+ self._respond_json(200, body)
7446
+
7447
+ def _handle_get_conversation_find(self, path: str) -> None:
7448
+ """``GET /api/conversation/<sid>/find?q=...&kind=...`` — in-conversation
7449
+ find → document-ordered rendered-turn anchors (#177 S6). Same fail-closed
7450
+ privacy gate as the sibling routes; unknown id → 404; an invalid ``kind``
7451
+ → 400. Matched BEFORE the ``<id>`` reader catch-all in ``do_GET``.
7452
+ """
7453
+ if not self._require_transcripts_allowed():
7454
+ return
7455
+ import urllib.parse as _u
7456
+ session_id = _u.unquote(path[len("/api/conversation/"):-len("/find")])
7457
+ if not session_id:
7458
+ self.send_error(404, "conversation not found")
7459
+ return
7460
+ q = _u.parse_qs(self.path.partition("?")[2])
7461
+ query = _qs_str(q, "q", "")
7462
+ kind = self._parse_search_kind(q)
7463
+ if kind is None:
7464
+ return
7465
+ ok, body = self._run_conversation_query(
7466
+ lambda conn: self._conversation_query().find_in_conversation(
7467
+ conn, session_id, query, kind=kind),
7468
+ "/api/conversation/find")
7469
+ if not ok:
7470
+ return
7471
+ if body is None:
7472
+ self.send_error(404, "conversation not found")
7473
+ return
7474
+ self._respond_json(200, body)
7475
+
7394
7476
  _MEDIA_FETCH_SITE_ALLOWED = ("same-origin", "same-site", "none")
7395
7477
 
7396
7478
  def _handle_get_conversation_media(self, path: str) -> None:
@@ -2407,6 +2407,16 @@ def _apply_cache_schema(conn: sqlite3.Connection) -> None:
2407
2407
  add_column_if_missing(conn, "conversation_messages", "attribution_plugin", "TEXT")
2408
2408
  add_column_if_missing(
2409
2409
  conn, "conversation_messages", "search_aux", "TEXT NOT NULL DEFAULT ''")
2410
+ # #177 S6: split the non-prose search index into two columns so kind facets
2411
+ # (Tools / Thinking) are exact in SQL. ``search_aux`` above is documented-dead
2412
+ # — the writer stops populating it (always ''); the consolidated multi-column
2413
+ # conversation_fts(text, search_tool, search_thinking) indexes these instead.
2414
+ # Idempotent column-adds (no marker, no version); migration 010 backfills the
2415
+ # values onto existing history from blocks_json under the cache.db.lock flock.
2416
+ add_column_if_missing(
2417
+ conn, "conversation_messages", "search_tool", "TEXT NOT NULL DEFAULT ''")
2418
+ add_column_if_missing(
2419
+ conn, "conversation_messages", "search_thinking", "TEXT NOT NULL DEFAULT ''")
2410
2420
  conn.execute(
2411
2421
  "CREATE INDEX IF NOT EXISTS idx_session_files_session_id "
2412
2422
  "ON session_files(session_id)"
@@ -2417,6 +2427,23 @@ def _apply_cache_schema(conn: sqlite3.Connection) -> None:
2417
2427
  # would itself error), set a persisted flag, and let search fall back to
2418
2428
  # LIKE. Spec §1. Idempotent (IF NOT EXISTS).
2419
2429
  if _fts5_available(conn):
2430
+ # #177 S6: if a LEGACY conversation_fts(text) shape is already present
2431
+ # (a pre-S6 install whose migration-010 backfill/swap has not yet run),
2432
+ # leave EVERYTHING untouched — the legacy table, the legacy aux table,
2433
+ # and the legacy triggers. The sync-side swap (_swap_conversation_fts_to_
2434
+ # split, under the cache.db.lock flock) owns the transition; re-applying
2435
+ # the schema must NOT race it or partially mutate the shape. A fresh
2436
+ # install (no conversation_fts) and an already-split shape both fall
2437
+ # through to the create/recover path below.
2438
+ legacy_present = (
2439
+ conn.execute(
2440
+ "SELECT 1 FROM sqlite_master "
2441
+ "WHERE type='table' AND name='conversation_fts'"
2442
+ ).fetchone() is not None
2443
+ and not _conversation_fts_is_split(conn))
2444
+ if legacy_present:
2445
+ conn.commit()
2446
+ return
2420
2447
  try:
2421
2448
  # Recovery (spec §1/P2): if a PRIOR run marked FTS unavailable,
2422
2449
  # conversation_messages rows were ingested (by sync_cache / the
@@ -2425,43 +2452,34 @@ def _apply_cache_schema(conn: sqlite3.Connection) -> None:
2425
2452
  # rows. Detect that BEFORE clearing the flag so we can rebuild the
2426
2453
  # external-content index from conversation_messages below. A fresh
2427
2454
  # install never sets the flag, so this stays False and no rebuild
2428
- # runs (the triggers index rows incrementally as they arrive).
2455
+ # runs (the triggers index rows incrementally as they arrive). This
2456
+ # is ALSO the FTS5-unavailable migration-010 recovery seam: the
2457
+ # backfill ran the base-column UPDATEs but skipped the vtable DDL, so
2458
+ # the first FTS-capable open lands the split shape + rebuilds here.
2429
2459
  recovering = conn.execute(
2430
2460
  "SELECT 1 FROM cache_meta WHERE key='fts5_unavailable'"
2431
2461
  ).fetchone() is not None
2432
- conn.execute(
2433
- "CREATE VIRTUAL TABLE IF NOT EXISTS conversation_fts "
2434
- "USING fts5(text, content='conversation_messages', content_rowid='id')")
2435
- # #177: the aux external-content index is ALL-OR-NOTHING with the
2436
- # prose FTS under the SINGLE fts5_unavailable flag created in this
2437
- # SAME try-envelope (via the _create_conversation_fts_aux_table seam,
2438
- # which tests monkeypatch to simulate an aux-create failure AFTER the
2439
- # prose create succeeded), so ANY OperationalError below drops BOTH
2440
- # tables + BOTH trigger sets (the except arm). A live aux trigger over
2441
- # a missing conversation_fts_aux would roll back the shared
2442
- # conversation_messages write transaction (which also carries cost
2443
- # ingest into session_entries). It is NOT given its own guarded block
2444
- # or its own flag.
2445
- _create_conversation_fts_aux_table(conn)
2446
- # Trigger DDL lives in ONE place (_CONV_FTS_TRIGGER_DDL +
2447
- # _CONV_FTS_AUX_TRIGGER_DDL) so this initial create and the #138
2448
- # storm-free full-clear (clear_conversation_messages, which drops +
2449
- # recreates BOTH trigger sets) can never drift.
2462
+ # #177 S6: the single consolidated multi-column external-content
2463
+ # table replaces the old prose + aux pair. Column names match the
2464
+ # content table BY NAME (external-content rule).
2465
+ conn.execute(_CONV_FTS_SPLIT_DDL)
2466
+ # Trigger DDL lives in ONE place (_CONV_FTS_TRIGGER_DDL) so this
2467
+ # initial create and the #138 storm-free full-clear
2468
+ # (clear_conversation_messages, which drops + recreates the trigger
2469
+ # set) can never drift.
2450
2470
  _create_conversation_fts_triggers(conn)
2451
2471
  if recovering:
2452
- # Repopulate the freshly-(re)created indexes from the base table
2472
+ # Repopulate the freshly-(re)created index from the base table
2453
2473
  # so pre-recovery history is searchable. Cheap no-op when
2454
2474
  # conversation_messages is empty.
2455
2475
  conn.execute(
2456
2476
  "INSERT INTO conversation_fts(conversation_fts) VALUES('rebuild')")
2457
- conn.execute(
2458
- "INSERT INTO conversation_fts_aux(conversation_fts_aux) VALUES('rebuild')")
2459
2477
  conn.execute("DELETE FROM cache_meta WHERE key='fts5_unavailable'")
2460
2478
  except sqlite3.OperationalError:
2461
- # partial create cleanup, then mark unavailable. _drop drops BOTH
2462
- # prose + aux trigger sets (#177), and we drop BOTH vtables, so a
2463
- # failed aux create can't leave a live aux trigger over a missing
2464
- # table.
2479
+ # partial create cleanup, then mark unavailable. _drop drops the
2480
+ # split trigger set (and any legacy aux trigger names, harmlessly),
2481
+ # and we drop both possible vtables, so a failed create can't leave a
2482
+ # live trigger over a missing table.
2465
2483
  _drop_conversation_fts_triggers(conn)
2466
2484
  try:
2467
2485
  conn.execute("DROP TABLE IF EXISTS conversation_fts")
@@ -2507,9 +2525,36 @@ def _set_cache_meta(conn: sqlite3.Connection, key: str, value: str) -> None:
2507
2525
  "ON CONFLICT(key) DO UPDATE SET value=excluded.value", (key, value))
2508
2526
 
2509
2527
 
2528
+ # #177 S6: the consolidated multi-column external-content FTS5 table that
2529
+ # replaces the old conversation_fts(text) + conversation_fts_aux(search_aux)
2530
+ # pair. The three column names MUST match the conversation_messages columns BY
2531
+ # NAME (external-content FTS5 resolves columns through the content table by
2532
+ # name): a mismatch creates fine but breaks 'rebuild' + iterdump
2533
+ # ("no such column"). Fresh installs create this directly; existing installs
2534
+ # swap to it under the cache.db.lock flock via _swap_conversation_fts_to_split.
2535
+ _CONV_FTS_SPLIT_DDL = (
2536
+ "CREATE VIRTUAL TABLE IF NOT EXISTS conversation_fts USING fts5("
2537
+ "text, search_tool, search_thinking, "
2538
+ "content='conversation_messages', content_rowid='id')"
2539
+ )
2540
+
2541
+
2542
+ def _conversation_fts_is_split(conn: sqlite3.Connection) -> bool:
2543
+ """True if conversation_fts is the #177 S6 multi-column shape (carries the
2544
+ ``search_tool`` column), False for the legacy single-column ``text`` shape or
2545
+ an absent table."""
2546
+ cols = [r[1] for r in conn.execute("PRAGMA table_info(conversation_fts)")]
2547
+ return "search_tool" in cols
2548
+
2549
+
2510
2550
  def _create_conversation_fts_aux_table(conn: sqlite3.Connection) -> None:
2511
2551
  """Create the #177 aux external-content FTS5 index over the ``search_aux``
2512
- blob. A standalone module-level seam (NOT inlined in ``_apply_cache_schema``)
2552
+ blob. TEST/FIXTURE STANDUP ONLY production code (``_apply_cache_schema``)
2553
+ builds the post-#177-S6 SPLIT index directly; this legacy single-aux-column
2554
+ shape is referenced solely by the migration-fixture builder + the swap tests
2555
+ (the migration's by-name DROP list still legitimately uses the trigger-name
2556
+ tuples, but never calls this helper). A standalone module-level seam (NOT
2557
+ inlined in ``_apply_cache_schema``)
2513
2558
  so the all-or-nothing regression test can monkeypatch it to raise
2514
2559
  ``OperationalError`` AFTER the prose ``conversation_fts`` create succeeded —
2515
2560
  proving the shared try-envelope drops BOTH indexes + BOTH trigger sets and a
@@ -2533,16 +2578,26 @@ def _create_conversation_fts_aux_table(conn: sqlite3.Connection) -> None:
2533
2578
  # the initial create in _apply_cache_schema and the #138 storm-free full-clear
2534
2579
  # in clear_conversation_messages (which drops + recreates them) can never drift.
2535
2580
  # conv_fts_ad / conv_fts_au use the external-content `'delete'` idiom.
2581
+ # #177 S6: three-column trigger set for the consolidated conversation_fts. Same
2582
+ # trigger names (conv_fts_ai/ad/au) as the legacy single-column set so the swap
2583
+ # reuses them; the AU trigger now fires AFTER UPDATE OF text, search_tool,
2584
+ # search_thinking (the backfill UPDATEs search_tool/search_thinking, which the
2585
+ # OLD text-only AU did NOT fire — that invisibility is load-bearing, see
2586
+ # _consume_search_split, but it applies only while the legacy triggers are still
2587
+ # installed; once swapped, these fire on all three columns).
2536
2588
  _CONV_FTS_TRIGGER_DDL = (
2537
2589
  "CREATE TRIGGER IF NOT EXISTS conv_fts_ai AFTER INSERT ON conversation_messages "
2538
- "BEGIN INSERT INTO conversation_fts(rowid, text) VALUES (new.id, new.text); END",
2590
+ "BEGIN INSERT INTO conversation_fts(rowid, text, search_tool, search_thinking) "
2591
+ "VALUES (new.id, new.text, new.search_tool, new.search_thinking); END",
2539
2592
  "CREATE TRIGGER IF NOT EXISTS conv_fts_ad AFTER DELETE ON conversation_messages "
2540
- "BEGIN INSERT INTO conversation_fts(conversation_fts, rowid, text) "
2541
- "VALUES('delete', old.id, old.text); END",
2542
- "CREATE TRIGGER IF NOT EXISTS conv_fts_au AFTER UPDATE OF text ON conversation_messages "
2543
- "BEGIN INSERT INTO conversation_fts(conversation_fts, rowid, text) "
2544
- "VALUES('delete', old.id, old.text); "
2545
- "INSERT INTO conversation_fts(rowid, text) VALUES (new.id, new.text); END",
2593
+ "BEGIN INSERT INTO conversation_fts(conversation_fts, rowid, text, search_tool, search_thinking) "
2594
+ "VALUES('delete', old.id, old.text, old.search_tool, old.search_thinking); END",
2595
+ "CREATE TRIGGER IF NOT EXISTS conv_fts_au "
2596
+ "AFTER UPDATE OF text, search_tool, search_thinking ON conversation_messages "
2597
+ "BEGIN INSERT INTO conversation_fts(conversation_fts, rowid, text, search_tool, search_thinking) "
2598
+ "VALUES('delete', old.id, old.text, old.search_tool, old.search_thinking); "
2599
+ "INSERT INTO conversation_fts(rowid, text, search_tool, search_thinking) "
2600
+ "VALUES (new.id, new.text, new.search_tool, new.search_thinking); END",
2546
2601
  )
2547
2602
  # Drop by name (the body is irrelevant to DROP TRIGGER); reverse order is
2548
2603
  # cosmetic — order doesn't matter for independent triggers.
@@ -2567,17 +2622,46 @@ _CONV_FTS_AUX_TRIGGER_DDL = (
2567
2622
  )
2568
2623
  _CONV_FTS_AUX_TRIGGER_NAMES = ("conv_fts_aux_au", "conv_fts_aux_ad", "conv_fts_aux_ai")
2569
2624
 
2625
+ # #177 S6: the LEGACY single-column prose trigger set, retained ONLY so a
2626
+ # pre-swap install (and the legacy-shape test fixture) can stand up the old
2627
+ # conversation_fts(text) shape. Fresh installs + the post-swap shape use the
2628
+ # three-column _CONV_FTS_TRIGGER_DDL above. Same trigger names, so the swap's
2629
+ # DROP-by-name (_CONV_FTS_TRIGGER_NAMES) covers either generation.
2630
+ _CONV_FTS_LEGACY_TRIGGER_DDL = (
2631
+ "CREATE TRIGGER IF NOT EXISTS conv_fts_ai AFTER INSERT ON conversation_messages "
2632
+ "BEGIN INSERT INTO conversation_fts(rowid, text) VALUES (new.id, new.text); END",
2633
+ "CREATE TRIGGER IF NOT EXISTS conv_fts_ad AFTER DELETE ON conversation_messages "
2634
+ "BEGIN INSERT INTO conversation_fts(conversation_fts, rowid, text) "
2635
+ "VALUES('delete', old.id, old.text); END",
2636
+ "CREATE TRIGGER IF NOT EXISTS conv_fts_au AFTER UPDATE OF text ON conversation_messages "
2637
+ "BEGIN INSERT INTO conversation_fts(conversation_fts, rowid, text) "
2638
+ "VALUES('delete', old.id, old.text); "
2639
+ "INSERT INTO conversation_fts(rowid, text) VALUES (new.id, new.text); END",
2640
+ )
2641
+
2570
2642
 
2571
2643
  def _create_conversation_fts_triggers(conn: sqlite3.Connection) -> None:
2572
- """Create BOTH external-content FTS5 sync trigger sets — prose
2573
- (conversation_fts) + aux (conversation_fts_aux, #177) — idempotent (each is
2574
- ``IF NOT EXISTS``). Single source of truth for the trigger DDL, shared by
2575
- ``_apply_cache_schema`` and ``clear_conversation_messages`` (#138). The
2576
- caller must have already created both ``conversation_fts`` and
2577
- ``conversation_fts_aux``; one call site creates both sets so they can never
2578
- drift."""
2644
+ """Create the #177 S6 split conversation_fts(text, search_tool,
2645
+ search_thinking) sync trigger set — idempotent (each is ``IF NOT EXISTS``).
2646
+ Single source of truth for the trigger DDL, shared by ``_apply_cache_schema``
2647
+ and ``clear_conversation_messages`` (#138). The caller must have already
2648
+ created ``conversation_fts`` in the split shape (the consolidated table
2649
+ replaced the old prose+aux pair, so there is no separate aux trigger set to
2650
+ create)."""
2579
2651
  for stmt in _CONV_FTS_TRIGGER_DDL:
2580
2652
  conn.execute(stmt)
2653
+
2654
+
2655
+ def _create_conversation_fts_legacy_triggers(conn: sqlite3.Connection) -> None:
2656
+ """Create the LEGACY single-column prose + aux trigger sets over the
2657
+ pre-#177-S6 conversation_fts(text) + conversation_fts_aux(search_aux) pair.
2658
+ TEST/FIXTURE STANDUP ONLY — referenced solely by the migration-fixture
2659
+ builder + the swap tests to materialize the legacy shape the migration then
2660
+ upgrades. Production never calls this (the swap drops these trigger sets BY
2661
+ NAME via the ``_CONV_FTS_*_TRIGGER_NAMES`` tuples and recreates the split
2662
+ set). The caller must have created both legacy tables."""
2663
+ for stmt in _CONV_FTS_LEGACY_TRIGGER_DDL:
2664
+ conn.execute(stmt)
2581
2665
  for stmt in _CONV_FTS_AUX_TRIGGER_DDL:
2582
2666
  conn.execute(stmt)
2583
2667
 
@@ -2639,10 +2723,16 @@ def clear_conversation_messages(conn: sqlite3.Connection) -> None:
2639
2723
  _drop_conversation_fts_triggers(conn)
2640
2724
  conn.execute("DELETE FROM conversation_messages")
2641
2725
  conn.execute("INSERT INTO conversation_fts(conversation_fts) VALUES('delete-all')")
2642
- # #177: the aux index is created all-or-nothing with the prose FTS, so when
2643
- # fts_unavailable is False BOTH vtables exist reset BOTH the same storm-free
2644
- # way (triggers already dropped above, so the base DELETE touched neither).
2645
- conn.execute("INSERT INTO conversation_fts_aux(conversation_fts_aux) VALUES('delete-all')")
2726
+ # #177 S6: the consolidated split table is the only FTS vtable on a swapped /
2727
+ # fresh DB. A pre-swap legacy install still carries conversation_fts_aux, so
2728
+ # reset it the same storm-free way ONLY when it physically exists (the swap
2729
+ # drops it; a fresh/split DB never had it). Triggers were dropped above, so
2730
+ # the base DELETE touched neither index.
2731
+ if conn.execute(
2732
+ "SELECT 1 FROM sqlite_master WHERE type='table' AND name='conversation_fts_aux'"
2733
+ ).fetchone() is not None:
2734
+ conn.execute(
2735
+ "INSERT INTO conversation_fts_aux(conversation_fts_aux) VALUES('delete-all')")
2646
2736
  _create_conversation_fts_triggers(conn)
2647
2737
 
2648
2738
 
@@ -3225,6 +3315,50 @@ def _009_conversation_media_reingest(conn: sqlite3.Connection) -> None:
3225
3315
  conn.commit()
3226
3316
 
3227
3317
 
3318
+ @cache_migration("010_conversation_search_split")
3319
+ def _010_conversation_search_split(conn: sqlite3.Connection) -> None:
3320
+ """Flag-only arm for the #177 S6 search-column split. Sets
3321
+ ``conversation_search_split_pending``; sync_cache consumes it under the
3322
+ cache.db.lock flock (cursor-resumable blocks_json backfill of
3323
+ search_tool/search_thinking, then an atomic legacy->split FTS swap+rebuild —
3324
+ see _cctally_cache._consume_search_split). The handler does NO data work so
3325
+ the dispatcher's central stamp (#140) marks a genuinely-complete handler; a
3326
+ fresh install stamps it WITHOUT running (its schema is already split via
3327
+ _apply_cache_schema, so the consumer finds no flag and no-ops). Mirrors the
3328
+ flag-only reingest pattern of 002/003/007/009."""
3329
+ _set_cache_meta(conn, "conversation_search_split_pending", "1")
3330
+ conn.commit()
3331
+
3332
+
3333
+ def _swap_conversation_fts_to_split(conn: sqlite3.Connection) -> None:
3334
+ """Legacy two-table shape -> split multi-column shape, in one transaction.
3335
+ Explicit DROPs of BOTH old tables + triggers (an ``IF NOT EXISTS`` create
3336
+ would silently keep the old same-name single-column conversation_fts — spec
3337
+ F9), then create the split DDL + new triggers and rebuild the index from the
3338
+ base table. Runs under the caller's held cache.db.lock flock (the
3339
+ _consume_search_split end-transaction)."""
3340
+ _drop_conversation_fts_triggers(conn)
3341
+ conn.execute("DROP TABLE IF EXISTS conversation_fts")
3342
+ conn.execute("DROP TABLE IF EXISTS conversation_fts_aux")
3343
+ conn.execute(_CONV_FTS_SPLIT_DDL)
3344
+ _create_conversation_fts_triggers(conn)
3345
+ conn.execute("INSERT INTO conversation_fts(conversation_fts) VALUES('rebuild')")
3346
+
3347
+
3348
+ def conversation_search_depth(conn: sqlite3.Connection) -> str:
3349
+ """'prose-only' while migration 010's backfill is pending, else 'full'.
3350
+ Surfaced additively on the search/find responses so the client can degrade
3351
+ the Tools/Thinking facets during the one-time index split. An
3352
+ OperationalError (no cache_meta table yet) degrades to 'full'."""
3353
+ try:
3354
+ pending = conn.execute(
3355
+ "SELECT 1 FROM cache_meta WHERE key='conversation_search_split_pending'"
3356
+ ).fetchone()
3357
+ except sqlite3.OperationalError:
3358
+ return "full"
3359
+ return "prose-only" if pending else "full"
3360
+
3361
+
3228
3362
  # === Region 7d: Stats migration 008_recompute_weekly_cost_snapshots_dedup_fix ===
3229
3363
 
3230
3364
  @stats_migration("008_recompute_weekly_cost_snapshots_dedup_fix")