cctally 1.27.1 → 1.29.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,620 @@
1
+ """Pure query kernel for the conversation viewer endpoints (Plan 2, spec §3).
2
+
3
+ Takes a sqlite3.Connection over a cache.db that already holds Plan 1's
4
+ conversation_messages (+ FTS) and session_entries. No clock, no network, no
5
+ global mutation — unit-tested against an in-memory cache.db seeded by
6
+ _apply_cache_schema. Three entry points back the three GET routes:
7
+ list_conversations (rail), get_conversation (reader), search_conversations.
8
+
9
+ Cost is joined ONCE per logical assistant turn (msg_id, req_id) to the single
10
+ deduped session_entries row (idx_entries_dedup), via the shared pricing helper
11
+ — never per physical fragment and never from cost_usd_raw (often NULL).
12
+ """
13
+ from __future__ import annotations
14
+ import json as _json
15
+ import os
16
+ import sqlite3
17
+
18
+ # Public surface (Plan 2): shipped in the npm tarball + brew formula + public
19
+ # mirror — imported by the dashboard's conversation endpoints at runtime.
20
+
21
+ from _lib_pricing import _calculate_entry_cost
22
+
23
+
24
+ def _project_label(cwd) -> str:
25
+ """Basename of the project cwd (dashboard label posture — no reveal). Falls
26
+ back to the raw path for root-ish cwds, '' when absent."""
27
+ if not cwd:
28
+ return ""
29
+ return os.path.basename(cwd.rstrip("/")) or cwd
30
+
31
+
32
+ def _subagent_key(source_path):
33
+ """Privacy-safe subagent-thread identity for the reader. Each subagent (Task)
34
+ invocation writes its own ``agent-<hash>.jsonl``; the main session is
35
+ ``<session_id>.jsonl``. Returns the agent hash (``agent-`` prefix + ``.jsonl``
36
+ suffix stripped; an ``acompact-`` middle is kept), or ``None`` for the main
37
+ file / a non-agent path. We expose ONLY this derived key — never the raw
38
+ absolute ``source_path`` (which leaks home dir / username / encoded project,
39
+ and the conversation routes are LAN-exposable via dashboard.expose_transcripts)."""
40
+ if not source_path:
41
+ return None
42
+ base = os.path.basename(source_path)
43
+ if not base.startswith("agent-"):
44
+ return None
45
+ stem = base[len("agent-"):]
46
+ if stem.endswith(".jsonl"):
47
+ stem = stem[: -len(".jsonl")]
48
+ return stem or None
49
+
50
+
51
+ def _entry_cost(model, inp, out, cc, cr, cost_usd_raw) -> float:
52
+ """Cost for one session_entries row via the shared pricing helper. Tokens →
53
+ the helper's usage dict. cost_usd_raw is passed as the optional override the
54
+ helper already understands (it is often NULL — never the primary source)."""
55
+ usage = {
56
+ "input_tokens": inp or 0,
57
+ "output_tokens": out or 0,
58
+ "cache_creation_input_tokens": cc or 0,
59
+ "cache_read_input_tokens": cr or 0,
60
+ }
61
+ return _calculate_entry_cost(model or "", usage, cost_usd=cost_usd_raw)
62
+
63
+
64
+ def _session_cost_map(conn, session_ids):
65
+ """{session_id: total_cost_usd} for the given sessions. Joins
66
+ conversation_messages turn keys to the single deduped session_entries row
67
+ per (msg_id, req_id), so a turn replayed across files contributes once.
68
+ (msg_id, req_id) is globally unique in session_entries and maps to exactly
69
+ one session_id, so per-session sums are clean."""
70
+ costs = {sid: 0.0 for sid in session_ids}
71
+ if not session_ids:
72
+ return costs
73
+ placeholders = ",".join("?" for _ in session_ids)
74
+ sql = (
75
+ "SELECT cm.session_id, se.model, se.input_tokens, se.output_tokens, "
76
+ " se.cache_create_tokens, se.cache_read_tokens, se.cost_usd_raw "
77
+ "FROM (SELECT DISTINCT session_id, msg_id, req_id "
78
+ " FROM conversation_messages "
79
+ " WHERE session_id IN (%s) AND msg_id IS NOT NULL AND req_id IS NOT NULL) cm "
80
+ "JOIN session_entries se ON se.msg_id = cm.msg_id AND se.req_id = cm.req_id"
81
+ % placeholders
82
+ )
83
+ for sid, model, inp, out, cc, cr, raw in conn.execute(sql, list(session_ids)):
84
+ costs[sid] = costs.get(sid, 0.0) + _entry_cost(model, inp, out, cc, cr, raw)
85
+ return costs
86
+
87
+
88
+ def _session_models_map(conn, session_ids):
89
+ """{session_id: sorted distinct non-null models}."""
90
+ out = {sid: [] for sid in session_ids}
91
+ if not session_ids:
92
+ return out
93
+ placeholders = ",".join("?" for _ in session_ids)
94
+ sql = (
95
+ "SELECT DISTINCT session_id, model FROM conversation_messages "
96
+ "WHERE session_id IN (%s) AND model IS NOT NULL AND model != '' "
97
+ "ORDER BY model" % placeholders
98
+ )
99
+ for sid, model in conn.execute(sql, list(session_ids)):
100
+ out.setdefault(sid, []).append(model)
101
+ return out
102
+
103
+
104
+ def _session_latest_meta_map(conn, session_ids):
105
+ """{session_id: (cwd, git_branch)} using the most-recent NON-NULL value per
106
+ column — the SAME posture as get_conversation's _latest, so the rail and the
107
+ reader agree on a session whose cwd/branch changed over its lifetime (a plain
108
+ MAX() picks the lexical max, not the latest). Bounded to the page's sessions
109
+ via per-session correlated lookups over idx (session_id, timestamp_utc, id),
110
+ mirroring _session_cost_map / _session_models_map."""
111
+ meta = {sid: (None, None) for sid in session_ids}
112
+ if not session_ids:
113
+ return meta
114
+ placeholders = ",".join("?" for _ in session_ids)
115
+ sql = (
116
+ "SELECT s.session_id, "
117
+ " (SELECT c.cwd FROM conversation_messages c "
118
+ " WHERE c.session_id = s.session_id AND c.cwd IS NOT NULL "
119
+ " ORDER BY c.timestamp_utc DESC, c.id DESC LIMIT 1), "
120
+ " (SELECT b.git_branch FROM conversation_messages b "
121
+ " WHERE b.session_id = s.session_id AND b.git_branch IS NOT NULL "
122
+ " ORDER BY b.timestamp_utc DESC, b.id DESC LIMIT 1) "
123
+ "FROM (SELECT DISTINCT session_id FROM conversation_messages "
124
+ " WHERE session_id IN (%s)) s" % placeholders
125
+ )
126
+ for sid, cwd, branch in conn.execute(sql, list(session_ids)):
127
+ meta[sid] = (cwd, branch)
128
+ return meta
129
+
130
+
131
+ _SORTS = {
132
+ "recent": "MAX(timestamp_utc) DESC, session_id DESC",
133
+ "oldest": "MIN(timestamp_utc) ASC, session_id ASC",
134
+ }
135
+
136
+
137
+ def list_conversations(conn, *, sort="recent", limit=50, offset=0) -> dict:
138
+ """All-history per-session browse rows (spec §3.1). NOT 365-day bounded."""
139
+ order = _SORTS.get(sort, _SORTS["recent"])
140
+ limit = max(1, min(int(limit), 200))
141
+ offset = max(0, int(offset))
142
+ rows = conn.execute(
143
+ "SELECT session_id, COUNT(*) AS msg_count, "
144
+ " MIN(timestamp_utc) AS started, MAX(timestamp_utc) AS last_activity "
145
+ "FROM conversation_messages "
146
+ "WHERE session_id IS NOT NULL "
147
+ "GROUP BY session_id "
148
+ "ORDER BY " + order + " LIMIT ? OFFSET ?",
149
+ (limit + 1, offset),
150
+ ).fetchall()
151
+ has_more = len(rows) > limit
152
+ rows = rows[:limit]
153
+ session_ids = [r[0] for r in rows]
154
+ costs = _session_cost_map(conn, session_ids)
155
+ models = _session_models_map(conn, session_ids)
156
+ # cwd/git_branch as the latest non-null (reader posture), NOT a lexical MAX().
157
+ meta = _session_latest_meta_map(conn, session_ids)
158
+ conversations = [
159
+ {
160
+ "session_id": sid,
161
+ "project_label": _project_label(meta.get(sid, (None, None))[0]),
162
+ "git_branch": meta.get(sid, (None, None))[1],
163
+ "started_utc": started,
164
+ "last_activity_utc": last_activity,
165
+ "msg_count": msg_count,
166
+ "cost_usd": round(costs.get(sid, 0.0), 6),
167
+ "models": models.get(sid, []),
168
+ }
169
+ for (sid, msg_count, started, last_activity) in rows
170
+ ]
171
+ return {
172
+ "conversations": conversations,
173
+ "page": {
174
+ "next_offset": offset + len(conversations) if has_more else None,
175
+ "has_more": has_more,
176
+ },
177
+ }
178
+
179
+
180
+ def _turn_cost_map(conn, turn_keys):
181
+ """{(msg_id, req_id): cost_usd} for the given non-null turn keys, joined ONCE
182
+ to the deduped session_entries row. Keys absent from session_entries (e.g.
183
+ <synthetic> walker-skipped rows) are simply not present → cost 0 by omission."""
184
+ costs = {}
185
+ keys = [(m, r) for (m, r) in turn_keys if m is not None and r is not None]
186
+ if not keys:
187
+ return costs
188
+ # Chunk the OR-of-pairs to stay well under SQLite's variable limit.
189
+ for i in range(0, len(keys), 400):
190
+ chunk = keys[i:i + 400]
191
+ cond = " OR ".join("(msg_id=? AND req_id=?)" for _ in chunk)
192
+ params = [v for pair in chunk for v in pair]
193
+ sql = ("SELECT msg_id, req_id, model, input_tokens, output_tokens, "
194
+ "cache_create_tokens, cache_read_tokens, cost_usd_raw "
195
+ "FROM session_entries WHERE " + cond)
196
+ for m, r, model, inp, out, cc, cr, raw in conn.execute(sql, params):
197
+ costs[(m, r)] = _entry_cost(model, inp, out, cc, cr, raw)
198
+ return costs
199
+
200
+
201
+ def get_conversation(conn, session_id, *, after=None, limit=500):
202
+ """Reader payload for one session (spec §3.2). Returns None for an unknown
203
+ session. Dedups logical messages by (session_id, uuid) (canonical = earliest
204
+ timestamp), groups assistant fragments into turn items by (msg_id, req_id),
205
+ joins cost once, anchors a turn on its prose-bearing fragment, and exposes
206
+ every member fragment uuid for jump resolution. Cursor over (timestamp_utc,
207
+ id); ~500 items/page."""
208
+ limit = max(1, min(int(limit), 1000))
209
+ exists = conn.execute(
210
+ "SELECT 1 FROM conversation_messages WHERE session_id=? LIMIT 1",
211
+ (session_id,)).fetchone()
212
+ if exists is None:
213
+ return None
214
+
215
+ # Pull the session ordered; dedup logical messages by (session_id, uuid),
216
+ # canonical row = earliest (timestamp_utc, id). Replays carry the original
217
+ # uuid, so the first occurrence in ascending order is canonical.
218
+ raw = conn.execute(
219
+ "SELECT id, uuid, timestamp_utc, entry_type, text, blocks_json, model, "
220
+ " msg_id, req_id, is_sidechain, cwd, git_branch, source_path, parent_uuid "
221
+ "FROM conversation_messages WHERE session_id=? "
222
+ "ORDER BY timestamp_utc, id", (session_id,)).fetchall()
223
+
224
+ seen_uuid = set()
225
+ logical = [] # canonical physical rows, in order
226
+ for row in raw:
227
+ u = row[1]
228
+ if u in seen_uuid:
229
+ continue
230
+ seen_uuid.add(u)
231
+ logical.append(row)
232
+
233
+ # Group assistant fragments sharing (msg_id, req_id) into one turn item over
234
+ # the WHOLE logical list — NOT by adjacency. Real tool-using transcripts
235
+ # interleave a tool_result (a `user`/tool_result item) between fragments of
236
+ # the SAME turn, so the same key recurs non-consecutively. We keep a turn-key
237
+ # → item-index map: first occurrence emits the turn item AT THIS POSITION;
238
+ # later same-key fragments fold their blocks/prose/uuids into the existing
239
+ # item. A turn → exactly ONE item → cost counted exactly once. Humans,
240
+ # tool_results, and assistant rows with a null msg_id emit as simple items at
241
+ # their own position.
242
+ items = []
243
+ turn_index = {} # (msg_id, req_id) -> index into items
244
+ for row in logical:
245
+ (rid, u, ts, etype, text, blocks, model, msg_id, req_id,
246
+ is_sc, cwd, branch, source_path, parent_uuid) = row
247
+ if etype == "assistant" and msg_id is not None:
248
+ key = (msg_id, req_id)
249
+ idx = turn_index.get(key)
250
+ if idx is None:
251
+ turn_index[key] = len(items)
252
+ items.append(_build_turn([row]))
253
+ else:
254
+ _extend_turn(items[idx], row)
255
+ else:
256
+ items.append(_build_simple(row))
257
+
258
+ costs = _turn_cost_map(conn, list(turn_index))
259
+ # Stamp per-item cost first, then derive the header from the SUM of the
260
+ # ROUNDED per-item assistant costs (M2) — so the §6.5 invariant
261
+ # sum(items.cost_usd) == header cost_usd holds EXACTLY to 1e-9 by
262
+ # construction OVER THE FULL ITEM LIST. 6dp is the deliberate JSON display
263
+ # precision. NOTE: the header is the whole-session total; the returned
264
+ # ``items`` is a page subset, so on page 2+ sum(page) < header by design.
265
+ header_cost = 0.0
266
+ for it in items:
267
+ if it["kind"] == "assistant" and "_msg_id" in it:
268
+ turn_cost = round(costs.get((it["_msg_id"], it["_req_id"]), 0.0), 6)
269
+ it["cost_usd"] = turn_cost
270
+ header_cost += turn_cost
271
+ del it["_msg_id"]
272
+ del it["_req_id"]
273
+ it.pop("_has_prose", None)
274
+ header_cost = round(header_cost, 6)
275
+
276
+ # Cursor pagination over the item list (anchored to each item's canonical id).
277
+ # A non-None `after` that matches no item's anchor (stale/deleted cursor)
278
+ # yields an EMPTY page — never silently re-serves the head (M1).
279
+ start = 0
280
+ if after is not None:
281
+ start = None
282
+ for k, it in enumerate(items):
283
+ if str(it["anchor"]["id"]) == str(after):
284
+ start = k + 1
285
+ break
286
+ if start is None:
287
+ return {
288
+ "session_id": session_id,
289
+ "project_label": _project_label(_latest(logical, 10)),
290
+ "git_branch": _latest(logical, 11),
291
+ "started_utc": logical[0][2],
292
+ "last_activity_utc": logical[-1][2],
293
+ "cost_usd": header_cost,
294
+ "models": sorted({r[6] for r in logical if r[6]}),
295
+ "items": [],
296
+ "page": {"next_after": None, "has_more": False},
297
+ }
298
+ page = items[start:start + limit]
299
+ has_more = start + limit < len(items)
300
+ next_after = page[-1]["anchor"]["id"] if (page and has_more) else None
301
+
302
+ # Stamp the session_id into each anchor (spec anchor is (session_id, uuid);
303
+ # the dict literals are built session-agnostic, so fill it here where the
304
+ # session id is known). NOT a no-op — the endpoint/clients rely on it.
305
+ for it in page:
306
+ it["anchor"]["session_id"] = session_id
307
+
308
+ first = logical[0]
309
+ last = logical[-1]
310
+ models = sorted({r[6] for r in logical if r[6]})
311
+ return {
312
+ "session_id": session_id,
313
+ "project_label": _project_label(_latest(logical, 10)),
314
+ "git_branch": _latest(logical, 11),
315
+ "started_utc": first[2],
316
+ "last_activity_utc": last[2],
317
+ "cost_usd": header_cost,
318
+ "models": models,
319
+ "items": page,
320
+ "page": {"next_after": next_after, "has_more": has_more},
321
+ }
322
+
323
+
324
+ def _latest(logical, col):
325
+ """Most-recent non-null value in a column across the session (project/branch
326
+ show the latest, matching the dashboard's session posture)."""
327
+ for row in reversed(logical):
328
+ if row[col]:
329
+ return row[col]
330
+ return "" if col == 10 else None
331
+
332
+
333
+ def _build_turn(members):
334
+ """Seed a turn item from its first fragment(s). Prose = joined non-empty
335
+ fragment text; anchor/model = the prose-bearing fragment (empirically exactly
336
+ one per turn); member_uuids = all fragment uuids. Fragments arriving later
337
+ (possibly non-consecutive — interleaved with a tool_result) fold in via
338
+ _extend_turn, which re-promotes the anchor/model once a prose fragment lands."""
339
+ first = members[0]
340
+ item = {
341
+ "kind": "assistant",
342
+ "anchor": {"session_id": None, "uuid": first[1], "id": first[0]},
343
+ "member_uuids": [first[1]],
344
+ "ts": first[2],
345
+ "text": "",
346
+ "blocks": [],
347
+ "model": first[6],
348
+ "is_sidechain": bool(first[9]),
349
+ # subagent_key / parent_uuid are SEED-sourced (the first fragment, the
350
+ # turn's entry point) and NOT re-promoted in _fold_fragment — the prose
351
+ # anchor's parent_uuid is an intra-turn link, not the entry point (Codex
352
+ # P1). subagent_key is uniform across a turn's fragments (one file).
353
+ "subagent_key": _subagent_key(first[12]),
354
+ "parent_uuid": first[13],
355
+ "_msg_id": first[7],
356
+ "_req_id": first[8],
357
+ "_has_prose": False,
358
+ }
359
+ _fold_fragment(item, first)
360
+ for m in members[1:]:
361
+ _extend_turn(item, m)
362
+ return item
363
+
364
+
365
+ def _extend_turn(item, row):
366
+ """Fold one more same-turn assistant fragment into an existing turn item:
367
+ append its uuid + blocks + non-empty prose. The FIRST fragment carrying prose
368
+ promotes the anchor/model to itself (the prose-bearing fragment is the
369
+ canonical anchor); subsequent prose fragments only extend the joined text."""
370
+ item["member_uuids"].append(row[1])
371
+ _fold_fragment(item, row)
372
+
373
+
374
+ def _fold_fragment(item, row):
375
+ blocks = item["blocks"]
376
+ try:
377
+ blocks.extend(_json.loads(row[5] or "[]"))
378
+ except (ValueError, TypeError):
379
+ pass
380
+ frag_text = (row[4] or "").strip()
381
+ if frag_text:
382
+ if not item["_has_prose"]:
383
+ # First prose fragment becomes the canonical anchor / model.
384
+ item["anchor"]["uuid"] = row[1]
385
+ item["anchor"]["id"] = row[0]
386
+ item["model"] = row[6]
387
+ item["is_sidechain"] = bool(row[9])
388
+ item["_msg_id"] = row[7]
389
+ item["_req_id"] = row[8]
390
+ item["_has_prose"] = True
391
+ item["text"] = frag_text
392
+ else:
393
+ item["text"] = item["text"] + "\n" + frag_text
394
+
395
+
396
+ def _build_simple(row):
397
+ """A human, tool_result, or assistant-with-null-msg_id item (no turn grouping,
398
+ no cost). An assistant row routes here only when its msg_id is NULL (no turn
399
+ key → no session_entries join); it carries an explicit cost_usd of 0.0 and NO
400
+ internal _msg_id/_req_id keys, so the cost loop's KeyError path can never fire
401
+ (I2). The model is preserved for assistant rows."""
402
+ (rid, u, ts, etype, text, blocks, model, msg_id, req_id, is_sc, cwd, branch,
403
+ source_path, parent_uuid) = row
404
+ try:
405
+ parsed = _json.loads(blocks or "[]")
406
+ except (ValueError, TypeError):
407
+ parsed = []
408
+ item = {
409
+ "kind": etype,
410
+ "anchor": {"session_id": None, "uuid": u, "id": rid},
411
+ "member_uuids": [u],
412
+ "ts": ts,
413
+ "text": text,
414
+ "blocks": parsed,
415
+ "is_sidechain": bool(is_sc),
416
+ "subagent_key": _subagent_key(source_path),
417
+ "parent_uuid": parent_uuid,
418
+ }
419
+ if etype == "assistant":
420
+ item["model"] = model
421
+ item["cost_usd"] = 0.0
422
+ return item
423
+
424
+
425
+ def _fts_flag_unavailable(conn) -> bool:
426
+ try:
427
+ row = conn.execute(
428
+ "SELECT value FROM cache_meta WHERE key='fts5_unavailable'").fetchone()
429
+ except sqlite3.OperationalError:
430
+ return False
431
+ return bool(row and row[0])
432
+
433
+
434
+ def search_conversations(conn, query, *, limit=50, offset=0,
435
+ fts_available=None) -> dict:
436
+ """Cross-session search (spec §3.3). Uses FTS5 when available (bm25 rank +
437
+ snippet); else a LIKE scan with a manual snippet. Hits deduped by
438
+ (session_id, uuid); each carries the turn's cost. `fts_available` overrides
439
+ detection (test seam / explicit LIKE)."""
440
+ q = (query or "").strip()
441
+ limit = max(1, min(int(limit), 200))
442
+ offset = max(0, int(offset))
443
+ if fts_available is None:
444
+ fts_available = not _fts_flag_unavailable(conn)
445
+ if not q:
446
+ return {"query": q, "mode": "fts" if fts_available else "like",
447
+ "hits": [], "total": 0}
448
+ if fts_available:
449
+ try:
450
+ return _search_fts(conn, q, limit, offset)
451
+ except sqlite3.OperationalError:
452
+ pass # corrupt/missing FTS at query time → fall through to LIKE
453
+ return _search_like(conn, q, limit, offset)
454
+
455
+
456
+ def _row_to_hit(uuid_, sid, ts, cwd, snippet, msg_id, req_id):
457
+ """Build one hit WITHOUT cost — cost is batched onto the FINAL page in
458
+ _attach_costs (I1: no per-hit _turn_cost_map round-trip). The turn key rides
459
+ on the private `_turn_key` field until the batch maps it to `cost_usd`."""
460
+ return {
461
+ "session_id": sid,
462
+ "uuid": uuid_,
463
+ "project_label": _project_label(cwd),
464
+ "ts": ts,
465
+ "snippet": snippet,
466
+ "_turn_key": (msg_id, req_id) if msg_id is not None and req_id is not None
467
+ else None,
468
+ }
469
+
470
+
471
+ def _attach_costs(conn, page):
472
+ """Compute turn cost for the FINAL page's hits in ONE _turn_cost_map call,
473
+ then map it onto each hit and drop the private `_turn_key`. Off-page and
474
+ duplicate hits never reach here, so we never compute cost for them (I1)."""
475
+ keys = [h["_turn_key"] for h in page if h.get("_turn_key") is not None]
476
+ costs = _turn_cost_map(conn, keys) if keys else {}
477
+ for h in page:
478
+ tk = h.pop("_turn_key", None)
479
+ h["cost_usd"] = round(costs.get(tk, 0.0), 6) if tk is not None else 0.0
480
+ return page
481
+
482
+
483
+ def _like_pattern(q):
484
+ """Build the LIKE pattern for `q`. Escape the ESCAPE char (\\) FIRST, then
485
+ the wildcards — otherwise a query containing a backslash (incl. a trailing
486
+ one) mis-escapes the appended '%' and the LIKE silently matches nothing
487
+ (paired with ESCAPE '\\' in the queries below)."""
488
+ return ("%" + q.replace("\\", "\\\\").replace("%", r"\%").replace("_", r"\_")
489
+ + "%")
490
+
491
+
492
+ def _fts_snippets(conn, fts_q, ids):
493
+ """{rowid: snippet} for the page rowids ONLY (#149). snippet() needs an
494
+ active MATCH, so it can't be deferred to an outer query over the page CTE;
495
+ a second bounded MATCH restricted to the page rowids generates snippets for
496
+ at most one page of hits instead of every corpus match."""
497
+ if not ids:
498
+ return {}
499
+ ph = ",".join("?" for _ in ids)
500
+ rows = conn.execute(
501
+ "SELECT cm.id, snippet(conversation_fts, 0, '[', ']', ' … ', 12) "
502
+ "FROM conversation_fts "
503
+ "JOIN conversation_messages cm ON cm.id = conversation_fts.rowid "
504
+ f"WHERE conversation_fts MATCH ? AND cm.id IN ({ph})",
505
+ (fts_q, *ids),
506
+ ).fetchall()
507
+ return {r[0]: r[1] for r in rows}
508
+
509
+
510
+ def _texts_for_ids(conn, ids):
511
+ """{rowid: text} for the page rowids ONLY (#149) — the LIKE page query omits
512
+ `text` so we never pull every matched row's body into Python; this fetches
513
+ it for just the page so `_manual_snippet` runs at most `limit` times."""
514
+ if not ids:
515
+ return {}
516
+ ph = ",".join("?" for _ in ids)
517
+ rows = conn.execute(
518
+ f"SELECT id, text FROM conversation_messages WHERE id IN ({ph})",
519
+ tuple(ids),
520
+ ).fetchall()
521
+ return {r[0]: r[1] for r in rows}
522
+
523
+
524
+ def _search_fts(conn, q, limit, offset):
525
+ # All of dedup + paging + total live in SQL (#149) so Python never holds
526
+ # more than one page of hits/snippets, regardless of corpus match count.
527
+ fts_q = _fts_query(q)
528
+ # Exact post-dedup logical total — counted in C with no snippet generation
529
+ # and no Python row materialization.
530
+ total = conn.execute(
531
+ "SELECT COUNT(*) FROM ("
532
+ " SELECT DISTINCT cm.session_id, cm.uuid "
533
+ " FROM conversation_fts "
534
+ " JOIN conversation_messages cm ON cm.id = conversation_fts.rowid "
535
+ " WHERE conversation_fts MATCH ?)",
536
+ (fts_q,),
537
+ ).fetchone()[0]
538
+ # One row per logical (session_id, uuid): ROW_NUMBER()=1 keeps the SAME row
539
+ # the old Python dedup kept as its FIRST occurrence (order: bm25, ts DESC,
540
+ # id DESC — cm.id is the final deterministic tiebreaker), so the surviving
541
+ # snippet/cost and the page boundary stay byte-stable. bm25 still ranks
542
+ # across all matches (inherent to relevance ordering).
543
+ #
544
+ # bm25 is materialized as a plain `rank` column in the inner `matched` CTE
545
+ # before the window function runs: FTS5 auxiliary functions (bm25/snippet)
546
+ # may only be used directly against the MATCH query, NOT inside a window
547
+ # ORDER BY ("unable to use function bm25 in the requested context").
548
+ page = conn.execute(
549
+ "WITH matched AS ("
550
+ " SELECT cm.id AS rid, cm.session_id AS sid, cm.uuid AS uuid, "
551
+ " cm.timestamp_utc AS ts, cm.cwd AS cwd, "
552
+ " cm.msg_id AS mid, cm.req_id AS rqd, "
553
+ " bm25(conversation_fts) AS rank "
554
+ " FROM conversation_fts "
555
+ " JOIN conversation_messages cm ON cm.id = conversation_fts.rowid "
556
+ " WHERE conversation_fts MATCH ?), "
557
+ "ranked AS ("
558
+ " SELECT *, ROW_NUMBER() OVER ("
559
+ " PARTITION BY sid, uuid ORDER BY rank, ts DESC, rid DESC"
560
+ " ) AS rn "
561
+ " FROM matched) "
562
+ "SELECT rid, sid, uuid, ts, cwd, mid, rqd FROM ranked WHERE rn = 1 "
563
+ "ORDER BY rank, ts DESC, rid DESC LIMIT ? OFFSET ?",
564
+ (fts_q, limit, offset),
565
+ ).fetchall()
566
+ snips = _fts_snippets(conn, fts_q, [r[0] for r in page])
567
+ hits = [_row_to_hit(uuid, sid, ts, cwd, snips.get(rid, ""), mid, rqd)
568
+ for (rid, sid, uuid, ts, cwd, mid, rqd) in page]
569
+ return {"query": q, "mode": "fts", "hits": _attach_costs(conn, hits),
570
+ "total": total}
571
+
572
+
573
+ def _search_like(conn, q, limit, offset):
574
+ # SQL-bounded mirror of _search_fts for the no-FTS5 fallback (#149); the
575
+ # COUNT + page each scan the table once (the degraded path already lacks an
576
+ # index for the substring match).
577
+ like = _like_pattern(q)
578
+ total = conn.execute(
579
+ "SELECT COUNT(*) FROM ("
580
+ " SELECT DISTINCT session_id, uuid FROM conversation_messages "
581
+ " WHERE text LIKE ? ESCAPE '\\' AND text != '')",
582
+ (like,),
583
+ ).fetchone()[0]
584
+ page = conn.execute(
585
+ "WITH ranked AS ("
586
+ " SELECT id AS rid, session_id AS sid, uuid AS uuid, "
587
+ " timestamp_utc AS ts, cwd AS cwd, msg_id AS mid, req_id AS rqd, "
588
+ " ROW_NUMBER() OVER ("
589
+ " PARTITION BY session_id, uuid "
590
+ " ORDER BY timestamp_utc DESC, id DESC"
591
+ " ) AS rn "
592
+ " FROM conversation_messages "
593
+ " WHERE text LIKE ? ESCAPE '\\' AND text != '') "
594
+ "SELECT rid, sid, uuid, ts, cwd, mid, rqd FROM ranked WHERE rn = 1 "
595
+ "ORDER BY ts DESC, rid DESC LIMIT ? OFFSET ?",
596
+ (like, limit, offset),
597
+ ).fetchall()
598
+ texts = _texts_for_ids(conn, [r[0] for r in page])
599
+ hits = [_row_to_hit(uuid, sid, ts, cwd,
600
+ _manual_snippet(texts.get(rid, ""), q), mid, rqd)
601
+ for (rid, sid, uuid, ts, cwd, mid, rqd) in page]
602
+ return {"query": q, "mode": "like", "hits": _attach_costs(conn, hits),
603
+ "total": total}
604
+
605
+
606
+ def _fts_query(q):
607
+ """Quote each whitespace term as an FTS5 string literal so punctuation /
608
+ operators in user input can't error the MATCH or inject FTS syntax."""
609
+ terms = [t for t in q.split() if t]
610
+ return " ".join('"' + t.replace('"', '""') + '"' for t in terms) or '""'
611
+
612
+
613
+ def _manual_snippet(text, q, width=80):
614
+ lo = text.lower().find(q.lower())
615
+ if lo < 0:
616
+ return text[:width]
617
+ start = max(0, lo - width // 2)
618
+ end = min(len(text), lo + len(q) + width // 2)
619
+ s = text[start:end]
620
+ return ("… " if start else "") + s + (" …" if end < len(text) else "")