cctally 1.40.0 → 1.41.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +5 -0
- package/bin/_cctally_cache.py +103 -6
- package/bin/_cctally_dashboard.py +60 -3
- package/bin/_cctally_db.py +179 -45
- package/bin/_lib_conversation.py +75 -21
- package/bin/_lib_conversation_query.py +385 -32
- package/dashboard/static/assets/{index-BXm_vBKA.css → index-BKlFTF9C.css} +1 -1
- package/dashboard/static/assets/{index-BtbVKnkm.js → index-DZ_higAv.js} +34 -34
- package/dashboard/static/dashboard.html +2 -2
- package/package.json +1 -1
|
@@ -1083,38 +1083,74 @@ def _fts_flag_unavailable(conn) -> bool:
|
|
|
1083
1083
|
return bool(row and row[0])
|
|
1084
1084
|
|
|
1085
1085
|
|
|
1086
|
+
def _search_depth(conn) -> str:
|
|
1087
|
+
"""'prose-only' while migration 010's column split is pending, else 'full'
|
|
1088
|
+
(#177 S6). Mirrors ``_cctally_db.conversation_search_depth`` but reads the
|
|
1089
|
+
flag inline (the kernel never imports the db sibling — same pattern as
|
|
1090
|
+
``_fts_flag_unavailable``). An OperationalError (no cache_meta) → 'full'."""
|
|
1091
|
+
try:
|
|
1092
|
+
pending = conn.execute(
|
|
1093
|
+
"SELECT 1 FROM cache_meta "
|
|
1094
|
+
"WHERE key='conversation_search_split_pending'").fetchone()
|
|
1095
|
+
except sqlite3.OperationalError:
|
|
1096
|
+
return "full"
|
|
1097
|
+
return "prose-only" if pending else "full"
|
|
1098
|
+
|
|
1099
|
+
|
|
1086
1100
|
def search_conversations(conn, query, *, limit=50, offset=0,
|
|
1087
|
-
fts_available=None) -> dict:
|
|
1101
|
+
kind="all", fts_available=None) -> dict:
|
|
1088
1102
|
"""Cross-session search (spec §3.3). Uses FTS5 when available (bm25 rank +
|
|
1089
1103
|
snippet); else a LIKE scan with a manual snippet. Hits deduped by
|
|
1090
1104
|
(session_id, uuid); each carries the turn's cost. `fts_available` overrides
|
|
1091
|
-
detection (test seam / explicit LIKE).
|
|
1105
|
+
detection (test seam / explicit LIKE).
|
|
1106
|
+
|
|
1107
|
+
#177 S6: ``kind`` (one of ``_SEARCH_KINDS``) scopes the search to a column
|
|
1108
|
+
family — ``all`` is unfiltered, ``prompts``/``assistant`` filter the prose
|
|
1109
|
+
column + entry_type, ``tools``/``thinking`` filter the split index columns.
|
|
1110
|
+
Every hit gains ``match_kinds`` (sorted ``['tool', 'thinking']`` badges;
|
|
1111
|
+
prose never badges). The response carries additive ``kind`` + ``search_depth``
|
|
1112
|
+
so the client can degrade the Tools/Thinking facets during the one-time
|
|
1113
|
+
column split (``search_depth == 'prose-only'`` short-circuits those two
|
|
1114
|
+
kinds to empty). An unknown ``kind`` raises ``ValueError`` (route → 400)."""
|
|
1115
|
+
if kind not in _SEARCH_KINDS:
|
|
1116
|
+
raise ValueError(f"unknown kind: {kind}")
|
|
1092
1117
|
q = (query or "").strip()
|
|
1093
1118
|
limit = max(1, min(int(limit), 200))
|
|
1094
1119
|
offset = max(0, int(offset))
|
|
1095
1120
|
if fts_available is None:
|
|
1096
1121
|
fts_available = not _fts_flag_unavailable(conn)
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1122
|
+
depth = _search_depth(conn)
|
|
1123
|
+
mode = "fts" if fts_available else "like"
|
|
1124
|
+
base = {"query": q, "mode": mode, "hits": [], "total": 0,
|
|
1125
|
+
"kind": kind, "search_depth": depth}
|
|
1126
|
+
# Prose-only interim: the split columns are not yet indexed, so tools /
|
|
1127
|
+
# thinking can't match — short-circuit them to empty (spec §1 interim).
|
|
1128
|
+
if not q or (depth == "prose-only" and kind in ("tools", "thinking")):
|
|
1129
|
+
return base
|
|
1100
1130
|
if fts_available:
|
|
1101
1131
|
try:
|
|
1102
|
-
|
|
1132
|
+
out = _search_fts(conn, q, limit, offset, kind, depth)
|
|
1133
|
+
out.update(kind=kind, search_depth=depth)
|
|
1134
|
+
return out
|
|
1103
1135
|
except sqlite3.OperationalError:
|
|
1104
1136
|
pass # corrupt/missing FTS at query time → fall through to LIKE
|
|
1105
|
-
|
|
1137
|
+
out = _search_like(conn, q, limit, offset, kind, depth)
|
|
1138
|
+
out.update(kind=kind, mode="like", search_depth=depth)
|
|
1139
|
+
return out
|
|
1106
1140
|
|
|
1107
1141
|
|
|
1108
|
-
def _row_to_hit(uuid_, sid, ts, cwd, snippet, msg_id, req_id):
|
|
1142
|
+
def _row_to_hit(uuid_, sid, ts, cwd, snippet, msg_id, req_id, match_kinds=None):
|
|
1109
1143
|
"""Build one hit WITHOUT cost — cost is batched onto the FINAL page in
|
|
1110
1144
|
_attach_costs (I1: no per-hit _turn_cost_map round-trip). The turn key rides
|
|
1111
|
-
on the private `_turn_key` field until the batch maps it to `cost_usd`.
|
|
1145
|
+
on the private `_turn_key` field until the batch maps it to `cost_usd`.
|
|
1146
|
+
#177 S6: ``match_kinds`` (sorted non-prose badges) is attached per hit."""
|
|
1112
1147
|
return {
|
|
1113
1148
|
"session_id": sid,
|
|
1114
1149
|
"uuid": uuid_,
|
|
1115
1150
|
"project_label": _project_label(cwd),
|
|
1116
1151
|
"ts": ts,
|
|
1117
1152
|
"snippet": snippet,
|
|
1153
|
+
"match_kinds": match_kinds or [],
|
|
1118
1154
|
"_turn_key": (msg_id, req_id) if msg_id is not None and req_id is not None
|
|
1119
1155
|
else None,
|
|
1120
1156
|
}
|
|
@@ -1154,16 +1190,18 @@ def _like_pattern(q):
|
|
|
1154
1190
|
+ "%")
|
|
1155
1191
|
|
|
1156
1192
|
|
|
1157
|
-
def _fts_snippets(conn, fts_q, ids):
|
|
1193
|
+
def _fts_snippets(conn, fts_q, ids, col=0):
|
|
1158
1194
|
"""{rowid: snippet} for the page rowids ONLY (#149). snippet() needs an
|
|
1159
1195
|
active MATCH, so it can't be deferred to an outer query over the page CTE;
|
|
1160
1196
|
a second bounded MATCH restricted to the page rowids generates snippets for
|
|
1161
|
-
at most one page of hits instead of every corpus match.
|
|
1197
|
+
at most one page of hits instead of every corpus match. #177 S6: ``col``
|
|
1198
|
+
selects which FTS column the snippet is drawn from (0=text, 1=search_tool,
|
|
1199
|
+
2=search_thinking) so a tool/thinking hit shows its matching content."""
|
|
1162
1200
|
if not ids:
|
|
1163
1201
|
return {}
|
|
1164
1202
|
ph = ",".join("?" for _ in ids)
|
|
1165
1203
|
rows = conn.execute(
|
|
1166
|
-
"SELECT cm.id, snippet(conversation_fts,
|
|
1204
|
+
f"SELECT cm.id, snippet(conversation_fts, {int(col)}, '[', ']', ' … ', 12) "
|
|
1167
1205
|
"FROM conversation_fts "
|
|
1168
1206
|
"JOIN conversation_messages cm ON cm.id = conversation_fts.rowid "
|
|
1169
1207
|
f"WHERE conversation_fts MATCH ? AND cm.id IN ({ph})",
|
|
@@ -1186,10 +1224,54 @@ def _texts_for_ids(conn, ids):
|
|
|
1186
1224
|
return {r[0]: r[1] for r in rows}
|
|
1187
1225
|
|
|
1188
1226
|
|
|
1189
|
-
|
|
1227
|
+
# #177 S6: FTS-column index → badge label (col 0 = prose = no badge).
|
|
1228
|
+
_KIND_PROBE_COLUMNS = (("search_tool", "tool"), ("search_thinking", "thinking"))
|
|
1229
|
+
_SNIPPET_COL_PREFERENCE = (("tool", 1), ("thinking", 2)) # prose (0) is default
|
|
1230
|
+
|
|
1231
|
+
|
|
1232
|
+
def _match_kinds(conn, fts_q, rids_by_group):
|
|
1233
|
+
"""{group -> sorted [badges]} via marker-based column probes (spec F3).
|
|
1234
|
+
|
|
1235
|
+
A column "matched" iff a column-filtered sub-MATCH returns its rowid — NOT
|
|
1236
|
+
iff snippet() is non-empty (snippet returns the column's unmarked text for
|
|
1237
|
+
non-matching columns). Probes aggregate across ALL matched rowids of each
|
|
1238
|
+
page group's ``(session_id, uuid)``, so a multi-row hit badges completely.
|
|
1239
|
+
Prose (col 0) is never a badge. ``fts_q`` is the un-column-filtered term
|
|
1240
|
+
expression (the per-column wrapping is applied here)."""
|
|
1241
|
+
all_rids = sorted({r for rids in rids_by_group.values() for r in rids})
|
|
1242
|
+
if not all_rids:
|
|
1243
|
+
return {grp: [] for grp in rids_by_group}
|
|
1244
|
+
ph = ",".join("?" for _ in all_rids)
|
|
1245
|
+
hits_by_col = {}
|
|
1246
|
+
for col, label in _KIND_PROBE_COLUMNS:
|
|
1247
|
+
got = conn.execute(
|
|
1248
|
+
"SELECT conversation_fts.rowid FROM conversation_fts "
|
|
1249
|
+
f"WHERE conversation_fts MATCH ? AND conversation_fts.rowid IN ({ph})",
|
|
1250
|
+
(f"{{{col}}}: ({fts_q})", *all_rids),
|
|
1251
|
+
).fetchall()
|
|
1252
|
+
hits_by_col[label] = {r[0] for r in got}
|
|
1253
|
+
return {grp: [lbl for (_c, lbl) in _KIND_PROBE_COLUMNS
|
|
1254
|
+
if set(rids) & hits_by_col[lbl]]
|
|
1255
|
+
for grp, rids in rids_by_group.items()}
|
|
1256
|
+
|
|
1257
|
+
|
|
1258
|
+
def _search_fts(conn, q, limit, offset, kind, depth):
|
|
1190
1259
|
# All of dedup + paging + total live in SQL (#149) so Python never holds
|
|
1191
1260
|
# more than one page of hits/snippets, regardless of corpus match count.
|
|
1192
|
-
|
|
1261
|
+
#
|
|
1262
|
+
# #177 S6: prose-only interim runs the LEGACY single-column shape (the split
|
|
1263
|
+
# columns are not yet indexed) — no column filter, no badge/snippet probes
|
|
1264
|
+
# against search_tool/search_thinking (those columns aren't in the legacy
|
|
1265
|
+
# FTS table). Full mode applies the kind column filter + entry_type predicate
|
|
1266
|
+
# and the marker-based badges.
|
|
1267
|
+
legacy = depth == "prose-only"
|
|
1268
|
+
fts_q = _fts_query(q, prefix_last=True)
|
|
1269
|
+
# legacy single-column MATCH (prose), no column filter; full mode applies
|
|
1270
|
+
# the kind column filter.
|
|
1271
|
+
match_expr = fts_q if legacy else _kind_match_expr(kind, fts_q)
|
|
1272
|
+
entry_type = _KIND_ENTRY_TYPE.get(kind)
|
|
1273
|
+
et_pred = " AND cm.entry_type = ?" if entry_type is not None else ""
|
|
1274
|
+
et_args = (entry_type,) if entry_type is not None else ()
|
|
1193
1275
|
# Exact post-dedup logical total — counted in C with no snippet generation
|
|
1194
1276
|
# and no Python row materialization.
|
|
1195
1277
|
total = conn.execute(
|
|
@@ -1197,8 +1279,8 @@ def _search_fts(conn, q, limit, offset):
|
|
|
1197
1279
|
" SELECT DISTINCT cm.session_id, cm.uuid "
|
|
1198
1280
|
" FROM conversation_fts "
|
|
1199
1281
|
" JOIN conversation_messages cm ON cm.id = conversation_fts.rowid "
|
|
1200
|
-
" WHERE conversation_fts MATCH ?)",
|
|
1201
|
-
(
|
|
1282
|
+
f" WHERE conversation_fts MATCH ?{et_pred})",
|
|
1283
|
+
(match_expr, *et_args),
|
|
1202
1284
|
).fetchone()[0]
|
|
1203
1285
|
# One row per logical (session_id, uuid): ROW_NUMBER()=1 keeps the SAME row
|
|
1204
1286
|
# the old Python dedup kept as its FIRST occurrence (order: bm25, ts DESC,
|
|
@@ -1209,16 +1291,20 @@ def _search_fts(conn, q, limit, offset):
|
|
|
1209
1291
|
# bm25 is materialized as a plain `rank` column in the inner `matched` CTE
|
|
1210
1292
|
# before the window function runs: FTS5 auxiliary functions (bm25/snippet)
|
|
1211
1293
|
# may only be used directly against the MATCH query, NOT inside a window
|
|
1212
|
-
# ORDER BY ("unable to use function bm25 in the requested context").
|
|
1294
|
+
# ORDER BY ("unable to use function bm25 in the requested context"). Weights
|
|
1295
|
+
# (prose > tool > thinking) only apply to the multi-column (full) shape;
|
|
1296
|
+
# the legacy single-column table takes the plain bm25.
|
|
1297
|
+
bm25_expr = ("bm25(conversation_fts)" if legacy
|
|
1298
|
+
else "bm25(conversation_fts, 10.0, 3.0, 1.0)")
|
|
1213
1299
|
page = conn.execute(
|
|
1214
1300
|
"WITH matched AS ("
|
|
1215
1301
|
" SELECT cm.id AS rid, cm.session_id AS sid, cm.uuid AS uuid, "
|
|
1216
1302
|
" cm.timestamp_utc AS ts, cm.cwd AS cwd, "
|
|
1217
1303
|
" cm.msg_id AS mid, cm.req_id AS rqd, "
|
|
1218
|
-
"
|
|
1304
|
+
f" {bm25_expr} AS rank "
|
|
1219
1305
|
" FROM conversation_fts "
|
|
1220
1306
|
" JOIN conversation_messages cm ON cm.id = conversation_fts.rowid "
|
|
1221
|
-
" WHERE conversation_fts MATCH ?), "
|
|
1307
|
+
f" WHERE conversation_fts MATCH ?{et_pred}), "
|
|
1222
1308
|
"ranked AS ("
|
|
1223
1309
|
" SELECT *, ROW_NUMBER() OVER ("
|
|
1224
1310
|
" PARTITION BY sid, uuid ORDER BY rank, ts DESC, rid DESC"
|
|
@@ -1226,26 +1312,101 @@ def _search_fts(conn, q, limit, offset):
|
|
|
1226
1312
|
" FROM matched) "
|
|
1227
1313
|
"SELECT rid, sid, uuid, ts, cwd, mid, rqd FROM ranked WHERE rn = 1 "
|
|
1228
1314
|
"ORDER BY rank, ts DESC, rid DESC LIMIT ? OFFSET ?",
|
|
1229
|
-
(
|
|
1315
|
+
(match_expr, *et_args, limit, offset),
|
|
1230
1316
|
).fetchall()
|
|
1231
|
-
|
|
1232
|
-
|
|
1317
|
+
page_groups = {(sid, uuid): rid for (rid, sid, uuid, ts, cwd, mid, rqd) in page}
|
|
1318
|
+
if legacy:
|
|
1319
|
+
badges = {grp: [] for grp in page_groups}
|
|
1320
|
+
else:
|
|
1321
|
+
rids_by_group = _all_matched_rids_by_group(
|
|
1322
|
+
conn, match_expr, et_pred, et_args, list(page_groups))
|
|
1323
|
+
badges = _match_kinds(conn, fts_q, rids_by_group)
|
|
1324
|
+
snips = _fts_snippets(conn, match_expr, [r[0] for r in page], col=0)
|
|
1325
|
+
# For hits badged tool/thinking but with no prose match, draw the snippet
|
|
1326
|
+
# from the matched column instead (prose → tool → thinking preference).
|
|
1327
|
+
if not legacy:
|
|
1328
|
+
snips = _prefer_snippet_columns(conn, fts_q, page, page_groups, badges, snips)
|
|
1329
|
+
hits = [_row_to_hit(uuid, sid, ts, cwd, snips.get(rid, ""), mid, rqd,
|
|
1330
|
+
match_kinds=badges.get((sid, uuid), []))
|
|
1233
1331
|
for (rid, sid, uuid, ts, cwd, mid, rqd) in page]
|
|
1234
1332
|
return {"query": q, "mode": "fts",
|
|
1235
1333
|
"hits": _attach_titles(conn, _attach_costs(conn, hits)),
|
|
1236
1334
|
"total": total}
|
|
1237
1335
|
|
|
1238
1336
|
|
|
1239
|
-
def
|
|
1337
|
+
def _all_matched_rids_by_group(conn, match_expr, et_pred, et_args, groups):
|
|
1338
|
+
"""{(sid, uuid) -> [rids]} for the page groups: ALL matched physical rows of
|
|
1339
|
+
each group (not just the rank-survivor), so badges aggregate completely."""
|
|
1340
|
+
if not groups:
|
|
1341
|
+
return {}
|
|
1342
|
+
rids_by_group = {g: [] for g in groups}
|
|
1343
|
+
rows = conn.execute(
|
|
1344
|
+
"SELECT cm.id, cm.session_id, cm.uuid FROM conversation_fts "
|
|
1345
|
+
"JOIN conversation_messages cm ON cm.id = conversation_fts.rowid "
|
|
1346
|
+
f"WHERE conversation_fts MATCH ?{et_pred}",
|
|
1347
|
+
(match_expr, *et_args),
|
|
1348
|
+
).fetchall()
|
|
1349
|
+
for rid, sid, uuid in rows:
|
|
1350
|
+
g = (sid, uuid)
|
|
1351
|
+
if g in rids_by_group:
|
|
1352
|
+
rids_by_group[g].append(rid)
|
|
1353
|
+
return rids_by_group
|
|
1354
|
+
|
|
1355
|
+
|
|
1356
|
+
def _prefer_snippet_columns(conn, fts_q, page, page_groups, badges, snips):
|
|
1357
|
+
"""Replace a hit's prose snippet with its matched column's snippet when the
|
|
1358
|
+
prose column did NOT match (prose → tool → thinking preference). Probes
|
|
1359
|
+
which column matched the survivor rowid, then re-snippets that column."""
|
|
1360
|
+
by_col = {} # snippet column index -> [rids needing it]
|
|
1361
|
+
for (rid, sid, uuid, ts, cwd, mid, rqd) in page:
|
|
1362
|
+
grp = (sid, uuid)
|
|
1363
|
+
kinds = badges.get(grp, [])
|
|
1364
|
+
if not kinds:
|
|
1365
|
+
continue # prose hit (or unbadged) — keep col-0 snippet
|
|
1366
|
+
# Does THIS survivor row match prose? If so keep col 0.
|
|
1367
|
+
prose_hit = conn.execute(
|
|
1368
|
+
"SELECT 1 FROM conversation_fts "
|
|
1369
|
+
"WHERE conversation_fts MATCH ? AND conversation_fts.rowid = ?",
|
|
1370
|
+
(f"{{text}}: ({fts_q})", rid)).fetchone()
|
|
1371
|
+
if prose_hit:
|
|
1372
|
+
continue
|
|
1373
|
+
for label, col in _SNIPPET_COL_PREFERENCE:
|
|
1374
|
+
if label in kinds:
|
|
1375
|
+
by_col.setdefault(col, []).append(rid)
|
|
1376
|
+
break
|
|
1377
|
+
for col, rids in by_col.items():
|
|
1378
|
+
col_fts = _kind_match_expr(
|
|
1379
|
+
"tools" if col == 1 else "thinking", fts_q)
|
|
1380
|
+
alt = _fts_snippets(conn, col_fts, rids, col=col)
|
|
1381
|
+
snips.update(alt)
|
|
1382
|
+
return snips
|
|
1383
|
+
|
|
1384
|
+
|
|
1385
|
+
def _search_like(conn, q, limit, offset, kind, depth):
|
|
1240
1386
|
# SQL-bounded mirror of _search_fts for the no-FTS5 fallback (#149); the
|
|
1241
1387
|
# COUNT + page each scan the table once (the degraded path already lacks an
|
|
1242
|
-
# index for the substring match).
|
|
1388
|
+
# index for the substring match). #177 S6: kind → column list (single-
|
|
1389
|
+
# substring semantics preserved — a documented degraded divergence from FTS
|
|
1390
|
+
# term-wise AND); badges from per-column LIKE probes on the page rows.
|
|
1391
|
+
legacy = depth == "prose-only"
|
|
1243
1392
|
like = _like_pattern(q)
|
|
1393
|
+
if legacy:
|
|
1394
|
+
cols = ["text"]
|
|
1395
|
+
else:
|
|
1396
|
+
cols = ({"prompts": ["text"], "assistant": ["text"],
|
|
1397
|
+
"tools": ["search_tool"], "thinking": ["search_thinking"]}
|
|
1398
|
+
.get(kind, ["text", "search_tool", "search_thinking"]))
|
|
1399
|
+
col_pred = "(" + " OR ".join(
|
|
1400
|
+
f"{c} LIKE ? ESCAPE '\\' AND {c} != ''" for c in cols) + ")"
|
|
1401
|
+
like_args = tuple(like for _ in cols)
|
|
1402
|
+
entry_type = _KIND_ENTRY_TYPE.get(kind)
|
|
1403
|
+
et_pred = " AND entry_type = ?" if entry_type is not None else ""
|
|
1404
|
+
et_args = (entry_type,) if entry_type is not None else ()
|
|
1244
1405
|
total = conn.execute(
|
|
1245
1406
|
"SELECT COUNT(*) FROM ("
|
|
1246
1407
|
" SELECT DISTINCT session_id, uuid FROM conversation_messages "
|
|
1247
|
-
" WHERE
|
|
1248
|
-
(
|
|
1408
|
+
f" WHERE {col_pred}{et_pred})",
|
|
1409
|
+
(*like_args, *et_args),
|
|
1249
1410
|
).fetchone()[0]
|
|
1250
1411
|
page = conn.execute(
|
|
1251
1412
|
"WITH ranked AS ("
|
|
@@ -1256,25 +1417,217 @@ def _search_like(conn, q, limit, offset):
|
|
|
1256
1417
|
" ORDER BY timestamp_utc DESC, id DESC"
|
|
1257
1418
|
" ) AS rn "
|
|
1258
1419
|
" FROM conversation_messages "
|
|
1259
|
-
" WHERE
|
|
1420
|
+
f" WHERE {col_pred}{et_pred}) "
|
|
1260
1421
|
"SELECT rid, sid, uuid, ts, cwd, mid, rqd FROM ranked WHERE rn = 1 "
|
|
1261
1422
|
"ORDER BY ts DESC, rid DESC LIMIT ? OFFSET ?",
|
|
1262
|
-
(
|
|
1423
|
+
(*like_args, *et_args, limit, offset),
|
|
1263
1424
|
).fetchall()
|
|
1264
1425
|
texts = _texts_for_ids(conn, [r[0] for r in page])
|
|
1426
|
+
if legacy:
|
|
1427
|
+
badges = {(sid, uuid): [] for (rid, sid, uuid, *_r) in page}
|
|
1428
|
+
else:
|
|
1429
|
+
badges = _like_badges(conn, like, list(
|
|
1430
|
+
{(sid, uuid) for (rid, sid, uuid, *_r) in page}))
|
|
1265
1431
|
hits = [_row_to_hit(uuid, sid, ts, cwd,
|
|
1266
|
-
_manual_snippet(texts.get(rid, ""), q), mid, rqd
|
|
1432
|
+
_manual_snippet(texts.get(rid, ""), q), mid, rqd,
|
|
1433
|
+
match_kinds=badges.get((sid, uuid), []))
|
|
1267
1434
|
for (rid, sid, uuid, ts, cwd, mid, rqd) in page]
|
|
1268
1435
|
return {"query": q, "mode": "like",
|
|
1269
1436
|
"hits": _attach_titles(conn, _attach_costs(conn, hits)),
|
|
1270
1437
|
"total": total}
|
|
1271
1438
|
|
|
1272
1439
|
|
|
1273
|
-
def
|
|
1440
|
+
def _like_badges(conn, like, groups):
|
|
1441
|
+
"""{(sid, uuid) -> sorted [badges]} via per-column LIKE probes across all
|
|
1442
|
+
physical rows of each page group (LIKE degraded mode; spec F7)."""
|
|
1443
|
+
if not groups:
|
|
1444
|
+
return {}
|
|
1445
|
+
out = {g: [] for g in groups}
|
|
1446
|
+
ph = " OR ".join("(session_id=? AND uuid=?)" for _ in groups)
|
|
1447
|
+
flat = [v for g in groups for v in g]
|
|
1448
|
+
for col, label in _KIND_PROBE_COLUMNS:
|
|
1449
|
+
rows = conn.execute(
|
|
1450
|
+
f"SELECT DISTINCT session_id, uuid FROM conversation_messages "
|
|
1451
|
+
f"WHERE {col} LIKE ? ESCAPE '\\' AND {col} != '' AND ({ph})",
|
|
1452
|
+
(like, *flat)).fetchall()
|
|
1453
|
+
for sid, uuid in rows:
|
|
1454
|
+
if (sid, uuid) in out:
|
|
1455
|
+
out[(sid, uuid)].append(label)
|
|
1456
|
+
return {g: sorted(v) for g, v in out.items()}
|
|
1457
|
+
|
|
1458
|
+
|
|
1459
|
+
# #177 S6: kind facets. `all` is the unfiltered MATCH; `prompts`/`assistant`
|
|
1460
|
+
# filter the prose column AND the entry_type; `tools`/`thinking` filter the
|
|
1461
|
+
# split index columns. Validated in search_conversations / find_in_conversation
|
|
1462
|
+
# (an unknown kind raises ValueError → the route maps it to a 400).
|
|
1463
|
+
_SEARCH_KINDS = ("all", "prompts", "assistant", "tools", "thinking")
|
|
1464
|
+
_KIND_COLUMN = {"prompts": "text", "assistant": "text",
|
|
1465
|
+
"tools": "search_tool", "thinking": "search_thinking"}
|
|
1466
|
+
_KIND_ENTRY_TYPE = {"prompts": "human", "assistant": "assistant"}
|
|
1467
|
+
|
|
1468
|
+
|
|
1469
|
+
def _fts_query(q, prefix_last=False):
|
|
1274
1470
|
"""Quote each whitespace term as an FTS5 string literal so punctuation /
|
|
1275
|
-
operators in user input can't error the MATCH or inject FTS syntax.
|
|
1471
|
+
operators in user input can't error the MATCH or inject FTS syntax. When
|
|
1472
|
+
``prefix_last`` is set, the final term gets a trailing ``*`` (valid FTS5
|
|
1473
|
+
quoted-prefix syntax) so ``cache.d`` matches ``cache.db`` while typing — a
|
|
1474
|
+
``*`` INSIDE the quotes is a literal char, so the prefix marker lives
|
|
1475
|
+
outside the closing quote (#177 S6)."""
|
|
1276
1476
|
terms = [t for t in q.split() if t]
|
|
1277
|
-
|
|
1477
|
+
if not terms:
|
|
1478
|
+
return '""'
|
|
1479
|
+
quoted = ['"' + t.replace('"', '""') + '"' for t in terms]
|
|
1480
|
+
if prefix_last:
|
|
1481
|
+
quoted[-1] += "*"
|
|
1482
|
+
return " ".join(quoted)
|
|
1483
|
+
|
|
1484
|
+
|
|
1485
|
+
def _kind_match_expr(kind, fts_q):
|
|
1486
|
+
"""Wrap the term expression in a column filter for the kind (#177 S6).
|
|
1487
|
+
``all`` stays unfiltered; ``prompts``/``assistant`` filter the prose column
|
|
1488
|
+
(the entry_type split is a separate SQL predicate, applied by the caller)."""
|
|
1489
|
+
col = _KIND_COLUMN.get(kind)
|
|
1490
|
+
return f"{{{col}}}: ({fts_q})" if col else fts_q
|
|
1491
|
+
|
|
1492
|
+
|
|
1493
|
+
# ===========================================================================
|
|
1494
|
+
# #177 S6: in-conversation find — rendered-turn anchors (spec §2 find endpoint).
|
|
1495
|
+
# ===========================================================================
|
|
1496
|
+
|
|
1497
|
+
_FIND_ANCHOR_CAP = 500
|
|
1498
|
+
|
|
1499
|
+
# Which physical-row columns the find match probes per kind, and the badge label
|
|
1500
|
+
# each non-prose column contributes. ``text`` maps to the synthetic ``prose``
|
|
1501
|
+
# label so a prose-only match still anchors a turn but never badges.
|
|
1502
|
+
_FIND_KIND_COLUMNS = {
|
|
1503
|
+
"all": (("text", "prose"), ("search_tool", "tool"),
|
|
1504
|
+
("search_thinking", "thinking")),
|
|
1505
|
+
"prompts": (("text", "prose"),),
|
|
1506
|
+
"assistant": (("text", "prose"),),
|
|
1507
|
+
"tools": (("search_tool", "tool"),),
|
|
1508
|
+
"thinking": (("search_thinking", "thinking"),),
|
|
1509
|
+
}
|
|
1510
|
+
|
|
1511
|
+
|
|
1512
|
+
def find_in_conversation(conn, session_id, query, *, kind="all",
|
|
1513
|
+
fts_available=None, cap=_FIND_ANCHOR_CAP):
|
|
1514
|
+
"""Document-ordered rendered-turn anchors for in-conversation find (#177 S6).
|
|
1515
|
+
|
|
1516
|
+
Anchor identity is rendered-turn identity (spec F1): the FTS/LIKE match for
|
|
1517
|
+
the session yields physical-row uuids, then ``_assemble_session`` (the S5
|
|
1518
|
+
outline precedent — 1:1 grouping parity by construction) maps each matched
|
|
1519
|
+
row onto its rendered item via ``member_uuids``. Matched rows folding into
|
|
1520
|
+
the same item (assistant fragments, owned tool results, skill bodies)
|
|
1521
|
+
collapse to ONE anchor whose ``match_kinds`` aggregates across its matched
|
|
1522
|
+
members; document order = assembly order (bm25 unused here). ``total`` counts
|
|
1523
|
+
rendered-turn anchors PRE-cap; the list caps at ``cap`` with
|
|
1524
|
+
``anchors_truncated``. Returns None for an unknown session; an unknown
|
|
1525
|
+
``kind`` raises ValueError (route → 400). Empty/whitespace query → empty;
|
|
1526
|
+
prose-only depth + tools/thinking kinds → empty (the split index is pending)."""
|
|
1527
|
+
if kind not in _SEARCH_KINDS:
|
|
1528
|
+
raise ValueError(f"unknown kind: {kind}")
|
|
1529
|
+
# Cheap existence probe (one indexed SELECT) BEFORE the full assembly, so an
|
|
1530
|
+
# empty/prose-only-blocked query opening the find bar pays nothing — yet the
|
|
1531
|
+
# unknown-session → None contract (the route's 404) is preserved, including
|
|
1532
|
+
# for an empty query (assembly used to run first and gave the same answer).
|
|
1533
|
+
if conn.execute(
|
|
1534
|
+
"SELECT 1 FROM conversation_messages WHERE session_id=? LIMIT 1",
|
|
1535
|
+
(session_id,)).fetchone() is None:
|
|
1536
|
+
return None
|
|
1537
|
+
depth = _search_depth(conn)
|
|
1538
|
+
if fts_available is None:
|
|
1539
|
+
fts_available = not _fts_flag_unavailable(conn)
|
|
1540
|
+
q = (query or "").strip()
|
|
1541
|
+
base = {"total": 0, "anchors": [], "anchors_truncated": False,
|
|
1542
|
+
"search_depth": depth, "kind": kind,
|
|
1543
|
+
"mode": "fts" if fts_available else "like"}
|
|
1544
|
+
if not q or (depth == "prose-only" and kind in ("tools", "thinking")):
|
|
1545
|
+
return base
|
|
1546
|
+
asm = _assemble_session(conn, session_id)
|
|
1547
|
+
if asm is None:
|
|
1548
|
+
return None
|
|
1549
|
+
mode, matched = _find_matched_rows(
|
|
1550
|
+
conn, session_id, q, kind, depth, fts_available)
|
|
1551
|
+
# matched: {uuid -> set of labels in {"prose", "tool", "thinking"}}
|
|
1552
|
+
anchors = []
|
|
1553
|
+
for it in asm["items"]:
|
|
1554
|
+
hit_kinds = set()
|
|
1555
|
+
hit = False
|
|
1556
|
+
for mu in it["member_uuids"]:
|
|
1557
|
+
labels = matched.get(mu)
|
|
1558
|
+
if labels:
|
|
1559
|
+
hit = True
|
|
1560
|
+
hit_kinds |= labels
|
|
1561
|
+
if hit:
|
|
1562
|
+
anchors.append({
|
|
1563
|
+
"uuid": it["anchor"]["uuid"],
|
|
1564
|
+
"match_kinds": sorted(k for k in hit_kinds if k != "prose")})
|
|
1565
|
+
total = len(anchors)
|
|
1566
|
+
return {**base, "mode": mode, "total": total,
|
|
1567
|
+
"anchors": anchors[:cap], "anchors_truncated": total > cap}
|
|
1568
|
+
|
|
1569
|
+
|
|
1570
|
+
def _find_matched_rows(conn, session_id, q, kind, depth, fts_available):
|
|
1571
|
+
"""({mode}, {uuid -> {labels}}) for one session. Runs a column-scoped MATCH
|
|
1572
|
+
(or LIKE) per relevant column and tags each matched row's uuid with that
|
|
1573
|
+
column's label. Prose-only depth uses the legacy single-column FTS (the
|
|
1574
|
+
split columns aren't indexed yet) for prose-bearing kinds."""
|
|
1575
|
+
if fts_available:
|
|
1576
|
+
try:
|
|
1577
|
+
return "fts", _find_matched_fts(conn, session_id, q, kind, depth)
|
|
1578
|
+
except sqlite3.OperationalError:
|
|
1579
|
+
pass # corrupt/missing FTS → fall through to LIKE
|
|
1580
|
+
return "like", _find_matched_like(conn, session_id, q, kind, depth)
|
|
1581
|
+
|
|
1582
|
+
|
|
1583
|
+
def _find_kind_columns(kind, depth):
|
|
1584
|
+
"""The (column, label) probes the find match runs for this (kind, depth).
|
|
1585
|
+
Prose-only depth has only the legacy prose column indexed, so the split
|
|
1586
|
+
tool/thinking columns drop out (a prose-bearing kind keeps its prose probe;
|
|
1587
|
+
tools/thinking yield nothing). Shared by _find_matched_fts / _find_matched_like
|
|
1588
|
+
so the two paths can never disagree on which columns a kind probes."""
|
|
1589
|
+
if depth == "prose-only":
|
|
1590
|
+
return (("text", "prose"),) if kind in ("all", "prompts", "assistant") else ()
|
|
1591
|
+
return _FIND_KIND_COLUMNS[kind]
|
|
1592
|
+
|
|
1593
|
+
|
|
1594
|
+
def _find_matched_fts(conn, session_id, q, kind, depth):
|
|
1595
|
+
fts_q = _fts_query(q, prefix_last=True)
|
|
1596
|
+
# entry_type predicate + the prose-only legacy MATCH shape are loop-invariant
|
|
1597
|
+
# — compute once. Legacy single-column FTS indexes prose only, so it MATCHes
|
|
1598
|
+
# the bare term (no column filter); full mode wraps each column.
|
|
1599
|
+
et = _KIND_ENTRY_TYPE.get(kind)
|
|
1600
|
+
et_pred = " AND cm.entry_type = ?" if et is not None else ""
|
|
1601
|
+
et_args = (et,) if et is not None else ()
|
|
1602
|
+
legacy = depth == "prose-only"
|
|
1603
|
+
out = {}
|
|
1604
|
+
for col, label in _find_kind_columns(kind, depth):
|
|
1605
|
+
match_expr = fts_q if legacy else f"{{{col}}}: ({fts_q})"
|
|
1606
|
+
rows = conn.execute(
|
|
1607
|
+
"SELECT cm.uuid FROM conversation_fts "
|
|
1608
|
+
"JOIN conversation_messages cm ON cm.id = conversation_fts.rowid "
|
|
1609
|
+
f"WHERE conversation_fts MATCH ? AND cm.session_id = ?{et_pred}",
|
|
1610
|
+
(match_expr, session_id, *et_args)).fetchall()
|
|
1611
|
+
for (u,) in rows:
|
|
1612
|
+
out.setdefault(u, set()).add(label)
|
|
1613
|
+
return out
|
|
1614
|
+
|
|
1615
|
+
|
|
1616
|
+
def _find_matched_like(conn, session_id, q, kind, depth):
|
|
1617
|
+
like = _like_pattern(q)
|
|
1618
|
+
et = _KIND_ENTRY_TYPE.get(kind)
|
|
1619
|
+
et_pred = " AND entry_type = ?" if et is not None else ""
|
|
1620
|
+
et_args = (et,) if et is not None else ()
|
|
1621
|
+
out = {}
|
|
1622
|
+
for col, label in _find_kind_columns(kind, depth):
|
|
1623
|
+
rows = conn.execute(
|
|
1624
|
+
f"SELECT uuid FROM conversation_messages "
|
|
1625
|
+
f"WHERE session_id = ? AND {col} LIKE ? ESCAPE '\\' "
|
|
1626
|
+
f"AND {col} != ''{et_pred}",
|
|
1627
|
+
(session_id, like, *et_args)).fetchall()
|
|
1628
|
+
for (u,) in rows:
|
|
1629
|
+
out.setdefault(u, set()).add(label)
|
|
1630
|
+
return out
|
|
1278
1631
|
|
|
1279
1632
|
|
|
1280
1633
|
def _manual_snippet(text, q, width=80):
|