cctally 1.27.1 → 1.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +37 -0
- package/bin/_cctally_alerts.py +26 -1
- package/bin/_cctally_cache.py +355 -31
- package/bin/_cctally_config.py +153 -11
- package/bin/_cctally_core.py +204 -42
- package/bin/_cctally_dashboard.py +510 -61
- package/bin/_cctally_db.py +756 -163
- package/bin/_cctally_doctor.py +11 -0
- package/bin/_cctally_forecast.py +700 -57
- package/bin/_cctally_milestones.py +252 -47
- package/bin/_cctally_parser.py +44 -4
- package/bin/_cctally_record.py +380 -133
- package/bin/_cctally_weekrefs.py +30 -6
- package/bin/_lib_alert_axes.py +12 -2
- package/bin/_lib_alerts_payload.py +95 -3
- package/bin/_lib_budget.py +48 -0
- package/bin/_lib_conversation.py +177 -0
- package/bin/_lib_conversation_query.py +620 -0
- package/bin/_lib_doctor.py +60 -1
- package/bin/_lib_jsonl.py +69 -50
- package/bin/_lib_transcript_access.py +80 -0
- package/bin/cctally +29 -2
- package/dashboard/static/assets/index-BGaWg6ys.js +47 -0
- package/dashboard/static/assets/{index-D34qf0LE.css → index-BqQ5xdX0.css} +1 -1
- package/dashboard/static/dashboard.html +2 -2
- package/package.json +4 -1
- package/dashboard/static/assets/index-C2F1_Mxt.js +0 -18
|
@@ -0,0 +1,620 @@
|
|
|
1
|
+
"""Pure query kernel for the conversation viewer endpoints (Plan 2, spec §3).
|
|
2
|
+
|
|
3
|
+
Takes a sqlite3.Connection over a cache.db that already holds Plan 1's
|
|
4
|
+
conversation_messages (+ FTS) and session_entries. No clock, no network, no
|
|
5
|
+
global mutation — unit-tested against an in-memory cache.db seeded by
|
|
6
|
+
_apply_cache_schema. Three entry points back the three GET routes:
|
|
7
|
+
list_conversations (rail), get_conversation (reader), search_conversations.
|
|
8
|
+
|
|
9
|
+
Cost is joined ONCE per logical assistant turn (msg_id, req_id) to the single
|
|
10
|
+
deduped session_entries row (idx_entries_dedup), via the shared pricing helper
|
|
11
|
+
— never per physical fragment and never from cost_usd_raw (often NULL).
|
|
12
|
+
"""
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
import json as _json
|
|
15
|
+
import os
|
|
16
|
+
import sqlite3
|
|
17
|
+
|
|
18
|
+
# Public surface (Plan 2): shipped in the npm tarball + brew formula + public
|
|
19
|
+
# mirror — imported by the dashboard's conversation endpoints at runtime.
|
|
20
|
+
|
|
21
|
+
from _lib_pricing import _calculate_entry_cost
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _project_label(cwd) -> str:
|
|
25
|
+
"""Basename of the project cwd (dashboard label posture — no reveal). Falls
|
|
26
|
+
back to the raw path for root-ish cwds, '' when absent."""
|
|
27
|
+
if not cwd:
|
|
28
|
+
return ""
|
|
29
|
+
return os.path.basename(cwd.rstrip("/")) or cwd
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _subagent_key(source_path):
|
|
33
|
+
"""Privacy-safe subagent-thread identity for the reader. Each subagent (Task)
|
|
34
|
+
invocation writes its own ``agent-<hash>.jsonl``; the main session is
|
|
35
|
+
``<session_id>.jsonl``. Returns the agent hash (``agent-`` prefix + ``.jsonl``
|
|
36
|
+
suffix stripped; an ``acompact-`` middle is kept), or ``None`` for the main
|
|
37
|
+
file / a non-agent path. We expose ONLY this derived key — never the raw
|
|
38
|
+
absolute ``source_path`` (which leaks home dir / username / encoded project,
|
|
39
|
+
and the conversation routes are LAN-exposable via dashboard.expose_transcripts)."""
|
|
40
|
+
if not source_path:
|
|
41
|
+
return None
|
|
42
|
+
base = os.path.basename(source_path)
|
|
43
|
+
if not base.startswith("agent-"):
|
|
44
|
+
return None
|
|
45
|
+
stem = base[len("agent-"):]
|
|
46
|
+
if stem.endswith(".jsonl"):
|
|
47
|
+
stem = stem[: -len(".jsonl")]
|
|
48
|
+
return stem or None
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _entry_cost(model, inp, out, cc, cr, cost_usd_raw) -> float:
|
|
52
|
+
"""Cost for one session_entries row via the shared pricing helper. Tokens →
|
|
53
|
+
the helper's usage dict. cost_usd_raw is passed as the optional override the
|
|
54
|
+
helper already understands (it is often NULL — never the primary source)."""
|
|
55
|
+
usage = {
|
|
56
|
+
"input_tokens": inp or 0,
|
|
57
|
+
"output_tokens": out or 0,
|
|
58
|
+
"cache_creation_input_tokens": cc or 0,
|
|
59
|
+
"cache_read_input_tokens": cr or 0,
|
|
60
|
+
}
|
|
61
|
+
return _calculate_entry_cost(model or "", usage, cost_usd=cost_usd_raw)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _session_cost_map(conn, session_ids):
|
|
65
|
+
"""{session_id: total_cost_usd} for the given sessions. Joins
|
|
66
|
+
conversation_messages turn keys to the single deduped session_entries row
|
|
67
|
+
per (msg_id, req_id), so a turn replayed across files contributes once.
|
|
68
|
+
(msg_id, req_id) is globally unique in session_entries and maps to exactly
|
|
69
|
+
one session_id, so per-session sums are clean."""
|
|
70
|
+
costs = {sid: 0.0 for sid in session_ids}
|
|
71
|
+
if not session_ids:
|
|
72
|
+
return costs
|
|
73
|
+
placeholders = ",".join("?" for _ in session_ids)
|
|
74
|
+
sql = (
|
|
75
|
+
"SELECT cm.session_id, se.model, se.input_tokens, se.output_tokens, "
|
|
76
|
+
" se.cache_create_tokens, se.cache_read_tokens, se.cost_usd_raw "
|
|
77
|
+
"FROM (SELECT DISTINCT session_id, msg_id, req_id "
|
|
78
|
+
" FROM conversation_messages "
|
|
79
|
+
" WHERE session_id IN (%s) AND msg_id IS NOT NULL AND req_id IS NOT NULL) cm "
|
|
80
|
+
"JOIN session_entries se ON se.msg_id = cm.msg_id AND se.req_id = cm.req_id"
|
|
81
|
+
% placeholders
|
|
82
|
+
)
|
|
83
|
+
for sid, model, inp, out, cc, cr, raw in conn.execute(sql, list(session_ids)):
|
|
84
|
+
costs[sid] = costs.get(sid, 0.0) + _entry_cost(model, inp, out, cc, cr, raw)
|
|
85
|
+
return costs
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _session_models_map(conn, session_ids):
|
|
89
|
+
"""{session_id: sorted distinct non-null models}."""
|
|
90
|
+
out = {sid: [] for sid in session_ids}
|
|
91
|
+
if not session_ids:
|
|
92
|
+
return out
|
|
93
|
+
placeholders = ",".join("?" for _ in session_ids)
|
|
94
|
+
sql = (
|
|
95
|
+
"SELECT DISTINCT session_id, model FROM conversation_messages "
|
|
96
|
+
"WHERE session_id IN (%s) AND model IS NOT NULL AND model != '' "
|
|
97
|
+
"ORDER BY model" % placeholders
|
|
98
|
+
)
|
|
99
|
+
for sid, model in conn.execute(sql, list(session_ids)):
|
|
100
|
+
out.setdefault(sid, []).append(model)
|
|
101
|
+
return out
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _session_latest_meta_map(conn, session_ids):
|
|
105
|
+
"""{session_id: (cwd, git_branch)} using the most-recent NON-NULL value per
|
|
106
|
+
column — the SAME posture as get_conversation's _latest, so the rail and the
|
|
107
|
+
reader agree on a session whose cwd/branch changed over its lifetime (a plain
|
|
108
|
+
MAX() picks the lexical max, not the latest). Bounded to the page's sessions
|
|
109
|
+
via per-session correlated lookups over idx (session_id, timestamp_utc, id),
|
|
110
|
+
mirroring _session_cost_map / _session_models_map."""
|
|
111
|
+
meta = {sid: (None, None) for sid in session_ids}
|
|
112
|
+
if not session_ids:
|
|
113
|
+
return meta
|
|
114
|
+
placeholders = ",".join("?" for _ in session_ids)
|
|
115
|
+
sql = (
|
|
116
|
+
"SELECT s.session_id, "
|
|
117
|
+
" (SELECT c.cwd FROM conversation_messages c "
|
|
118
|
+
" WHERE c.session_id = s.session_id AND c.cwd IS NOT NULL "
|
|
119
|
+
" ORDER BY c.timestamp_utc DESC, c.id DESC LIMIT 1), "
|
|
120
|
+
" (SELECT b.git_branch FROM conversation_messages b "
|
|
121
|
+
" WHERE b.session_id = s.session_id AND b.git_branch IS NOT NULL "
|
|
122
|
+
" ORDER BY b.timestamp_utc DESC, b.id DESC LIMIT 1) "
|
|
123
|
+
"FROM (SELECT DISTINCT session_id FROM conversation_messages "
|
|
124
|
+
" WHERE session_id IN (%s)) s" % placeholders
|
|
125
|
+
)
|
|
126
|
+
for sid, cwd, branch in conn.execute(sql, list(session_ids)):
|
|
127
|
+
meta[sid] = (cwd, branch)
|
|
128
|
+
return meta
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
_SORTS = {
|
|
132
|
+
"recent": "MAX(timestamp_utc) DESC, session_id DESC",
|
|
133
|
+
"oldest": "MIN(timestamp_utc) ASC, session_id ASC",
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def list_conversations(conn, *, sort="recent", limit=50, offset=0) -> dict:
|
|
138
|
+
"""All-history per-session browse rows (spec §3.1). NOT 365-day bounded."""
|
|
139
|
+
order = _SORTS.get(sort, _SORTS["recent"])
|
|
140
|
+
limit = max(1, min(int(limit), 200))
|
|
141
|
+
offset = max(0, int(offset))
|
|
142
|
+
rows = conn.execute(
|
|
143
|
+
"SELECT session_id, COUNT(*) AS msg_count, "
|
|
144
|
+
" MIN(timestamp_utc) AS started, MAX(timestamp_utc) AS last_activity "
|
|
145
|
+
"FROM conversation_messages "
|
|
146
|
+
"WHERE session_id IS NOT NULL "
|
|
147
|
+
"GROUP BY session_id "
|
|
148
|
+
"ORDER BY " + order + " LIMIT ? OFFSET ?",
|
|
149
|
+
(limit + 1, offset),
|
|
150
|
+
).fetchall()
|
|
151
|
+
has_more = len(rows) > limit
|
|
152
|
+
rows = rows[:limit]
|
|
153
|
+
session_ids = [r[0] for r in rows]
|
|
154
|
+
costs = _session_cost_map(conn, session_ids)
|
|
155
|
+
models = _session_models_map(conn, session_ids)
|
|
156
|
+
# cwd/git_branch as the latest non-null (reader posture), NOT a lexical MAX().
|
|
157
|
+
meta = _session_latest_meta_map(conn, session_ids)
|
|
158
|
+
conversations = [
|
|
159
|
+
{
|
|
160
|
+
"session_id": sid,
|
|
161
|
+
"project_label": _project_label(meta.get(sid, (None, None))[0]),
|
|
162
|
+
"git_branch": meta.get(sid, (None, None))[1],
|
|
163
|
+
"started_utc": started,
|
|
164
|
+
"last_activity_utc": last_activity,
|
|
165
|
+
"msg_count": msg_count,
|
|
166
|
+
"cost_usd": round(costs.get(sid, 0.0), 6),
|
|
167
|
+
"models": models.get(sid, []),
|
|
168
|
+
}
|
|
169
|
+
for (sid, msg_count, started, last_activity) in rows
|
|
170
|
+
]
|
|
171
|
+
return {
|
|
172
|
+
"conversations": conversations,
|
|
173
|
+
"page": {
|
|
174
|
+
"next_offset": offset + len(conversations) if has_more else None,
|
|
175
|
+
"has_more": has_more,
|
|
176
|
+
},
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def _turn_cost_map(conn, turn_keys):
|
|
181
|
+
"""{(msg_id, req_id): cost_usd} for the given non-null turn keys, joined ONCE
|
|
182
|
+
to the deduped session_entries row. Keys absent from session_entries (e.g.
|
|
183
|
+
<synthetic> walker-skipped rows) are simply not present → cost 0 by omission."""
|
|
184
|
+
costs = {}
|
|
185
|
+
keys = [(m, r) for (m, r) in turn_keys if m is not None and r is not None]
|
|
186
|
+
if not keys:
|
|
187
|
+
return costs
|
|
188
|
+
# Chunk the OR-of-pairs to stay well under SQLite's variable limit.
|
|
189
|
+
for i in range(0, len(keys), 400):
|
|
190
|
+
chunk = keys[i:i + 400]
|
|
191
|
+
cond = " OR ".join("(msg_id=? AND req_id=?)" for _ in chunk)
|
|
192
|
+
params = [v for pair in chunk for v in pair]
|
|
193
|
+
sql = ("SELECT msg_id, req_id, model, input_tokens, output_tokens, "
|
|
194
|
+
"cache_create_tokens, cache_read_tokens, cost_usd_raw "
|
|
195
|
+
"FROM session_entries WHERE " + cond)
|
|
196
|
+
for m, r, model, inp, out, cc, cr, raw in conn.execute(sql, params):
|
|
197
|
+
costs[(m, r)] = _entry_cost(model, inp, out, cc, cr, raw)
|
|
198
|
+
return costs
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def get_conversation(conn, session_id, *, after=None, limit=500):
|
|
202
|
+
"""Reader payload for one session (spec §3.2). Returns None for an unknown
|
|
203
|
+
session. Dedups logical messages by (session_id, uuid) (canonical = earliest
|
|
204
|
+
timestamp), groups assistant fragments into turn items by (msg_id, req_id),
|
|
205
|
+
joins cost once, anchors a turn on its prose-bearing fragment, and exposes
|
|
206
|
+
every member fragment uuid for jump resolution. Cursor over (timestamp_utc,
|
|
207
|
+
id); ~500 items/page."""
|
|
208
|
+
limit = max(1, min(int(limit), 1000))
|
|
209
|
+
exists = conn.execute(
|
|
210
|
+
"SELECT 1 FROM conversation_messages WHERE session_id=? LIMIT 1",
|
|
211
|
+
(session_id,)).fetchone()
|
|
212
|
+
if exists is None:
|
|
213
|
+
return None
|
|
214
|
+
|
|
215
|
+
# Pull the session ordered; dedup logical messages by (session_id, uuid),
|
|
216
|
+
# canonical row = earliest (timestamp_utc, id). Replays carry the original
|
|
217
|
+
# uuid, so the first occurrence in ascending order is canonical.
|
|
218
|
+
raw = conn.execute(
|
|
219
|
+
"SELECT id, uuid, timestamp_utc, entry_type, text, blocks_json, model, "
|
|
220
|
+
" msg_id, req_id, is_sidechain, cwd, git_branch, source_path, parent_uuid "
|
|
221
|
+
"FROM conversation_messages WHERE session_id=? "
|
|
222
|
+
"ORDER BY timestamp_utc, id", (session_id,)).fetchall()
|
|
223
|
+
|
|
224
|
+
seen_uuid = set()
|
|
225
|
+
logical = [] # canonical physical rows, in order
|
|
226
|
+
for row in raw:
|
|
227
|
+
u = row[1]
|
|
228
|
+
if u in seen_uuid:
|
|
229
|
+
continue
|
|
230
|
+
seen_uuid.add(u)
|
|
231
|
+
logical.append(row)
|
|
232
|
+
|
|
233
|
+
# Group assistant fragments sharing (msg_id, req_id) into one turn item over
|
|
234
|
+
# the WHOLE logical list — NOT by adjacency. Real tool-using transcripts
|
|
235
|
+
# interleave a tool_result (a `user`/tool_result item) between fragments of
|
|
236
|
+
# the SAME turn, so the same key recurs non-consecutively. We keep a turn-key
|
|
237
|
+
# → item-index map: first occurrence emits the turn item AT THIS POSITION;
|
|
238
|
+
# later same-key fragments fold their blocks/prose/uuids into the existing
|
|
239
|
+
# item. A turn → exactly ONE item → cost counted exactly once. Humans,
|
|
240
|
+
# tool_results, and assistant rows with a null msg_id emit as simple items at
|
|
241
|
+
# their own position.
|
|
242
|
+
items = []
|
|
243
|
+
turn_index = {} # (msg_id, req_id) -> index into items
|
|
244
|
+
for row in logical:
|
|
245
|
+
(rid, u, ts, etype, text, blocks, model, msg_id, req_id,
|
|
246
|
+
is_sc, cwd, branch, source_path, parent_uuid) = row
|
|
247
|
+
if etype == "assistant" and msg_id is not None:
|
|
248
|
+
key = (msg_id, req_id)
|
|
249
|
+
idx = turn_index.get(key)
|
|
250
|
+
if idx is None:
|
|
251
|
+
turn_index[key] = len(items)
|
|
252
|
+
items.append(_build_turn([row]))
|
|
253
|
+
else:
|
|
254
|
+
_extend_turn(items[idx], row)
|
|
255
|
+
else:
|
|
256
|
+
items.append(_build_simple(row))
|
|
257
|
+
|
|
258
|
+
costs = _turn_cost_map(conn, list(turn_index))
|
|
259
|
+
# Stamp per-item cost first, then derive the header from the SUM of the
|
|
260
|
+
# ROUNDED per-item assistant costs (M2) — so the §6.5 invariant
|
|
261
|
+
# sum(items.cost_usd) == header cost_usd holds EXACTLY to 1e-9 by
|
|
262
|
+
# construction OVER THE FULL ITEM LIST. 6dp is the deliberate JSON display
|
|
263
|
+
# precision. NOTE: the header is the whole-session total; the returned
|
|
264
|
+
# ``items`` is a page subset, so on page 2+ sum(page) < header by design.
|
|
265
|
+
header_cost = 0.0
|
|
266
|
+
for it in items:
|
|
267
|
+
if it["kind"] == "assistant" and "_msg_id" in it:
|
|
268
|
+
turn_cost = round(costs.get((it["_msg_id"], it["_req_id"]), 0.0), 6)
|
|
269
|
+
it["cost_usd"] = turn_cost
|
|
270
|
+
header_cost += turn_cost
|
|
271
|
+
del it["_msg_id"]
|
|
272
|
+
del it["_req_id"]
|
|
273
|
+
it.pop("_has_prose", None)
|
|
274
|
+
header_cost = round(header_cost, 6)
|
|
275
|
+
|
|
276
|
+
# Cursor pagination over the item list (anchored to each item's canonical id).
|
|
277
|
+
# A non-None `after` that matches no item's anchor (stale/deleted cursor)
|
|
278
|
+
# yields an EMPTY page — never silently re-serves the head (M1).
|
|
279
|
+
start = 0
|
|
280
|
+
if after is not None:
|
|
281
|
+
start = None
|
|
282
|
+
for k, it in enumerate(items):
|
|
283
|
+
if str(it["anchor"]["id"]) == str(after):
|
|
284
|
+
start = k + 1
|
|
285
|
+
break
|
|
286
|
+
if start is None:
|
|
287
|
+
return {
|
|
288
|
+
"session_id": session_id,
|
|
289
|
+
"project_label": _project_label(_latest(logical, 10)),
|
|
290
|
+
"git_branch": _latest(logical, 11),
|
|
291
|
+
"started_utc": logical[0][2],
|
|
292
|
+
"last_activity_utc": logical[-1][2],
|
|
293
|
+
"cost_usd": header_cost,
|
|
294
|
+
"models": sorted({r[6] for r in logical if r[6]}),
|
|
295
|
+
"items": [],
|
|
296
|
+
"page": {"next_after": None, "has_more": False},
|
|
297
|
+
}
|
|
298
|
+
page = items[start:start + limit]
|
|
299
|
+
has_more = start + limit < len(items)
|
|
300
|
+
next_after = page[-1]["anchor"]["id"] if (page and has_more) else None
|
|
301
|
+
|
|
302
|
+
# Stamp the session_id into each anchor (spec anchor is (session_id, uuid);
|
|
303
|
+
# the dict literals are built session-agnostic, so fill it here where the
|
|
304
|
+
# session id is known). NOT a no-op — the endpoint/clients rely on it.
|
|
305
|
+
for it in page:
|
|
306
|
+
it["anchor"]["session_id"] = session_id
|
|
307
|
+
|
|
308
|
+
first = logical[0]
|
|
309
|
+
last = logical[-1]
|
|
310
|
+
models = sorted({r[6] for r in logical if r[6]})
|
|
311
|
+
return {
|
|
312
|
+
"session_id": session_id,
|
|
313
|
+
"project_label": _project_label(_latest(logical, 10)),
|
|
314
|
+
"git_branch": _latest(logical, 11),
|
|
315
|
+
"started_utc": first[2],
|
|
316
|
+
"last_activity_utc": last[2],
|
|
317
|
+
"cost_usd": header_cost,
|
|
318
|
+
"models": models,
|
|
319
|
+
"items": page,
|
|
320
|
+
"page": {"next_after": next_after, "has_more": has_more},
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
def _latest(logical, col):
|
|
325
|
+
"""Most-recent non-null value in a column across the session (project/branch
|
|
326
|
+
show the latest, matching the dashboard's session posture)."""
|
|
327
|
+
for row in reversed(logical):
|
|
328
|
+
if row[col]:
|
|
329
|
+
return row[col]
|
|
330
|
+
return "" if col == 10 else None
|
|
331
|
+
|
|
332
|
+
|
|
333
|
+
def _build_turn(members):
|
|
334
|
+
"""Seed a turn item from its first fragment(s). Prose = joined non-empty
|
|
335
|
+
fragment text; anchor/model = the prose-bearing fragment (empirically exactly
|
|
336
|
+
one per turn); member_uuids = all fragment uuids. Fragments arriving later
|
|
337
|
+
(possibly non-consecutive — interleaved with a tool_result) fold in via
|
|
338
|
+
_extend_turn, which re-promotes the anchor/model once a prose fragment lands."""
|
|
339
|
+
first = members[0]
|
|
340
|
+
item = {
|
|
341
|
+
"kind": "assistant",
|
|
342
|
+
"anchor": {"session_id": None, "uuid": first[1], "id": first[0]},
|
|
343
|
+
"member_uuids": [first[1]],
|
|
344
|
+
"ts": first[2],
|
|
345
|
+
"text": "",
|
|
346
|
+
"blocks": [],
|
|
347
|
+
"model": first[6],
|
|
348
|
+
"is_sidechain": bool(first[9]),
|
|
349
|
+
# subagent_key / parent_uuid are SEED-sourced (the first fragment, the
|
|
350
|
+
# turn's entry point) and NOT re-promoted in _fold_fragment — the prose
|
|
351
|
+
# anchor's parent_uuid is an intra-turn link, not the entry point (Codex
|
|
352
|
+
# P1). subagent_key is uniform across a turn's fragments (one file).
|
|
353
|
+
"subagent_key": _subagent_key(first[12]),
|
|
354
|
+
"parent_uuid": first[13],
|
|
355
|
+
"_msg_id": first[7],
|
|
356
|
+
"_req_id": first[8],
|
|
357
|
+
"_has_prose": False,
|
|
358
|
+
}
|
|
359
|
+
_fold_fragment(item, first)
|
|
360
|
+
for m in members[1:]:
|
|
361
|
+
_extend_turn(item, m)
|
|
362
|
+
return item
|
|
363
|
+
|
|
364
|
+
|
|
365
|
+
def _extend_turn(item, row):
|
|
366
|
+
"""Fold one more same-turn assistant fragment into an existing turn item:
|
|
367
|
+
append its uuid + blocks + non-empty prose. The FIRST fragment carrying prose
|
|
368
|
+
promotes the anchor/model to itself (the prose-bearing fragment is the
|
|
369
|
+
canonical anchor); subsequent prose fragments only extend the joined text."""
|
|
370
|
+
item["member_uuids"].append(row[1])
|
|
371
|
+
_fold_fragment(item, row)
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
def _fold_fragment(item, row):
|
|
375
|
+
blocks = item["blocks"]
|
|
376
|
+
try:
|
|
377
|
+
blocks.extend(_json.loads(row[5] or "[]"))
|
|
378
|
+
except (ValueError, TypeError):
|
|
379
|
+
pass
|
|
380
|
+
frag_text = (row[4] or "").strip()
|
|
381
|
+
if frag_text:
|
|
382
|
+
if not item["_has_prose"]:
|
|
383
|
+
# First prose fragment becomes the canonical anchor / model.
|
|
384
|
+
item["anchor"]["uuid"] = row[1]
|
|
385
|
+
item["anchor"]["id"] = row[0]
|
|
386
|
+
item["model"] = row[6]
|
|
387
|
+
item["is_sidechain"] = bool(row[9])
|
|
388
|
+
item["_msg_id"] = row[7]
|
|
389
|
+
item["_req_id"] = row[8]
|
|
390
|
+
item["_has_prose"] = True
|
|
391
|
+
item["text"] = frag_text
|
|
392
|
+
else:
|
|
393
|
+
item["text"] = item["text"] + "\n" + frag_text
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
def _build_simple(row):
|
|
397
|
+
"""A human, tool_result, or assistant-with-null-msg_id item (no turn grouping,
|
|
398
|
+
no cost). An assistant row routes here only when its msg_id is NULL (no turn
|
|
399
|
+
key → no session_entries join); it carries an explicit cost_usd of 0.0 and NO
|
|
400
|
+
internal _msg_id/_req_id keys, so the cost loop's KeyError path can never fire
|
|
401
|
+
(I2). The model is preserved for assistant rows."""
|
|
402
|
+
(rid, u, ts, etype, text, blocks, model, msg_id, req_id, is_sc, cwd, branch,
|
|
403
|
+
source_path, parent_uuid) = row
|
|
404
|
+
try:
|
|
405
|
+
parsed = _json.loads(blocks or "[]")
|
|
406
|
+
except (ValueError, TypeError):
|
|
407
|
+
parsed = []
|
|
408
|
+
item = {
|
|
409
|
+
"kind": etype,
|
|
410
|
+
"anchor": {"session_id": None, "uuid": u, "id": rid},
|
|
411
|
+
"member_uuids": [u],
|
|
412
|
+
"ts": ts,
|
|
413
|
+
"text": text,
|
|
414
|
+
"blocks": parsed,
|
|
415
|
+
"is_sidechain": bool(is_sc),
|
|
416
|
+
"subagent_key": _subagent_key(source_path),
|
|
417
|
+
"parent_uuid": parent_uuid,
|
|
418
|
+
}
|
|
419
|
+
if etype == "assistant":
|
|
420
|
+
item["model"] = model
|
|
421
|
+
item["cost_usd"] = 0.0
|
|
422
|
+
return item
|
|
423
|
+
|
|
424
|
+
|
|
425
|
+
def _fts_flag_unavailable(conn) -> bool:
|
|
426
|
+
try:
|
|
427
|
+
row = conn.execute(
|
|
428
|
+
"SELECT value FROM cache_meta WHERE key='fts5_unavailable'").fetchone()
|
|
429
|
+
except sqlite3.OperationalError:
|
|
430
|
+
return False
|
|
431
|
+
return bool(row and row[0])
|
|
432
|
+
|
|
433
|
+
|
|
434
|
+
def search_conversations(conn, query, *, limit=50, offset=0,
|
|
435
|
+
fts_available=None) -> dict:
|
|
436
|
+
"""Cross-session search (spec §3.3). Uses FTS5 when available (bm25 rank +
|
|
437
|
+
snippet); else a LIKE scan with a manual snippet. Hits deduped by
|
|
438
|
+
(session_id, uuid); each carries the turn's cost. `fts_available` overrides
|
|
439
|
+
detection (test seam / explicit LIKE)."""
|
|
440
|
+
q = (query or "").strip()
|
|
441
|
+
limit = max(1, min(int(limit), 200))
|
|
442
|
+
offset = max(0, int(offset))
|
|
443
|
+
if fts_available is None:
|
|
444
|
+
fts_available = not _fts_flag_unavailable(conn)
|
|
445
|
+
if not q:
|
|
446
|
+
return {"query": q, "mode": "fts" if fts_available else "like",
|
|
447
|
+
"hits": [], "total": 0}
|
|
448
|
+
if fts_available:
|
|
449
|
+
try:
|
|
450
|
+
return _search_fts(conn, q, limit, offset)
|
|
451
|
+
except sqlite3.OperationalError:
|
|
452
|
+
pass # corrupt/missing FTS at query time → fall through to LIKE
|
|
453
|
+
return _search_like(conn, q, limit, offset)
|
|
454
|
+
|
|
455
|
+
|
|
456
|
+
def _row_to_hit(uuid_, sid, ts, cwd, snippet, msg_id, req_id):
|
|
457
|
+
"""Build one hit WITHOUT cost — cost is batched onto the FINAL page in
|
|
458
|
+
_attach_costs (I1: no per-hit _turn_cost_map round-trip). The turn key rides
|
|
459
|
+
on the private `_turn_key` field until the batch maps it to `cost_usd`."""
|
|
460
|
+
return {
|
|
461
|
+
"session_id": sid,
|
|
462
|
+
"uuid": uuid_,
|
|
463
|
+
"project_label": _project_label(cwd),
|
|
464
|
+
"ts": ts,
|
|
465
|
+
"snippet": snippet,
|
|
466
|
+
"_turn_key": (msg_id, req_id) if msg_id is not None and req_id is not None
|
|
467
|
+
else None,
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
|
|
471
|
+
def _attach_costs(conn, page):
|
|
472
|
+
"""Compute turn cost for the FINAL page's hits in ONE _turn_cost_map call,
|
|
473
|
+
then map it onto each hit and drop the private `_turn_key`. Off-page and
|
|
474
|
+
duplicate hits never reach here, so we never compute cost for them (I1)."""
|
|
475
|
+
keys = [h["_turn_key"] for h in page if h.get("_turn_key") is not None]
|
|
476
|
+
costs = _turn_cost_map(conn, keys) if keys else {}
|
|
477
|
+
for h in page:
|
|
478
|
+
tk = h.pop("_turn_key", None)
|
|
479
|
+
h["cost_usd"] = round(costs.get(tk, 0.0), 6) if tk is not None else 0.0
|
|
480
|
+
return page
|
|
481
|
+
|
|
482
|
+
|
|
483
|
+
def _like_pattern(q):
|
|
484
|
+
"""Build the LIKE pattern for `q`. Escape the ESCAPE char (\\) FIRST, then
|
|
485
|
+
the wildcards — otherwise a query containing a backslash (incl. a trailing
|
|
486
|
+
one) mis-escapes the appended '%' and the LIKE silently matches nothing
|
|
487
|
+
(paired with ESCAPE '\\' in the queries below)."""
|
|
488
|
+
return ("%" + q.replace("\\", "\\\\").replace("%", r"\%").replace("_", r"\_")
|
|
489
|
+
+ "%")
|
|
490
|
+
|
|
491
|
+
|
|
492
|
+
def _fts_snippets(conn, fts_q, ids):
|
|
493
|
+
"""{rowid: snippet} for the page rowids ONLY (#149). snippet() needs an
|
|
494
|
+
active MATCH, so it can't be deferred to an outer query over the page CTE;
|
|
495
|
+
a second bounded MATCH restricted to the page rowids generates snippets for
|
|
496
|
+
at most one page of hits instead of every corpus match."""
|
|
497
|
+
if not ids:
|
|
498
|
+
return {}
|
|
499
|
+
ph = ",".join("?" for _ in ids)
|
|
500
|
+
rows = conn.execute(
|
|
501
|
+
"SELECT cm.id, snippet(conversation_fts, 0, '[', ']', ' … ', 12) "
|
|
502
|
+
"FROM conversation_fts "
|
|
503
|
+
"JOIN conversation_messages cm ON cm.id = conversation_fts.rowid "
|
|
504
|
+
f"WHERE conversation_fts MATCH ? AND cm.id IN ({ph})",
|
|
505
|
+
(fts_q, *ids),
|
|
506
|
+
).fetchall()
|
|
507
|
+
return {r[0]: r[1] for r in rows}
|
|
508
|
+
|
|
509
|
+
|
|
510
|
+
def _texts_for_ids(conn, ids):
|
|
511
|
+
"""{rowid: text} for the page rowids ONLY (#149) — the LIKE page query omits
|
|
512
|
+
`text` so we never pull every matched row's body into Python; this fetches
|
|
513
|
+
it for just the page so `_manual_snippet` runs at most `limit` times."""
|
|
514
|
+
if not ids:
|
|
515
|
+
return {}
|
|
516
|
+
ph = ",".join("?" for _ in ids)
|
|
517
|
+
rows = conn.execute(
|
|
518
|
+
f"SELECT id, text FROM conversation_messages WHERE id IN ({ph})",
|
|
519
|
+
tuple(ids),
|
|
520
|
+
).fetchall()
|
|
521
|
+
return {r[0]: r[1] for r in rows}
|
|
522
|
+
|
|
523
|
+
|
|
524
|
+
def _search_fts(conn, q, limit, offset):
|
|
525
|
+
# All of dedup + paging + total live in SQL (#149) so Python never holds
|
|
526
|
+
# more than one page of hits/snippets, regardless of corpus match count.
|
|
527
|
+
fts_q = _fts_query(q)
|
|
528
|
+
# Exact post-dedup logical total — counted in C with no snippet generation
|
|
529
|
+
# and no Python row materialization.
|
|
530
|
+
total = conn.execute(
|
|
531
|
+
"SELECT COUNT(*) FROM ("
|
|
532
|
+
" SELECT DISTINCT cm.session_id, cm.uuid "
|
|
533
|
+
" FROM conversation_fts "
|
|
534
|
+
" JOIN conversation_messages cm ON cm.id = conversation_fts.rowid "
|
|
535
|
+
" WHERE conversation_fts MATCH ?)",
|
|
536
|
+
(fts_q,),
|
|
537
|
+
).fetchone()[0]
|
|
538
|
+
# One row per logical (session_id, uuid): ROW_NUMBER()=1 keeps the SAME row
|
|
539
|
+
# the old Python dedup kept as its FIRST occurrence (order: bm25, ts DESC,
|
|
540
|
+
# id DESC — cm.id is the final deterministic tiebreaker), so the surviving
|
|
541
|
+
# snippet/cost and the page boundary stay byte-stable. bm25 still ranks
|
|
542
|
+
# across all matches (inherent to relevance ordering).
|
|
543
|
+
#
|
|
544
|
+
# bm25 is materialized as a plain `rank` column in the inner `matched` CTE
|
|
545
|
+
# before the window function runs: FTS5 auxiliary functions (bm25/snippet)
|
|
546
|
+
# may only be used directly against the MATCH query, NOT inside a window
|
|
547
|
+
# ORDER BY ("unable to use function bm25 in the requested context").
|
|
548
|
+
page = conn.execute(
|
|
549
|
+
"WITH matched AS ("
|
|
550
|
+
" SELECT cm.id AS rid, cm.session_id AS sid, cm.uuid AS uuid, "
|
|
551
|
+
" cm.timestamp_utc AS ts, cm.cwd AS cwd, "
|
|
552
|
+
" cm.msg_id AS mid, cm.req_id AS rqd, "
|
|
553
|
+
" bm25(conversation_fts) AS rank "
|
|
554
|
+
" FROM conversation_fts "
|
|
555
|
+
" JOIN conversation_messages cm ON cm.id = conversation_fts.rowid "
|
|
556
|
+
" WHERE conversation_fts MATCH ?), "
|
|
557
|
+
"ranked AS ("
|
|
558
|
+
" SELECT *, ROW_NUMBER() OVER ("
|
|
559
|
+
" PARTITION BY sid, uuid ORDER BY rank, ts DESC, rid DESC"
|
|
560
|
+
" ) AS rn "
|
|
561
|
+
" FROM matched) "
|
|
562
|
+
"SELECT rid, sid, uuid, ts, cwd, mid, rqd FROM ranked WHERE rn = 1 "
|
|
563
|
+
"ORDER BY rank, ts DESC, rid DESC LIMIT ? OFFSET ?",
|
|
564
|
+
(fts_q, limit, offset),
|
|
565
|
+
).fetchall()
|
|
566
|
+
snips = _fts_snippets(conn, fts_q, [r[0] for r in page])
|
|
567
|
+
hits = [_row_to_hit(uuid, sid, ts, cwd, snips.get(rid, ""), mid, rqd)
|
|
568
|
+
for (rid, sid, uuid, ts, cwd, mid, rqd) in page]
|
|
569
|
+
return {"query": q, "mode": "fts", "hits": _attach_costs(conn, hits),
|
|
570
|
+
"total": total}
|
|
571
|
+
|
|
572
|
+
|
|
573
|
+
def _search_like(conn, q, limit, offset):
|
|
574
|
+
# SQL-bounded mirror of _search_fts for the no-FTS5 fallback (#149); the
|
|
575
|
+
# COUNT + page each scan the table once (the degraded path already lacks an
|
|
576
|
+
# index for the substring match).
|
|
577
|
+
like = _like_pattern(q)
|
|
578
|
+
total = conn.execute(
|
|
579
|
+
"SELECT COUNT(*) FROM ("
|
|
580
|
+
" SELECT DISTINCT session_id, uuid FROM conversation_messages "
|
|
581
|
+
" WHERE text LIKE ? ESCAPE '\\' AND text != '')",
|
|
582
|
+
(like,),
|
|
583
|
+
).fetchone()[0]
|
|
584
|
+
page = conn.execute(
|
|
585
|
+
"WITH ranked AS ("
|
|
586
|
+
" SELECT id AS rid, session_id AS sid, uuid AS uuid, "
|
|
587
|
+
" timestamp_utc AS ts, cwd AS cwd, msg_id AS mid, req_id AS rqd, "
|
|
588
|
+
" ROW_NUMBER() OVER ("
|
|
589
|
+
" PARTITION BY session_id, uuid "
|
|
590
|
+
" ORDER BY timestamp_utc DESC, id DESC"
|
|
591
|
+
" ) AS rn "
|
|
592
|
+
" FROM conversation_messages "
|
|
593
|
+
" WHERE text LIKE ? ESCAPE '\\' AND text != '') "
|
|
594
|
+
"SELECT rid, sid, uuid, ts, cwd, mid, rqd FROM ranked WHERE rn = 1 "
|
|
595
|
+
"ORDER BY ts DESC, rid DESC LIMIT ? OFFSET ?",
|
|
596
|
+
(like, limit, offset),
|
|
597
|
+
).fetchall()
|
|
598
|
+
texts = _texts_for_ids(conn, [r[0] for r in page])
|
|
599
|
+
hits = [_row_to_hit(uuid, sid, ts, cwd,
|
|
600
|
+
_manual_snippet(texts.get(rid, ""), q), mid, rqd)
|
|
601
|
+
for (rid, sid, uuid, ts, cwd, mid, rqd) in page]
|
|
602
|
+
return {"query": q, "mode": "like", "hits": _attach_costs(conn, hits),
|
|
603
|
+
"total": total}
|
|
604
|
+
|
|
605
|
+
|
|
606
|
+
def _fts_query(q):
|
|
607
|
+
"""Quote each whitespace term as an FTS5 string literal so punctuation /
|
|
608
|
+
operators in user input can't error the MATCH or inject FTS syntax."""
|
|
609
|
+
terms = [t for t in q.split() if t]
|
|
610
|
+
return " ".join('"' + t.replace('"', '""') + '"' for t in terms) or '""'
|
|
611
|
+
|
|
612
|
+
|
|
613
|
+
def _manual_snippet(text, q, width=80):
|
|
614
|
+
lo = text.lower().find(q.lower())
|
|
615
|
+
if lo < 0:
|
|
616
|
+
return text[:width]
|
|
617
|
+
start = max(0, lo - width // 2)
|
|
618
|
+
end = min(len(text), lo + len(q) + width // 2)
|
|
619
|
+
s = text[start:end]
|
|
620
|
+
return ("… " if start else "") + s + (" …" if end < len(text) else "")
|