cctally 1.37.1 → 1.38.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +13 -0
- package/bin/_cctally_cache.py +101 -10
- package/bin/_cctally_dashboard.py +66 -3
- package/bin/_lib_conversation.py +30 -0
- package/bin/_lib_conversation_query.py +190 -0
- package/dashboard/static/assets/{index-a_BOAOJ_.css → index-BEpz2fLq.css} +1 -1
- package/dashboard/static/assets/index-CTfLoXHj.js +66 -0
- package/dashboard/static/dashboard.html +2 -2
- package/package.json +1 -1
- package/dashboard/static/assets/index-BTGcbDcB.js +0 -59
package/CHANGELOG.md
CHANGED
|
@@ -5,6 +5,19 @@ based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
|
|
|
5
5
|
|
|
6
6
|
## [Unreleased]
|
|
7
7
|
|
|
8
|
+
## [1.38.0] - 2026-06-12
|
|
9
|
+
|
|
10
|
+
### Added
|
|
11
|
+
- Conversation viewer: Edit, MultiEdit, and Write now render as a unified red/green word-diff card — a `+N −M` stat header with the file's basename and dir, intra-line word-level highlighting on the changed text, per-language syntax coloring on the unchanged context lines, one hunk per edit for MultiEdit, and an all-added "wrote N lines" view for Write — with a collapsed `result · cat -n` sub-panel beneath carrying the real file line numbers, instead of a raw JSON tool chip (#177 S3).
|
|
12
|
+
- Conversation viewer: Bash now renders as a terminal — a `$ <command>` prompt over the output, with `stderr` split into a red block, an `error`/`interrupted` status badge, and ANSI foreground colors honored — instead of an undifferentiated output blob; legacy transcripts captured before the stream split degrade to a merged terminal view (#177 S3).
|
|
13
|
+
- Conversation viewer: a truncated tool input or result can now be expanded on demand — a "load full input"/"load full output" affordance re-reads the original session JSONL behind the existing transcript privacy gate to recompute the diff or re-render the terminal, without enlarging the on-disk cache; a rotated or deleted source surfaces a clear "source no longer available" note (#178).
|
|
14
|
+
|
|
15
|
+
## [1.37.2] - 2026-06-12
|
|
16
|
+
|
|
17
|
+
### Fixed
|
|
18
|
+
- Dashboard: the web dashboard no longer hangs on startup when you have a large conversation history. It now binds its HTTP port and serves the six cost/usage panels immediately, building the first heavy cache sync on a background thread and pushing the enriched conversation data over SSE once it completes — instead of blocking the bind behind a multi-minute (and, if interrupted, indefinite) first sync. `--no-sync` startup is byte-identical (#179).
|
|
19
|
+
- Dashboard conversation reader: the one-time conversation-enrichment reingest (the #177 re-derivation that runs once over existing transcripts) is now resumable. It walks transcript files under a sorted-path cursor, re-enriching one file per atomic transaction, so a Ctrl-C or crash mid-reingest resumes where it left off on the next sync instead of restarting the full clear-and-backfill from scratch — which previously could hold the cache writer lock and hang. `cache-sync --rebuild` remains the clean one-shot recovery path (#179).
|
|
20
|
+
|
|
8
21
|
## [1.37.1] - 2026-06-11
|
|
9
22
|
|
|
10
23
|
### Fixed
|
package/bin/_cctally_cache.py
CHANGED
|
@@ -268,6 +268,9 @@ _CACHE_MIGRATIONS = _cctally_db_sib._CACHE_MIGRATIONS
|
|
|
268
268
|
# drop/recreate dance so the per-row delete trigger never fires O(rows) under
|
|
269
269
|
# the held lock on a rebuild / truncation escalation.
|
|
270
270
|
clear_conversation_messages = _cctally_db_sib.clear_conversation_messages
|
|
271
|
+
# cache_meta key/value upsert helper — reused by the resumable reingest cursor
|
|
272
|
+
# writes (#179) so the ON CONFLICT idiom lives in one place. Caller commits.
|
|
273
|
+
_set_cache_meta = _cctally_db_sib._set_cache_meta
|
|
271
274
|
|
|
272
275
|
|
|
273
276
|
# === BEGIN MOVED REGIONS ===
|
|
@@ -640,7 +643,9 @@ def sync_cache(
|
|
|
640
643
|
"DELETE FROM cache_meta WHERE key IN "
|
|
641
644
|
"('conversation_reingest_pending',"
|
|
642
645
|
" 'conversation_source_tool_use_reingest_pending',"
|
|
643
|
-
" 'conversation_reingest_enrichment_pending'
|
|
646
|
+
" 'conversation_reingest_enrichment_pending',"
|
|
647
|
+
" 'conversation_reingest_cursor',"
|
|
648
|
+
" 'conversation_reingest_cursor_gen')")
|
|
644
649
|
conn.commit()
|
|
645
650
|
eprint("[cache-sync] rebuild: cleared Claude cached entries")
|
|
646
651
|
|
|
@@ -720,15 +725,12 @@ def sync_cache(
|
|
|
720
725
|
except sqlite3.OperationalError:
|
|
721
726
|
_reingest = False
|
|
722
727
|
if _reingest:
|
|
723
|
-
clear_conversation_messages
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
" 'conversation_reingest_enrichment_pending')"
|
|
730
|
-
)
|
|
731
|
-
conn.commit()
|
|
728
|
+
# #179: resumable per-file reingest (was a global clear_conversation_messages
|
|
729
|
+
# + offset-0 backfill that re-armed the entire ~2.5min rebuild on any
|
|
730
|
+
# interrupt). The helper checkpoints a sorted-path cursor and clears the
|
|
731
|
+
# three flags + cursor + gen atomically on completion. Never on the rebuild
|
|
732
|
+
# path (which already wipes + repopulates id-aware via the normal walk).
|
|
733
|
+
_resumable_reingest_conversation_messages(conn)
|
|
732
734
|
|
|
733
735
|
paths: list[pathlib.Path] = list(_iter_claude_jsonl_files())
|
|
734
736
|
stats.files_total = len(paths)
|
|
@@ -1157,6 +1159,95 @@ def backfill_conversation_messages(conn: sqlite3.Connection) -> int:
|
|
|
1157
1159
|
return inserted
|
|
1158
1160
|
|
|
1159
1161
|
|
|
1162
|
+
_REINGEST_FLAG_KEYS = (
|
|
1163
|
+
"conversation_reingest_pending",
|
|
1164
|
+
"conversation_source_tool_use_reingest_pending",
|
|
1165
|
+
"conversation_reingest_enrichment_pending",
|
|
1166
|
+
)
|
|
1167
|
+
|
|
1168
|
+
|
|
1169
|
+
def _reingest_parse_file(jp, path_str):
|
|
1170
|
+
"""Parse one Claude JSONL into enriched ``conversation_messages`` row tuples
|
|
1171
|
+
(``_CONV_INSERT_SQL`` column order). Mirrors ``backfill_conversation_messages``'s
|
|
1172
|
+
inner read+flatten, factored to a module-level seam so the resumable reingest
|
|
1173
|
+
builds rows BEFORE any write (a parse failure does no DML) and tests can inject.
|
|
1174
|
+
Raises ``OSError`` if the file can't be opened/read."""
|
|
1175
|
+
rows = []
|
|
1176
|
+
with open(jp, "r", encoding="utf-8", errors="replace") as fh:
|
|
1177
|
+
for m in _iter_message_rows(fh, path_str):
|
|
1178
|
+
rows.append(_conv_row_tuple(m, path_str))
|
|
1179
|
+
return rows
|
|
1180
|
+
|
|
1181
|
+
|
|
1182
|
+
def _resumable_reingest_conversation_messages(conn):
|
|
1183
|
+
"""#179: resumable, lock-friendly replacement for the old global
|
|
1184
|
+
``clear_conversation_messages`` + offset-0 ``backfill_conversation_messages``
|
|
1185
|
+
reingest, which re-armed the whole ~2.5min rebuild on any interrupt. Walks
|
|
1186
|
+
every Claude JSONL in deterministic sorted-path order, re-enriching one file
|
|
1187
|
+
per atomic transaction and checkpointing ``conversation_reingest_cursor`` so an
|
|
1188
|
+
interrupt resumes instead of restarting. A ``conversation_reingest_cursor_gen``
|
|
1189
|
+
fingerprint (the set of pending reingest flags) resets the cursor whenever the
|
|
1190
|
+
pending-flag set changes, so a newly-armed flag forces a fresh pass. The caller
|
|
1191
|
+
(``sync_cache``) already holds the cache.db flock; per-file commits bound only
|
|
1192
|
+
the SQLite write transaction, not the flock. Clears the three flags + cursor +
|
|
1193
|
+
gen atomically on completion."""
|
|
1194
|
+
# 1. Generation guard: reset the cursor if the live pending-flag set differs.
|
|
1195
|
+
set_flags = [k for k in _REINGEST_FLAG_KEYS
|
|
1196
|
+
if conn.execute("SELECT 1 FROM cache_meta WHERE key=?", (k,)).fetchone()]
|
|
1197
|
+
gen = ",".join(sorted(set_flags))
|
|
1198
|
+
gen_row = conn.execute(
|
|
1199
|
+
"SELECT value FROM cache_meta WHERE key='conversation_reingest_cursor_gen'"
|
|
1200
|
+
).fetchone()
|
|
1201
|
+
if (gen_row[0] if gen_row else None) != gen:
|
|
1202
|
+
_set_cache_meta(conn, "conversation_reingest_cursor_gen", gen)
|
|
1203
|
+
conn.execute("DELETE FROM cache_meta WHERE key='conversation_reingest_cursor'")
|
|
1204
|
+
conn.commit()
|
|
1205
|
+
cursor = ""
|
|
1206
|
+
else:
|
|
1207
|
+
crow = conn.execute(
|
|
1208
|
+
"SELECT value FROM cache_meta WHERE key='conversation_reingest_cursor'"
|
|
1209
|
+
).fetchone()
|
|
1210
|
+
cursor = crow[0] if crow and crow[0] is not None else ""
|
|
1211
|
+
|
|
1212
|
+
# 2. Per-file resumable walk in deterministic sorted-path order.
|
|
1213
|
+
for jp in sorted(_iter_claude_jsonl_files(), key=str):
|
|
1214
|
+
path_str = str(jp)
|
|
1215
|
+
if path_str <= cursor:
|
|
1216
|
+
continue
|
|
1217
|
+
try:
|
|
1218
|
+
rows = _reingest_parse_file(jp, path_str) # parse FIRST — no DML on failure
|
|
1219
|
+
except OSError as exc:
|
|
1220
|
+
# Read/parse failed BEFORE any conversation_messages DML — the file's
|
|
1221
|
+
# existing rows are untouched (preserved, not dropped). Only advance the
|
|
1222
|
+
# cursor; this cursor-only write needs no rollback envelope (no message
|
|
1223
|
+
# DML to undo, and an interrupt mid-commit just re-runs this file).
|
|
1224
|
+
eprint(f"[conversation-reingest] could not read {jp}: {exc}; "
|
|
1225
|
+
"preserving existing rows")
|
|
1226
|
+
_set_cache_meta(conn, "conversation_reingest_cursor", path_str)
|
|
1227
|
+
conn.commit()
|
|
1228
|
+
continue
|
|
1229
|
+
try:
|
|
1230
|
+
conn.execute("DELETE FROM conversation_messages WHERE source_path=?",
|
|
1231
|
+
(path_str,))
|
|
1232
|
+
if rows:
|
|
1233
|
+
conn.executemany(_CONV_INSERT_SQL, rows)
|
|
1234
|
+
_set_cache_meta(conn, "conversation_reingest_cursor", path_str)
|
|
1235
|
+
conn.commit()
|
|
1236
|
+
except BaseException:
|
|
1237
|
+
conn.rollback()
|
|
1238
|
+
raise
|
|
1239
|
+
|
|
1240
|
+
# 3. Completion: clear flags + cursor + gen atomically.
|
|
1241
|
+
conn.execute(
|
|
1242
|
+
"DELETE FROM cache_meta WHERE key IN "
|
|
1243
|
+
"('conversation_reingest_pending',"
|
|
1244
|
+
" 'conversation_source_tool_use_reingest_pending',"
|
|
1245
|
+
" 'conversation_reingest_enrichment_pending',"
|
|
1246
|
+
" 'conversation_reingest_cursor',"
|
|
1247
|
+
" 'conversation_reingest_cursor_gen')")
|
|
1248
|
+
conn.commit()
|
|
1249
|
+
|
|
1250
|
+
|
|
1160
1251
|
def iter_entries(
|
|
1161
1252
|
conn: sqlite3.Connection,
|
|
1162
1253
|
range_start: dt.datetime,
|
|
@@ -5341,6 +5341,10 @@ class DashboardHTTPHandler(BaseHTTPRequestHandler):
|
|
|
5341
5341
|
self._handle_get_conversations()
|
|
5342
5342
|
elif path == "/api/conversation/search":
|
|
5343
5343
|
self._handle_get_conversation_search()
|
|
5344
|
+
elif path.startswith("/api/conversation/") and path.endswith("/payload"):
|
|
5345
|
+
# #178: on-demand load-full. Matched BEFORE the <id> reader
|
|
5346
|
+
# catch-all (same precedence as /api/conversation/search).
|
|
5347
|
+
self._handle_get_conversation_payload(path)
|
|
5344
5348
|
elif path.startswith("/api/conversation/"):
|
|
5345
5349
|
self._handle_get_conversation_detail(path)
|
|
5346
5350
|
else:
|
|
@@ -7344,6 +7348,46 @@ class DashboardHTTPHandler(BaseHTTPRequestHandler):
|
|
|
7344
7348
|
return
|
|
7345
7349
|
self._respond_json(200, body)
|
|
7346
7350
|
|
|
7351
|
+
def _handle_get_conversation_payload(self, path: str) -> None:
|
|
7352
|
+
"""``GET /api/conversation/<sid>/payload?tool_use_id=<id>&which=<result|input>``
|
|
7353
|
+
— the #178 on-demand load-full route. Re-reads the source JSONL line so
|
|
7354
|
+
a clipped result/input can be expanded without enlarging the cache.
|
|
7355
|
+
|
|
7356
|
+
Gated FIRST by the same loopback/Host transcript privacy predicate the
|
|
7357
|
+
three other conversation routes use (fail-closed 403). ``locate_tool_payload``
|
|
7358
|
+
runs against cache.db (via the shared 500-envelope scaffold); the actual
|
|
7359
|
+
full body is re-read from disk by ``read_full_payload`` (no cache conn).
|
|
7360
|
+
``which`` is validated to ``result``/``input`` (else 400); a missing
|
|
7361
|
+
tool_use_id is 400; an unknown id is 404; a gone/unparseable source line
|
|
7362
|
+
is 410 (the documented consequence of storing only capped text).
|
|
7363
|
+
"""
|
|
7364
|
+
if not self._require_transcripts_allowed():
|
|
7365
|
+
return
|
|
7366
|
+
import urllib.parse as _u
|
|
7367
|
+
session_id = _u.unquote(
|
|
7368
|
+
path[len("/api/conversation/"):-len("/payload")])
|
|
7369
|
+
q = _u.parse_qs(self.path.partition("?")[2])
|
|
7370
|
+
tool_use_id = _qs_str(q, "tool_use_id", "")
|
|
7371
|
+
which = _qs_str(q, "which", "result")
|
|
7372
|
+
if not session_id or which not in ("result", "input") or not tool_use_id:
|
|
7373
|
+
self._respond_json(400, {"error": "bad request"})
|
|
7374
|
+
return
|
|
7375
|
+
cq = self._conversation_query()
|
|
7376
|
+
ok, loc = self._run_conversation_query(
|
|
7377
|
+
lambda conn: cq.locate_tool_payload(
|
|
7378
|
+
conn, session_id, tool_use_id, which),
|
|
7379
|
+
"/api/conversation/payload")
|
|
7380
|
+
if not ok:
|
|
7381
|
+
return
|
|
7382
|
+
if loc is None:
|
|
7383
|
+
self._respond_json(404, {"error": "not found"})
|
|
7384
|
+
return
|
|
7385
|
+
payload = cq.read_full_payload(loc[0], loc[1], tool_use_id, which)
|
|
7386
|
+
if payload is None:
|
|
7387
|
+
self._respond_json(410, {"error": "source no longer available"})
|
|
7388
|
+
return
|
|
7389
|
+
self._respond_json(200, payload)
|
|
7390
|
+
|
|
7347
7391
|
def _handle_get_project_detail(self) -> None:
|
|
7348
7392
|
"""Return ProjectDetail JSON for ``GET /api/project/<key>?weeks=N``
|
|
7349
7393
|
(spec §5.3 / §6.5).
|
|
@@ -7829,6 +7873,21 @@ def _dashboard_wait_for_signal(
|
|
|
7829
7873
|
os.close(write_fd)
|
|
7830
7874
|
|
|
7831
7875
|
|
|
7876
|
+
def _dashboard_initial_snapshot(args, *, pinned_now, display_tz_pref_override):
|
|
7877
|
+
"""#179: build the dashboard's first snapshot WITHOUT the heavy sync so the
|
|
7878
|
+
HTTP port binds promptly. The background ``_DashboardSyncThread`` (started in
|
|
7879
|
+
cmd_dashboard before the ThreadingHTTPServer bind) runs the first full
|
|
7880
|
+
``sync_cache`` — including any pending conversation reingest — and SSE-pushes
|
|
7881
|
+
the populated snapshot on completion. ``--no-sync`` already passed
|
|
7882
|
+
``skip_sync=True`` here, so that path is byte-identical; only the normal launch
|
|
7883
|
+
changes (its previously-redundant foreground sync is removed, not duplicated —
|
|
7884
|
+
the background thread's first tick was always going to sync anyway)."""
|
|
7885
|
+
return sys.modules["cctally"]._tui_build_snapshot(
|
|
7886
|
+
now_utc=pinned_now, skip_sync=True,
|
|
7887
|
+
display_tz_pref_override=display_tz_pref_override,
|
|
7888
|
+
)
|
|
7889
|
+
|
|
7890
|
+
|
|
7832
7891
|
def cmd_dashboard(args: argparse.Namespace) -> int:
|
|
7833
7892
|
"""Launch the live web dashboard."""
|
|
7834
7893
|
import signal as _signal
|
|
@@ -7906,9 +7965,13 @@ def cmd_dashboard(args: argparse.Namespace) -> int:
|
|
|
7906
7965
|
# weekly, project, forecast subcommands already respect.
|
|
7907
7966
|
pinned_now = _command_as_of() if os.environ.get("CCTALLY_AS_OF") else None
|
|
7908
7967
|
|
|
7909
|
-
#
|
|
7910
|
-
|
|
7911
|
-
|
|
7968
|
+
# #179: build the initial snapshot WITHOUT the heavy sync so the HTTP port
|
|
7969
|
+
# binds promptly even on a heavy-history instance. The background
|
|
7970
|
+
# _DashboardSyncThread (started below, before the bind) owns the first full
|
|
7971
|
+
# sync_cache — including any pending conversation-enrichment reingest — and
|
|
7972
|
+
# SSE-pushes the populated snapshot on completion.
|
|
7973
|
+
initial = _dashboard_initial_snapshot(
|
|
7974
|
+
args, pinned_now=pinned_now,
|
|
7912
7975
|
display_tz_pref_override=display_tz_pref_override,
|
|
7913
7976
|
)
|
|
7914
7977
|
if args.no_sync:
|
package/bin/_lib_conversation.py
CHANGED
|
@@ -106,6 +106,7 @@ def _normalize(obj, t, offset):
|
|
|
106
106
|
entry_type = TOOL_RESULT
|
|
107
107
|
_attach_subagent_result(blocks, obj) # #166: record-level toolUseResult
|
|
108
108
|
_attach_ask_answers(blocks, obj) # #177 S2: AskUserQuestion answers
|
|
109
|
+
_attach_bash_streams(blocks, obj) # #177 S3: Bash stderr/interrupted
|
|
109
110
|
_attach_task_meta(blocks, obj) # task checklist identity
|
|
110
111
|
# tool_result rows are stored but NOT indexed as prose (spec §2). A
|
|
111
112
|
# user line that mixes a text block with a tool_result block must not
|
|
@@ -289,6 +290,35 @@ def _attach_ask_answers(blocks, obj):
|
|
|
289
290
|
results[0]["ask_annotations"] = bounded_anno
|
|
290
291
|
|
|
291
292
|
|
|
293
|
+
def _attach_bash_streams(blocks, obj):
|
|
294
|
+
"""Stash a Bash tool_result's structured stderr + interrupted onto its single
|
|
295
|
+
tool_result block (#177 S3). Self-identifying: fires only when toolUseResult
|
|
296
|
+
is a dict carrying a ``stdout``/``stderr`` key — a shape distinctive to Bash —
|
|
297
|
+
so no cross-record tool-name lookup is needed (same posture as
|
|
298
|
+
_attach_ask_answers' ``answers`` gate). We do NOT store stdout: the existing
|
|
299
|
+
``result.text`` already equals stdout+stderr (the merged Bash output), so
|
|
300
|
+
storing stdout would roughly double the at-rest payload; the stdout/stderr
|
|
301
|
+
split is derived client-side by stripping the stderr suffix.
|
|
302
|
+
|
|
303
|
+
Parser-private keys ``bash_stderr`` / ``bash_interrupted`` are popped in the
|
|
304
|
+
query layer's Phase 1 so they never leak into emitted/orphan blocks. stderr is
|
|
305
|
+
bounded with the same cap as result.text (_TOOL_RESULT_CAP). Empty stderr +
|
|
306
|
+
not-interrupted is a no-op (the common case — stderr is empty in ~99% of
|
|
307
|
+
results), keeping the additive contract: absent on every non-Bash result and
|
|
308
|
+
on old rows. Same exactly-one-result-block guard as _attach_subagent_result."""
|
|
309
|
+
tur = obj.get("toolUseResult")
|
|
310
|
+
if not isinstance(tur, dict) or ("stdout" not in tur and "stderr" not in tur):
|
|
311
|
+
return
|
|
312
|
+
results = [b for b in blocks if b.get("kind") == "tool_result"]
|
|
313
|
+
if len(results) != 1:
|
|
314
|
+
return
|
|
315
|
+
stderr = tur.get("stderr")
|
|
316
|
+
if isinstance(stderr, str) and stderr:
|
|
317
|
+
results[0]["bash_stderr"] = stderr[:_TOOL_RESULT_CAP]
|
|
318
|
+
if bool(tur.get("interrupted")):
|
|
319
|
+
results[0]["bash_interrupted"] = True
|
|
320
|
+
|
|
321
|
+
|
|
292
322
|
# Subagent Task tools record toolUseResult=null and put the identity in the
|
|
293
323
|
# human-readable result text instead; the id is the only thing the fold needs
|
|
294
324
|
# from the result (subject/status come from the call input). Anchored to the
|
|
@@ -20,6 +20,10 @@ import sqlite3
|
|
|
20
20
|
# mirror — imported by the dashboard's conversation endpoints at runtime.
|
|
21
21
|
|
|
22
22
|
from _lib_pricing import _calculate_entry_cost
|
|
23
|
+
# #178: the on-demand load-full re-read helper re-stringifies a raw tool_result
|
|
24
|
+
# content block the same way the parser does at ingest — reuse the parser's
|
|
25
|
+
# _stringify so the full (un-capped) result text matches the cached/capped one.
|
|
26
|
+
from _lib_conversation import _stringify
|
|
23
27
|
|
|
24
28
|
|
|
25
29
|
# Mirror of dashboard/web/src/conversations/systemMarkers.ts::MARKER_RE — anchored
|
|
@@ -484,6 +488,7 @@ def get_conversation(conn, session_id, *, after=None, limit=500):
|
|
|
484
488
|
spawn_kind = {} # tool_use id -> subagent_type
|
|
485
489
|
agent_link = {} # tool_use id -> (agent_id, raw_meta)
|
|
486
490
|
ask_link = {} # tool_use id -> (answers, annotations) (#177 S2)
|
|
491
|
+
bash_link = {} # tool_use id -> (stderr, interrupted) (#177 S3)
|
|
487
492
|
task_link = {} # tool_use id -> {"task_id", "task_list"} (Task* checklist)
|
|
488
493
|
for it in items:
|
|
489
494
|
for b in it["blocks"]:
|
|
@@ -501,6 +506,10 @@ def get_conversation(conn, session_id, *, after=None, limit=500):
|
|
|
501
506
|
anno = b.pop("ask_annotations", None)
|
|
502
507
|
if ans is not None and b.get("tool_use_id") is not None:
|
|
503
508
|
ask_link[b["tool_use_id"]] = (ans, anno)
|
|
509
|
+
bstderr = b.pop("bash_stderr", None) # #177 S3
|
|
510
|
+
bintr = b.pop("bash_interrupted", None)
|
|
511
|
+
if b.get("tool_use_id") is not None and (bstderr is not None or bintr):
|
|
512
|
+
bash_link[b["tool_use_id"]] = (bstderr, bool(bintr))
|
|
504
513
|
tid_ = b.pop("task_id", None) # Task* checklist
|
|
505
514
|
tlist_ = b.pop("task_list", None)
|
|
506
515
|
if b.get("tool_use_id") is not None and (tid_ is not None or tlist_ is not None):
|
|
@@ -568,6 +577,12 @@ def get_conversation(conn, session_id, *, after=None, limit=500):
|
|
|
568
577
|
b["answers"] = link[0]
|
|
569
578
|
if link[1]:
|
|
570
579
|
b["annotations"] = link[1]
|
|
580
|
+
blink = bash_link.get(b["tool_use_id"]) # #177 S3
|
|
581
|
+
if blink is not None:
|
|
582
|
+
if blink[0] is not None:
|
|
583
|
+
b["stderr"] = blink[0]
|
|
584
|
+
if blink[1]:
|
|
585
|
+
b["interrupted"] = True
|
|
571
586
|
|
|
572
587
|
# ---- Phase 3b: fold the Task* op stream into per-run checklist snapshots ----
|
|
573
588
|
_fold_task_runs(items, task_link)
|
|
@@ -1127,3 +1142,178 @@ def _manual_snippet(text, q, width=80):
|
|
|
1127
1142
|
end = min(len(text), lo + len(q) + width // 2)
|
|
1128
1143
|
s = text[start:end]
|
|
1129
1144
|
return ("… " if start else "") + s + (" …" if end < len(text) else "")
|
|
1145
|
+
|
|
1146
|
+
|
|
1147
|
+
# ---------------------------------------------------------------------------
|
|
1148
|
+
# #178: on-demand "load full result/input" kernels. Back the dashboard's
|
|
1149
|
+
# /api/conversation/<sid>/payload route. The cache stores only CAPPED tool text
|
|
1150
|
+
# (result clipped to _TOOL_RESULT_CAP, input leaves clipped to _INPUT_LEAF_CAP),
|
|
1151
|
+
# so the full body is re-derived from the source JSONL line here — the cache at
|
|
1152
|
+
# rest never grows. Pure (sqlite3.Connection + filesystem); no clock/network.
|
|
1153
|
+
# ---------------------------------------------------------------------------
|
|
1154
|
+
_FULL_PAYLOAD_CEILING = 1_000_000 # serve up to ~1 MB; protects the HTTP server / browser
|
|
1155
|
+
|
|
1156
|
+
|
|
1157
|
+
def locate_tool_payload(conn, session_id, tool_use_id, which):
|
|
1158
|
+
"""``(source_path, byte_offset)`` for the JSONL line holding the tool_use
|
|
1159
|
+
(``which='input'``) or tool_result (``which='result'``) carrying this
|
|
1160
|
+
``tool_use_id`` in this session, else ``None``.
|
|
1161
|
+
|
|
1162
|
+
The prefilter uses ``instr(blocks_json, ?) > 0`` — NOT ``LIKE`` (Codex P1.4):
|
|
1163
|
+
tool_use_ids contain ``_`` (e.g. ``toolu_01SEQ…``), which ``LIKE`` treats as a
|
|
1164
|
+
single-char wildcard, so a near-miss id would false-match. ``instr`` is a
|
|
1165
|
+
literal substring test. It is also NOT a ``≤2 rows`` situation — the DB holds
|
|
1166
|
+
duplicate physical rows for one logical message (``get_conversation`` dedups
|
|
1167
|
+
by uuid) — so every candidate is parsed and EXACT-matched on the block id
|
|
1168
|
+
(``tool_use.id`` for input / ``tool_result.tool_use_id`` for result), under the
|
|
1169
|
+
same deterministic ``ORDER BY timestamp_utc, id`` as ``get_conversation``. The
|
|
1170
|
+
SELECT runs here (not via ``get_conversation``) because that reader omits
|
|
1171
|
+
``byte_offset`` (Codex P2.5)."""
|
|
1172
|
+
rows = conn.execute(
|
|
1173
|
+
"SELECT source_path, byte_offset, blocks_json FROM conversation_messages "
|
|
1174
|
+
"WHERE session_id=? AND instr(blocks_json, ?) > 0 "
|
|
1175
|
+
"ORDER BY timestamp_utc, id", (session_id, tool_use_id)).fetchall()
|
|
1176
|
+
for source_path, byte_offset, blocks_json in rows:
|
|
1177
|
+
try:
|
|
1178
|
+
blocks = _json.loads(blocks_json)
|
|
1179
|
+
except (ValueError, TypeError):
|
|
1180
|
+
continue
|
|
1181
|
+
if not isinstance(blocks, list):
|
|
1182
|
+
continue
|
|
1183
|
+
for b in blocks:
|
|
1184
|
+
if not isinstance(b, dict):
|
|
1185
|
+
continue
|
|
1186
|
+
k = b.get("kind")
|
|
1187
|
+
if which == "input" and k == "tool_use" and b.get("id") == tool_use_id:
|
|
1188
|
+
return source_path, byte_offset
|
|
1189
|
+
if which == "result" and k == "tool_result" and b.get("tool_use_id") == tool_use_id:
|
|
1190
|
+
return source_path, byte_offset
|
|
1191
|
+
return None
|
|
1192
|
+
|
|
1193
|
+
|
|
1194
|
+
def _clip_payload_input(inp, ceiling):
|
|
1195
|
+
"""Clip a structured input so the returned dict serializes to ``ceiling`` chars
|
|
1196
|
+
or fewer, and report whether anything was clipped — the input-side analogue of
|
|
1197
|
+
the result-side ceiling. A degenerate multi-MB input (one giant leaf OR many
|
|
1198
|
+
sub-ceiling leaves that sum past the ceiling) is bounded so the HTTP server /
|
|
1199
|
+
browser is protected, while every real payload returns whole.
|
|
1200
|
+
|
|
1201
|
+
The guarantee is AGGREGATE, not merely per-leaf: a shared remaining-char budget
|
|
1202
|
+
is threaded through the walk (mirroring ``_bound_input``'s total-size backstop) —
|
|
1203
|
+
each string leaf is clipped against the running budget and the budget is
|
|
1204
|
+
decremented as we go, so once it is exhausted later leaves clip to ''.
|
|
1205
|
+
``truncated`` is True iff any leaf was clipped (or the post-walk serialized size
|
|
1206
|
+
still exceeds ``ceiling``, the structural-overhead backstop). Post-condition:
|
|
1207
|
+
``len(json.dumps(clipped, ensure_ascii=False)) <= ceiling`` always."""
|
|
1208
|
+
truncated = False
|
|
1209
|
+
remaining = [ceiling] # boxed so the nested walk can decrement it
|
|
1210
|
+
|
|
1211
|
+
def walk(v):
|
|
1212
|
+
nonlocal truncated
|
|
1213
|
+
if isinstance(v, str):
|
|
1214
|
+
if len(v) > remaining[0]:
|
|
1215
|
+
truncated = True
|
|
1216
|
+
v = v[:remaining[0]]
|
|
1217
|
+
remaining[0] -= len(v)
|
|
1218
|
+
return v
|
|
1219
|
+
if isinstance(v, dict):
|
|
1220
|
+
return {k: walk(x) for k, x in v.items()}
|
|
1221
|
+
if isinstance(v, list):
|
|
1222
|
+
return [walk(x) for x in v]
|
|
1223
|
+
return v
|
|
1224
|
+
|
|
1225
|
+
clipped = walk(inp)
|
|
1226
|
+
# Backstop: structural JSON overhead (braces/quotes/keys) can push a
|
|
1227
|
+
# budget-exact payload a few chars past the ceiling. Hard-clip the largest
|
|
1228
|
+
# remaining string leaf(s) until the whole dict serializes within the ceiling.
|
|
1229
|
+
while len(_json.dumps(clipped, ensure_ascii=False)) > ceiling:
|
|
1230
|
+
truncated = True
|
|
1231
|
+
if not _shrink_largest_leaf(clipped):
|
|
1232
|
+
break # no string leaf left to shrink (e.g. pure numeric/structural)
|
|
1233
|
+
return clipped, truncated
|
|
1234
|
+
|
|
1235
|
+
|
|
1236
|
+
def _shrink_largest_leaf(obj):
|
|
1237
|
+
"""Halve the longest string leaf reachable in ``obj`` (a dict/list/scalar),
|
|
1238
|
+
in place, and return True if one was shrunk — the post-walk backstop for the
|
|
1239
|
+
rare structural-overhead overshoot. A leaf already at length 1 is truncated to
|
|
1240
|
+
''. Returns False when no non-empty string leaf exists."""
|
|
1241
|
+
best = {"len": 0, "container": None, "key": None}
|
|
1242
|
+
|
|
1243
|
+
def scan(v, container, key):
|
|
1244
|
+
if isinstance(v, str):
|
|
1245
|
+
if len(v) > best["len"]:
|
|
1246
|
+
best.update(len=len(v), container=container, key=key)
|
|
1247
|
+
elif isinstance(v, dict):
|
|
1248
|
+
for k, x in v.items():
|
|
1249
|
+
scan(x, v, k)
|
|
1250
|
+
elif isinstance(v, list):
|
|
1251
|
+
for i, x in enumerate(v):
|
|
1252
|
+
scan(x, v, i)
|
|
1253
|
+
|
|
1254
|
+
scan(obj, None, None)
|
|
1255
|
+
if best["container"] is None or best["len"] == 0:
|
|
1256
|
+
return False
|
|
1257
|
+
s = best["container"][best["key"]]
|
|
1258
|
+
best["container"][best["key"]] = s[: len(s) // 2]
|
|
1259
|
+
return True
|
|
1260
|
+
|
|
1261
|
+
|
|
1262
|
+
def read_full_payload(source_path, byte_offset, tool_use_id, which):
|
|
1263
|
+
"""Re-read the raw JSONL line at ``(source_path, byte_offset)`` and return the
|
|
1264
|
+
FULL (un-capped) payload for ``tool_use_id``:
|
|
1265
|
+
|
|
1266
|
+
- ``which='input'`` -> ``{"which":"input", "tool_use_id", "input", "full_length",
|
|
1267
|
+
"truncated"}`` — the matching tool_use block's complete ``input`` dict, so the
|
|
1268
|
+
DiffCard can pull old/new strings straight into computeDiff.
|
|
1269
|
+
- ``which='result'`` -> ``{"which":"result", "tool_use_id", "text", "full_length",
|
|
1270
|
+
"truncated", "is_error", [stderr]}`` — the full ``_stringify(content)`` plus,
|
|
1271
|
+
for Bash, the full ``toolUseResult.stderr``.
|
|
1272
|
+
|
|
1273
|
+
``None`` when the source is gone / the line is unparseable (rotated or deleted
|
|
1274
|
+
JSONL — the documented 410 path) or the id is no longer present in that line.
|
|
1275
|
+
``full_length``/``truncated`` describe the payload against ``_FULL_PAYLOAD_CEILING``
|
|
1276
|
+
— honoring #178's "un-capped" spirit for real payloads while bounding the
|
|
1277
|
+
degenerate multi-MB case."""
|
|
1278
|
+
try:
|
|
1279
|
+
with open(source_path, "rb") as fh:
|
|
1280
|
+
fh.seek(byte_offset)
|
|
1281
|
+
line = fh.readline()
|
|
1282
|
+
obj = _json.loads(line)
|
|
1283
|
+
except (OSError, ValueError):
|
|
1284
|
+
return None
|
|
1285
|
+
if not isinstance(obj, dict):
|
|
1286
|
+
return None
|
|
1287
|
+
content = (obj.get("message") or {}).get("content")
|
|
1288
|
+
if not isinstance(content, list):
|
|
1289
|
+
return None
|
|
1290
|
+
if which == "input":
|
|
1291
|
+
for b in content:
|
|
1292
|
+
if (isinstance(b, dict) and b.get("type") == "tool_use"
|
|
1293
|
+
and b.get("id") == tool_use_id):
|
|
1294
|
+
inp = b.get("input") if isinstance(b.get("input"), dict) else {}
|
|
1295
|
+
full_length = len(_json.dumps(inp, ensure_ascii=False))
|
|
1296
|
+
clipped, truncated = _clip_payload_input(inp, _FULL_PAYLOAD_CEILING)
|
|
1297
|
+
return {"which": "input", "tool_use_id": tool_use_id,
|
|
1298
|
+
"input": clipped, "full_length": full_length,
|
|
1299
|
+
"truncated": truncated}
|
|
1300
|
+
return None
|
|
1301
|
+
for b in content:
|
|
1302
|
+
if (isinstance(b, dict) and b.get("type") == "tool_result"
|
|
1303
|
+
and b.get("tool_use_id") == tool_use_id):
|
|
1304
|
+
raw = _stringify(b.get("content"))
|
|
1305
|
+
# The bound is PER-STREAM by design: `text` and the Bash `stderr`
|
|
1306
|
+
# below are each clipped to _FULL_PAYLOAD_CEILING independently, so a
|
|
1307
|
+
# result carrying both can serialize to ~2× the ceiling. That is
|
|
1308
|
+
# intentional — they are distinct streams the DiffCard renders side by
|
|
1309
|
+
# side, and each is individually bounded against the HTTP/browser DoS.
|
|
1310
|
+
resp = {"which": "result", "tool_use_id": tool_use_id,
|
|
1311
|
+
"text": raw[:_FULL_PAYLOAD_CEILING], "full_length": len(raw),
|
|
1312
|
+
"truncated": len(raw) > _FULL_PAYLOAD_CEILING,
|
|
1313
|
+
"is_error": bool(b.get("is_error"))}
|
|
1314
|
+
tur = obj.get("toolUseResult")
|
|
1315
|
+
if (isinstance(tur, dict) and isinstance(tur.get("stderr"), str)
|
|
1316
|
+
and tur.get("stderr")):
|
|
1317
|
+
resp["stderr"] = tur["stderr"][:_FULL_PAYLOAD_CEILING] # per-stream bound (see above)
|
|
1318
|
+
return resp
|
|
1319
|
+
return None
|