cctally 1.37.1 → 1.38.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -5,6 +5,19 @@ based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
5
5
 
6
6
  ## [Unreleased]
7
7
 
8
+ ## [1.38.0] - 2026-06-12
9
+
10
+ ### Added
11
+ - Conversation viewer: Edit, MultiEdit, and Write now render as a unified red/green word-diff card — a `+N −M` stat header with the file's basename and dir, intra-line word-level highlighting on the changed text, per-language syntax coloring on the unchanged context lines, one hunk per edit for MultiEdit, and an all-added "wrote N lines" view for Write — with a collapsed `result · cat -n` sub-panel beneath carrying the real file line numbers, instead of a raw JSON tool chip (#177 S3).
12
+ - Conversation viewer: Bash now renders as a terminal — a `$ <command>` prompt over the output, with `stderr` split into a red block, an `error`/`interrupted` status badge, and ANSI foreground colors honored — instead of an undifferentiated output blob; legacy transcripts captured before the stream split degrade to a merged terminal view (#177 S3).
13
+ - Conversation viewer: a truncated tool input or result can now be expanded on demand — a "load full input"/"load full output" affordance re-reads the original session JSONL behind the existing transcript privacy gate to recompute the diff or re-render the terminal, without enlarging the on-disk cache; a rotated or deleted source surfaces a clear "source no longer available" note (#178).
14
+
15
+ ## [1.37.2] - 2026-06-12
16
+
17
+ ### Fixed
18
+ - Dashboard: the web dashboard no longer hangs on startup when you have a large conversation history. It now binds its HTTP port and serves the six cost/usage panels immediately, building the first heavy cache sync on a background thread and pushing the enriched conversation data over SSE once it completes — instead of blocking the bind behind a multi-minute (and, if interrupted, indefinite) first sync. `--no-sync` startup is byte-identical (#179).
19
+ - Dashboard conversation reader: the one-time conversation-enrichment reingest (the #177 re-derivation that runs once over existing transcripts) is now resumable. It walks transcript files under a sorted-path cursor, re-enriching one file per atomic transaction, so a Ctrl-C or crash mid-reingest resumes where it left off on the next sync instead of restarting the full clear-and-backfill from scratch — which previously could hold the cache writer lock and hang. `cache-sync --rebuild` remains the clean one-shot recovery path (#179).
20
+
8
21
  ## [1.37.1] - 2026-06-11
9
22
 
10
23
  ### Fixed
@@ -268,6 +268,9 @@ _CACHE_MIGRATIONS = _cctally_db_sib._CACHE_MIGRATIONS
268
268
  # drop/recreate dance so the per-row delete trigger never fires O(rows) under
269
269
  # the held lock on a rebuild / truncation escalation.
270
270
  clear_conversation_messages = _cctally_db_sib.clear_conversation_messages
271
+ # cache_meta key/value upsert helper — reused by the resumable reingest cursor
272
+ # writes (#179) so the ON CONFLICT idiom lives in one place. Caller commits.
273
+ _set_cache_meta = _cctally_db_sib._set_cache_meta
271
274
 
272
275
 
273
276
  # === BEGIN MOVED REGIONS ===
@@ -640,7 +643,9 @@ def sync_cache(
640
643
  "DELETE FROM cache_meta WHERE key IN "
641
644
  "('conversation_reingest_pending',"
642
645
  " 'conversation_source_tool_use_reingest_pending',"
643
- " 'conversation_reingest_enrichment_pending')")
646
+ " 'conversation_reingest_enrichment_pending',"
647
+ " 'conversation_reingest_cursor',"
648
+ " 'conversation_reingest_cursor_gen')")
644
649
  conn.commit()
645
650
  eprint("[cache-sync] rebuild: cleared Claude cached entries")
646
651
 
@@ -720,15 +725,12 @@ def sync_cache(
720
725
  except sqlite3.OperationalError:
721
726
  _reingest = False
722
727
  if _reingest:
723
- clear_conversation_messages(conn)
724
- backfill_conversation_messages(conn)
725
- conn.execute(
726
- "DELETE FROM cache_meta WHERE key IN "
727
- "('conversation_reingest_pending',"
728
- " 'conversation_source_tool_use_reingest_pending',"
729
- " 'conversation_reingest_enrichment_pending')"
730
- )
731
- conn.commit()
728
+ # #179: resumable per-file reingest (was a global clear_conversation_messages
729
+ # + offset-0 backfill that re-armed the entire ~2.5min rebuild on any
730
+ # interrupt). The helper checkpoints a sorted-path cursor and clears the
731
+ # three flags + cursor + gen atomically on completion. Never on the rebuild
732
+ # path (which already wipes + repopulates id-aware via the normal walk).
733
+ _resumable_reingest_conversation_messages(conn)
732
734
 
733
735
  paths: list[pathlib.Path] = list(_iter_claude_jsonl_files())
734
736
  stats.files_total = len(paths)
@@ -1157,6 +1159,95 @@ def backfill_conversation_messages(conn: sqlite3.Connection) -> int:
1157
1159
  return inserted
1158
1160
 
1159
1161
 
1162
+ _REINGEST_FLAG_KEYS = (
1163
+ "conversation_reingest_pending",
1164
+ "conversation_source_tool_use_reingest_pending",
1165
+ "conversation_reingest_enrichment_pending",
1166
+ )
1167
+
1168
+
1169
+ def _reingest_parse_file(jp, path_str):
1170
+ """Parse one Claude JSONL into enriched ``conversation_messages`` row tuples
1171
+ (``_CONV_INSERT_SQL`` column order). Mirrors ``backfill_conversation_messages``'s
1172
+ inner read+flatten, factored to a module-level seam so the resumable reingest
1173
+ builds rows BEFORE any write (a parse failure does no DML) and tests can inject.
1174
+ Raises ``OSError`` if the file can't be opened/read."""
1175
+ rows = []
1176
+ with open(jp, "r", encoding="utf-8", errors="replace") as fh:
1177
+ for m in _iter_message_rows(fh, path_str):
1178
+ rows.append(_conv_row_tuple(m, path_str))
1179
+ return rows
1180
+
1181
+
1182
+ def _resumable_reingest_conversation_messages(conn):
1183
+ """#179: resumable, lock-friendly replacement for the old global
1184
+ ``clear_conversation_messages`` + offset-0 ``backfill_conversation_messages``
1185
+ reingest, which re-armed the whole ~2.5min rebuild on any interrupt. Walks
1186
+ every Claude JSONL in deterministic sorted-path order, re-enriching one file
1187
+ per atomic transaction and checkpointing ``conversation_reingest_cursor`` so an
1188
+ interrupt resumes instead of restarting. A ``conversation_reingest_cursor_gen``
1189
+ fingerprint (the set of pending reingest flags) resets the cursor whenever the
1190
+ pending-flag set changes, so a newly-armed flag forces a fresh pass. The caller
1191
+ (``sync_cache``) already holds the cache.db flock; per-file commits bound only
1192
+ the SQLite write transaction, not the flock. Clears the three flags + cursor +
1193
+ gen atomically on completion."""
1194
+ # 1. Generation guard: reset the cursor if the live pending-flag set differs.
1195
+ set_flags = [k for k in _REINGEST_FLAG_KEYS
1196
+ if conn.execute("SELECT 1 FROM cache_meta WHERE key=?", (k,)).fetchone()]
1197
+ gen = ",".join(sorted(set_flags))
1198
+ gen_row = conn.execute(
1199
+ "SELECT value FROM cache_meta WHERE key='conversation_reingest_cursor_gen'"
1200
+ ).fetchone()
1201
+ if (gen_row[0] if gen_row else None) != gen:
1202
+ _set_cache_meta(conn, "conversation_reingest_cursor_gen", gen)
1203
+ conn.execute("DELETE FROM cache_meta WHERE key='conversation_reingest_cursor'")
1204
+ conn.commit()
1205
+ cursor = ""
1206
+ else:
1207
+ crow = conn.execute(
1208
+ "SELECT value FROM cache_meta WHERE key='conversation_reingest_cursor'"
1209
+ ).fetchone()
1210
+ cursor = crow[0] if crow and crow[0] is not None else ""
1211
+
1212
+ # 2. Per-file resumable walk in deterministic sorted-path order.
1213
+ for jp in sorted(_iter_claude_jsonl_files(), key=str):
1214
+ path_str = str(jp)
1215
+ if path_str <= cursor:
1216
+ continue
1217
+ try:
1218
+ rows = _reingest_parse_file(jp, path_str) # parse FIRST — no DML on failure
1219
+ except OSError as exc:
1220
+ # Read/parse failed BEFORE any conversation_messages DML — the file's
1221
+ # existing rows are untouched (preserved, not dropped). Only advance the
1222
+ # cursor; this cursor-only write needs no rollback envelope (no message
1223
+ # DML to undo, and an interrupt mid-commit just re-runs this file).
1224
+ eprint(f"[conversation-reingest] could not read {jp}: {exc}; "
1225
+ "preserving existing rows")
1226
+ _set_cache_meta(conn, "conversation_reingest_cursor", path_str)
1227
+ conn.commit()
1228
+ continue
1229
+ try:
1230
+ conn.execute("DELETE FROM conversation_messages WHERE source_path=?",
1231
+ (path_str,))
1232
+ if rows:
1233
+ conn.executemany(_CONV_INSERT_SQL, rows)
1234
+ _set_cache_meta(conn, "conversation_reingest_cursor", path_str)
1235
+ conn.commit()
1236
+ except BaseException:
1237
+ conn.rollback()
1238
+ raise
1239
+
1240
+ # 3. Completion: clear flags + cursor + gen atomically.
1241
+ conn.execute(
1242
+ "DELETE FROM cache_meta WHERE key IN "
1243
+ "('conversation_reingest_pending',"
1244
+ " 'conversation_source_tool_use_reingest_pending',"
1245
+ " 'conversation_reingest_enrichment_pending',"
1246
+ " 'conversation_reingest_cursor',"
1247
+ " 'conversation_reingest_cursor_gen')")
1248
+ conn.commit()
1249
+
1250
+
1160
1251
  def iter_entries(
1161
1252
  conn: sqlite3.Connection,
1162
1253
  range_start: dt.datetime,
@@ -5341,6 +5341,10 @@ class DashboardHTTPHandler(BaseHTTPRequestHandler):
5341
5341
  self._handle_get_conversations()
5342
5342
  elif path == "/api/conversation/search":
5343
5343
  self._handle_get_conversation_search()
5344
+ elif path.startswith("/api/conversation/") and path.endswith("/payload"):
5345
+ # #178: on-demand load-full. Matched BEFORE the <id> reader
5346
+ # catch-all (same precedence as /api/conversation/search).
5347
+ self._handle_get_conversation_payload(path)
5344
5348
  elif path.startswith("/api/conversation/"):
5345
5349
  self._handle_get_conversation_detail(path)
5346
5350
  else:
@@ -7344,6 +7348,46 @@ class DashboardHTTPHandler(BaseHTTPRequestHandler):
7344
7348
  return
7345
7349
  self._respond_json(200, body)
7346
7350
 
7351
+ def _handle_get_conversation_payload(self, path: str) -> None:
7352
+ """``GET /api/conversation/<sid>/payload?tool_use_id=<id>&which=<result|input>``
7353
+ — the #178 on-demand load-full route. Re-reads the source JSONL line so
7354
+ a clipped result/input can be expanded without enlarging the cache.
7355
+
7356
+ Gated FIRST by the same loopback/Host transcript privacy predicate the
7357
+ three other conversation routes use (fail-closed 403). ``locate_tool_payload``
7358
+ runs against cache.db (via the shared 500-envelope scaffold); the actual
7359
+ full body is re-read from disk by ``read_full_payload`` (no cache conn).
7360
+ ``which`` is validated to ``result``/``input`` (else 400); a missing
7361
+ tool_use_id is 400; an unknown id is 404; a gone/unparseable source line
7362
+ is 410 (the documented consequence of storing only capped text).
7363
+ """
7364
+ if not self._require_transcripts_allowed():
7365
+ return
7366
+ import urllib.parse as _u
7367
+ session_id = _u.unquote(
7368
+ path[len("/api/conversation/"):-len("/payload")])
7369
+ q = _u.parse_qs(self.path.partition("?")[2])
7370
+ tool_use_id = _qs_str(q, "tool_use_id", "")
7371
+ which = _qs_str(q, "which", "result")
7372
+ if not session_id or which not in ("result", "input") or not tool_use_id:
7373
+ self._respond_json(400, {"error": "bad request"})
7374
+ return
7375
+ cq = self._conversation_query()
7376
+ ok, loc = self._run_conversation_query(
7377
+ lambda conn: cq.locate_tool_payload(
7378
+ conn, session_id, tool_use_id, which),
7379
+ "/api/conversation/payload")
7380
+ if not ok:
7381
+ return
7382
+ if loc is None:
7383
+ self._respond_json(404, {"error": "not found"})
7384
+ return
7385
+ payload = cq.read_full_payload(loc[0], loc[1], tool_use_id, which)
7386
+ if payload is None:
7387
+ self._respond_json(410, {"error": "source no longer available"})
7388
+ return
7389
+ self._respond_json(200, payload)
7390
+
7347
7391
  def _handle_get_project_detail(self) -> None:
7348
7392
  """Return ProjectDetail JSON for ``GET /api/project/<key>?weeks=N``
7349
7393
  (spec §5.3 / §6.5).
@@ -7829,6 +7873,21 @@ def _dashboard_wait_for_signal(
7829
7873
  os.close(write_fd)
7830
7874
 
7831
7875
 
7876
+ def _dashboard_initial_snapshot(args, *, pinned_now, display_tz_pref_override):
7877
+ """#179: build the dashboard's first snapshot WITHOUT the heavy sync so the
7878
+ HTTP port binds promptly. The background ``_DashboardSyncThread`` (started in
7879
+ cmd_dashboard before the ThreadingHTTPServer bind) runs the first full
7880
+ ``sync_cache`` — including any pending conversation reingest — and SSE-pushes
7881
+ the populated snapshot on completion. ``--no-sync`` already passed
7882
+ ``skip_sync=True`` here, so that path is byte-identical; only the normal launch
7883
+ changes (its previously-redundant foreground sync is removed, not duplicated —
7884
+ the background thread's first tick was always going to sync anyway)."""
7885
+ return sys.modules["cctally"]._tui_build_snapshot(
7886
+ now_utc=pinned_now, skip_sync=True,
7887
+ display_tz_pref_override=display_tz_pref_override,
7888
+ )
7889
+
7890
+
7832
7891
  def cmd_dashboard(args: argparse.Namespace) -> int:
7833
7892
  """Launch the live web dashboard."""
7834
7893
  import signal as _signal
@@ -7906,9 +7965,13 @@ def cmd_dashboard(args: argparse.Namespace) -> int:
7906
7965
  # weekly, project, forecast subcommands already respect.
7907
7966
  pinned_now = _command_as_of() if os.environ.get("CCTALLY_AS_OF") else None
7908
7967
 
7909
- # Build initial snapshot blocking, serves immediately.
7910
- initial = sys.modules["cctally"]._tui_build_snapshot(
7911
- now_utc=pinned_now, skip_sync=args.no_sync,
7968
+ # #179: build the initial snapshot WITHOUT the heavy sync so the HTTP port
7969
+ # binds promptly even on a heavy-history instance. The background
7970
+ # _DashboardSyncThread (started below, before the bind) owns the first full
7971
+ # sync_cache — including any pending conversation-enrichment reingest — and
7972
+ # SSE-pushes the populated snapshot on completion.
7973
+ initial = _dashboard_initial_snapshot(
7974
+ args, pinned_now=pinned_now,
7912
7975
  display_tz_pref_override=display_tz_pref_override,
7913
7976
  )
7914
7977
  if args.no_sync:
@@ -106,6 +106,7 @@ def _normalize(obj, t, offset):
106
106
  entry_type = TOOL_RESULT
107
107
  _attach_subagent_result(blocks, obj) # #166: record-level toolUseResult
108
108
  _attach_ask_answers(blocks, obj) # #177 S2: AskUserQuestion answers
109
+ _attach_bash_streams(blocks, obj) # #177 S3: Bash stderr/interrupted
109
110
  _attach_task_meta(blocks, obj) # task checklist identity
110
111
  # tool_result rows are stored but NOT indexed as prose (spec §2). A
111
112
  # user line that mixes a text block with a tool_result block must not
@@ -289,6 +290,35 @@ def _attach_ask_answers(blocks, obj):
289
290
  results[0]["ask_annotations"] = bounded_anno
290
291
 
291
292
 
293
+ def _attach_bash_streams(blocks, obj):
294
+ """Stash a Bash tool_result's structured stderr + interrupted onto its single
295
+ tool_result block (#177 S3). Self-identifying: fires only when toolUseResult
296
+ is a dict carrying a ``stdout``/``stderr`` key — a shape distinctive to Bash —
297
+ so no cross-record tool-name lookup is needed (same posture as
298
+ _attach_ask_answers' ``answers`` gate). We do NOT store stdout: the existing
299
+ ``result.text`` already equals stdout+stderr (the merged Bash output), so
300
+ storing stdout would roughly double the at-rest payload; the stdout/stderr
301
+ split is derived client-side by stripping the stderr suffix.
302
+
303
+ Parser-private keys ``bash_stderr`` / ``bash_interrupted`` are popped in the
304
+ query layer's Phase 1 so they never leak into emitted/orphan blocks. stderr is
305
+ bounded with the same cap as result.text (_TOOL_RESULT_CAP). Empty stderr +
306
+ not-interrupted is a no-op (the common case — stderr is empty in ~99% of
307
+ results), keeping the additive contract: absent on every non-Bash result and
308
+ on old rows. Same exactly-one-result-block guard as _attach_subagent_result."""
309
+ tur = obj.get("toolUseResult")
310
+ if not isinstance(tur, dict) or ("stdout" not in tur and "stderr" not in tur):
311
+ return
312
+ results = [b for b in blocks if b.get("kind") == "tool_result"]
313
+ if len(results) != 1:
314
+ return
315
+ stderr = tur.get("stderr")
316
+ if isinstance(stderr, str) and stderr:
317
+ results[0]["bash_stderr"] = stderr[:_TOOL_RESULT_CAP]
318
+ if bool(tur.get("interrupted")):
319
+ results[0]["bash_interrupted"] = True
320
+
321
+
292
322
  # Subagent Task tools record toolUseResult=null and put the identity in the
293
323
  # human-readable result text instead; the id is the only thing the fold needs
294
324
  # from the result (subject/status come from the call input). Anchored to the
@@ -20,6 +20,10 @@ import sqlite3
20
20
  # mirror — imported by the dashboard's conversation endpoints at runtime.
21
21
 
22
22
  from _lib_pricing import _calculate_entry_cost
23
+ # #178: the on-demand load-full re-read helper re-stringifies a raw tool_result
24
+ # content block the same way the parser does at ingest — reuse the parser's
25
+ # _stringify so the full (un-capped) result text matches the cached/capped one.
26
+ from _lib_conversation import _stringify
23
27
 
24
28
 
25
29
  # Mirror of dashboard/web/src/conversations/systemMarkers.ts::MARKER_RE — anchored
@@ -484,6 +488,7 @@ def get_conversation(conn, session_id, *, after=None, limit=500):
484
488
  spawn_kind = {} # tool_use id -> subagent_type
485
489
  agent_link = {} # tool_use id -> (agent_id, raw_meta)
486
490
  ask_link = {} # tool_use id -> (answers, annotations) (#177 S2)
491
+ bash_link = {} # tool_use id -> (stderr, interrupted) (#177 S3)
487
492
  task_link = {} # tool_use id -> {"task_id", "task_list"} (Task* checklist)
488
493
  for it in items:
489
494
  for b in it["blocks"]:
@@ -501,6 +506,10 @@ def get_conversation(conn, session_id, *, after=None, limit=500):
501
506
  anno = b.pop("ask_annotations", None)
502
507
  if ans is not None and b.get("tool_use_id") is not None:
503
508
  ask_link[b["tool_use_id"]] = (ans, anno)
509
+ bstderr = b.pop("bash_stderr", None) # #177 S3
510
+ bintr = b.pop("bash_interrupted", None)
511
+ if b.get("tool_use_id") is not None and (bstderr is not None or bintr):
512
+ bash_link[b["tool_use_id"]] = (bstderr, bool(bintr))
504
513
  tid_ = b.pop("task_id", None) # Task* checklist
505
514
  tlist_ = b.pop("task_list", None)
506
515
  if b.get("tool_use_id") is not None and (tid_ is not None or tlist_ is not None):
@@ -568,6 +577,12 @@ def get_conversation(conn, session_id, *, after=None, limit=500):
568
577
  b["answers"] = link[0]
569
578
  if link[1]:
570
579
  b["annotations"] = link[1]
580
+ blink = bash_link.get(b["tool_use_id"]) # #177 S3
581
+ if blink is not None:
582
+ if blink[0] is not None:
583
+ b["stderr"] = blink[0]
584
+ if blink[1]:
585
+ b["interrupted"] = True
571
586
 
572
587
  # ---- Phase 3b: fold the Task* op stream into per-run checklist snapshots ----
573
588
  _fold_task_runs(items, task_link)
@@ -1127,3 +1142,178 @@ def _manual_snippet(text, q, width=80):
1127
1142
  end = min(len(text), lo + len(q) + width // 2)
1128
1143
  s = text[start:end]
1129
1144
  return ("… " if start else "") + s + (" …" if end < len(text) else "")
1145
+
1146
+
1147
+ # ---------------------------------------------------------------------------
1148
+ # #178: on-demand "load full result/input" kernels. Back the dashboard's
1149
+ # /api/conversation/<sid>/payload route. The cache stores only CAPPED tool text
1150
+ # (result clipped to _TOOL_RESULT_CAP, input leaves clipped to _INPUT_LEAF_CAP),
1151
+ # so the full body is re-derived from the source JSONL line here — the cache at
1152
+ # rest never grows. Pure (sqlite3.Connection + filesystem); no clock/network.
1153
+ # ---------------------------------------------------------------------------
1154
+ _FULL_PAYLOAD_CEILING = 1_000_000 # serve up to ~1 MB; protects the HTTP server / browser
1155
+
1156
+
1157
+ def locate_tool_payload(conn, session_id, tool_use_id, which):
1158
+ """``(source_path, byte_offset)`` for the JSONL line holding the tool_use
1159
+ (``which='input'``) or tool_result (``which='result'``) carrying this
1160
+ ``tool_use_id`` in this session, else ``None``.
1161
+
1162
+ The prefilter uses ``instr(blocks_json, ?) > 0`` — NOT ``LIKE`` (Codex P1.4):
1163
+ tool_use_ids contain ``_`` (e.g. ``toolu_01SEQ…``), which ``LIKE`` treats as a
1164
+ single-char wildcard, so a near-miss id would false-match. ``instr`` is a
1165
+ literal substring test. It is also NOT a ``≤2 rows`` situation — the DB holds
1166
+ duplicate physical rows for one logical message (``get_conversation`` dedups
1167
+ by uuid) — so every candidate is parsed and EXACT-matched on the block id
1168
+ (``tool_use.id`` for input / ``tool_result.tool_use_id`` for result), under the
1169
+ same deterministic ``ORDER BY timestamp_utc, id`` as ``get_conversation``. The
1170
+ SELECT runs here (not via ``get_conversation``) because that reader omits
1171
+ ``byte_offset`` (Codex P2.5)."""
1172
+ rows = conn.execute(
1173
+ "SELECT source_path, byte_offset, blocks_json FROM conversation_messages "
1174
+ "WHERE session_id=? AND instr(blocks_json, ?) > 0 "
1175
+ "ORDER BY timestamp_utc, id", (session_id, tool_use_id)).fetchall()
1176
+ for source_path, byte_offset, blocks_json in rows:
1177
+ try:
1178
+ blocks = _json.loads(blocks_json)
1179
+ except (ValueError, TypeError):
1180
+ continue
1181
+ if not isinstance(blocks, list):
1182
+ continue
1183
+ for b in blocks:
1184
+ if not isinstance(b, dict):
1185
+ continue
1186
+ k = b.get("kind")
1187
+ if which == "input" and k == "tool_use" and b.get("id") == tool_use_id:
1188
+ return source_path, byte_offset
1189
+ if which == "result" and k == "tool_result" and b.get("tool_use_id") == tool_use_id:
1190
+ return source_path, byte_offset
1191
+ return None
1192
+
1193
+
1194
+ def _clip_payload_input(inp, ceiling):
1195
+ """Clip a structured input so the returned dict serializes to ``ceiling`` chars
1196
+ or fewer, and report whether anything was clipped — the input-side analogue of
1197
+ the result-side ceiling. A degenerate multi-MB input (one giant leaf OR many
1198
+ sub-ceiling leaves that sum past the ceiling) is bounded so the HTTP server /
1199
+ browser is protected, while every real payload returns whole.
1200
+
1201
+ The guarantee is AGGREGATE, not merely per-leaf: a shared remaining-char budget
1202
+ is threaded through the walk (mirroring ``_bound_input``'s total-size backstop) —
1203
+ each string leaf is clipped against the running budget and the budget is
1204
+ decremented as we go, so once it is exhausted later leaves clip to ''.
1205
+ ``truncated`` is True iff any leaf was clipped (or the post-walk serialized size
1206
+ still exceeds ``ceiling``, the structural-overhead backstop). Post-condition:
1207
+ ``len(json.dumps(clipped, ensure_ascii=False)) <= ceiling`` always."""
1208
+ truncated = False
1209
+ remaining = [ceiling] # boxed so the nested walk can decrement it
1210
+
1211
+ def walk(v):
1212
+ nonlocal truncated
1213
+ if isinstance(v, str):
1214
+ if len(v) > remaining[0]:
1215
+ truncated = True
1216
+ v = v[:remaining[0]]
1217
+ remaining[0] -= len(v)
1218
+ return v
1219
+ if isinstance(v, dict):
1220
+ return {k: walk(x) for k, x in v.items()}
1221
+ if isinstance(v, list):
1222
+ return [walk(x) for x in v]
1223
+ return v
1224
+
1225
+ clipped = walk(inp)
1226
+ # Backstop: structural JSON overhead (braces/quotes/keys) can push a
1227
+ # budget-exact payload a few chars past the ceiling. Hard-clip the largest
1228
+ # remaining string leaf(s) until the whole dict serializes within the ceiling.
1229
+ while len(_json.dumps(clipped, ensure_ascii=False)) > ceiling:
1230
+ truncated = True
1231
+ if not _shrink_largest_leaf(clipped):
1232
+ break # no string leaf left to shrink (e.g. pure numeric/structural)
1233
+ return clipped, truncated
1234
+
1235
+
1236
+ def _shrink_largest_leaf(obj):
1237
+ """Halve the longest string leaf reachable in ``obj`` (a dict/list/scalar),
1238
+ in place, and return True if one was shrunk — the post-walk backstop for the
1239
+ rare structural-overhead overshoot. A leaf already at length 1 is truncated to
1240
+ ''. Returns False when no non-empty string leaf exists."""
1241
+ best = {"len": 0, "container": None, "key": None}
1242
+
1243
+ def scan(v, container, key):
1244
+ if isinstance(v, str):
1245
+ if len(v) > best["len"]:
1246
+ best.update(len=len(v), container=container, key=key)
1247
+ elif isinstance(v, dict):
1248
+ for k, x in v.items():
1249
+ scan(x, v, k)
1250
+ elif isinstance(v, list):
1251
+ for i, x in enumerate(v):
1252
+ scan(x, v, i)
1253
+
1254
+ scan(obj, None, None)
1255
+ if best["container"] is None or best["len"] == 0:
1256
+ return False
1257
+ s = best["container"][best["key"]]
1258
+ best["container"][best["key"]] = s[: len(s) // 2]
1259
+ return True
1260
+
1261
+
1262
+ def read_full_payload(source_path, byte_offset, tool_use_id, which):
1263
+ """Re-read the raw JSONL line at ``(source_path, byte_offset)`` and return the
1264
+ FULL (un-capped) payload for ``tool_use_id``:
1265
+
1266
+ - ``which='input'`` -> ``{"which":"input", "tool_use_id", "input", "full_length",
1267
+ "truncated"}`` — the matching tool_use block's complete ``input`` dict, so the
1268
+ DiffCard can pull old/new strings straight into computeDiff.
1269
+ - ``which='result'`` -> ``{"which":"result", "tool_use_id", "text", "full_length",
1270
+ "truncated", "is_error", [stderr]}`` — the full ``_stringify(content)`` plus,
1271
+ for Bash, the full ``toolUseResult.stderr``.
1272
+
1273
+ ``None`` when the source is gone / the line is unparseable (rotated or deleted
1274
+ JSONL — the documented 410 path) or the id is no longer present in that line.
1275
+ ``full_length``/``truncated`` describe the payload against ``_FULL_PAYLOAD_CEILING``
1276
+ — honoring #178's "un-capped" spirit for real payloads while bounding the
1277
+ degenerate multi-MB case."""
1278
+ try:
1279
+ with open(source_path, "rb") as fh:
1280
+ fh.seek(byte_offset)
1281
+ line = fh.readline()
1282
+ obj = _json.loads(line)
1283
+ except (OSError, ValueError):
1284
+ return None
1285
+ if not isinstance(obj, dict):
1286
+ return None
1287
+ content = (obj.get("message") or {}).get("content")
1288
+ if not isinstance(content, list):
1289
+ return None
1290
+ if which == "input":
1291
+ for b in content:
1292
+ if (isinstance(b, dict) and b.get("type") == "tool_use"
1293
+ and b.get("id") == tool_use_id):
1294
+ inp = b.get("input") if isinstance(b.get("input"), dict) else {}
1295
+ full_length = len(_json.dumps(inp, ensure_ascii=False))
1296
+ clipped, truncated = _clip_payload_input(inp, _FULL_PAYLOAD_CEILING)
1297
+ return {"which": "input", "tool_use_id": tool_use_id,
1298
+ "input": clipped, "full_length": full_length,
1299
+ "truncated": truncated}
1300
+ return None
1301
+ for b in content:
1302
+ if (isinstance(b, dict) and b.get("type") == "tool_result"
1303
+ and b.get("tool_use_id") == tool_use_id):
1304
+ raw = _stringify(b.get("content"))
1305
+ # The bound is PER-STREAM by design: `text` and the Bash `stderr`
1306
+ # below are each clipped to _FULL_PAYLOAD_CEILING independently, so a
1307
+ # result carrying both can serialize to ~2× the ceiling. That is
1308
+ # intentional — they are distinct streams the DiffCard renders side by
1309
+ # side, and each is individually bounded against the HTTP/browser DoS.
1310
+ resp = {"which": "result", "tool_use_id": tool_use_id,
1311
+ "text": raw[:_FULL_PAYLOAD_CEILING], "full_length": len(raw),
1312
+ "truncated": len(raw) > _FULL_PAYLOAD_CEILING,
1313
+ "is_error": bool(b.get("is_error"))}
1314
+ tur = obj.get("toolUseResult")
1315
+ if (isinstance(tur, dict) and isinstance(tur.get("stderr"), str)
1316
+ and tur.get("stderr")):
1317
+ resp["stderr"] = tur["stderr"][:_FULL_PAYLOAD_CEILING] # per-stream bound (see above)
1318
+ return resp
1319
+ return None