cctally 1.37.2 → 1.38.1

package/CHANGELOG.md

@@ -5,6 +5,18 @@ based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 
 ## [Unreleased]
 
+## [1.38.1] - 2026-06-12
+
+### Fixed
+- Dashboard: stop the steady-state one-core CPU peg on large caches. The background sync thread rebuilds the full snapshot every few seconds, and several panels read wide historical ranges that re-parsed the per-row `usage_extra_json` JSON blob on every tick — on a ~260K-entry cache a single rebuild cost roughly a core-second, so rebuilds ran back-to-back and one core stayed pinned. The only non-token field any reader consumes (`speed`, for the `<model>-fast` label) is now materialized into a real `session_entries.speed` column, so the hot cache read paths reconstruct usage with zero JSON parsing; a one-time cache migration backfills the column from the existing blobs, and new ingests write the column directly (the now-unused blob is written `NULL` and reclaimed on the next `cache-sync --rebuild`). Output is byte-identical everywhere `speed` matters; on a 250K-row synthetic cache the wide read dropped from ~1.6s to ~0.35s per call (#181).
+
+## [1.38.0] - 2026-06-12
+
+### Added
+- Conversation viewer: Edit, MultiEdit, and Write now render as a unified red/green word-diff card — a `+N −M` stat header with the file's basename and dir, intra-line word-level highlighting on the changed text, per-language syntax coloring on the unchanged context lines, one hunk per edit for MultiEdit, and an all-added "wrote N lines" view for Write — with a collapsed `result · cat -n` sub-panel beneath carrying the real file line numbers, instead of a raw JSON tool chip (#177 S3).
+- Conversation viewer: Bash now renders as a terminal — a `$ <command>` prompt over the output, with `stderr` split into a red block, an `error`/`interrupted` status badge, and ANSI foreground colors honored — instead of an undifferentiated output blob; legacy transcripts captured before the stream split degrade to a merged terminal view (#177 S3).
+- Conversation viewer: a truncated tool input or result can now be expanded on demand — a "load full input"/"load full output" affordance re-reads the original session JSONL behind the existing transcript privacy gate to recompute the diff or re-render the terminal, without enlarging the on-disk cache; a rotated or deleted source surfaces a clear "source no longer available" note (#178).
+
 ## [1.37.2] - 2026-06-12
 
 ### Fixed

package/bin/_cctally_cache.py

@@ -928,14 +928,12 @@ def sync_cache(
                             out = int(usage.get("output_tokens", 0) or 0)
                             cc = int(usage.get("cache_creation_input_tokens", 0) or 0)
                             cr = int(usage.get("cache_read_input_tokens", 0) or 0)
-                            extras = {
-                                k: v for k, v in usage.items()
-                                if k not in (
-                                    "input_tokens", "output_tokens",
-                                    "cache_creation_input_tokens",
-                                    "cache_read_input_tokens",
-                                )
-                            }
+                            # #181: `speed` is the ONLY non-token usage key any
+                            # consumer reads, so materialize just that scalar and
+                            # write NULL into usage_extra_json — no more
+                            # serializing the deeply-nested blob the read paths
+                            # used to json.loads per row.
+                            speed = usage.get("speed")
                             rows.append((
                                 path_str,
                                 offset,
@@ -944,7 +942,8 @@ def sync_cache(
                                 msg_id,
                                 req_id,
                                 inp, out, cc, cr,
-                                json.dumps(extras, sort_keys=True) if extras else None,
+                                None,    # usage_extra_json — bloat no longer written (#181)
+                                speed,   # materialized speed column
                                 entry.cost_usd,
                             ))
                         if mrow is not None:
@@ -1005,8 +1004,8 @@ def sync_cache(
                            (source_path, line_offset, timestamp_utc, model,
                             msg_id, req_id, input_tokens, output_tokens,
                             cache_create_tokens, cache_read_tokens,
-                            usage_extra_json, cost_usd_raw)
-                           VALUES (?,?,?,?,?,?,?,?,?,?,?,?)
+                            usage_extra_json, speed, cost_usd_raw)
+                           VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)
                            ON CONFLICT(msg_id, req_id)
                            WHERE msg_id IS NOT NULL AND req_id IS NOT NULL
                            DO UPDATE SET
@@ -1017,6 +1016,7 @@ def sync_cache(
                                cache_create_tokens = excluded.cache_create_tokens,
                                cache_read_tokens = excluded.cache_read_tokens,
                                usage_extra_json = excluded.usage_extra_json,
+                               speed = excluded.speed,
                                cost_usd_raw = excluded.cost_usd_raw
                            WHERE
                                (excluded.input_tokens + excluded.output_tokens
@@ -1030,8 +1030,8 @@ def sync_cache(
                                =
                                (session_entries.input_tokens + session_entries.output_tokens
                                 + session_entries.cache_create_tokens + session_entries.cache_read_tokens)
-                               AND json_extract(excluded.usage_extra_json, '$.speed') IS NOT NULL
-                               AND json_extract(session_entries.usage_extra_json, '$.speed') IS NULL
+                               AND excluded.speed IS NOT NULL
+                               AND session_entries.speed IS NULL
                             )""",
                         rows,
                     )
@@ -1266,7 +1266,7 @@ def iter_entries(
 
     sql = (
         "SELECT timestamp_utc, model, input_tokens, output_tokens, "
-        "cache_create_tokens, cache_read_tokens, usage_extra_json, "
+        "cache_create_tokens, cache_read_tokens, speed, "
         "cost_usd_raw, source_path "
         "FROM session_entries "
         "WHERE timestamp_utc >= ? AND timestamp_utc <= ?"
@@ -1292,12 +1292,12 @@ def iter_entries(
             "cache_creation_input_tokens": row[4],
             "cache_read_input_tokens":     row[5],
         }
-        if row[6]:
-            # Safe because sync_cache strips the four token keys from
-            # extras before storing them in usage_extra_json. If that
-            # write-side invariant ever changes, extras could shadow
-            # the int-normalized token columns.
-            usage.update(json.loads(row[6]))
+        # speed is the only non-token usage key any consumer reads (#181);
+        # materialized into its own column so this hot path never parses JSON.
+        # `is not None` (not truthiness) so an empty-string speed still surfaces,
+        # mirroring the SQL `json_extract(...) IS NOT NULL` parity.
+        if row[6] is not None:
+            usage["speed"] = row[6]
         entries.append(UsageEntry(
             timestamp=dt.datetime.fromisoformat(row[0]),
             model=row[1],
@@ -1430,7 +1430,7 @@ def get_claude_session_entries(
         "  se.cache_create_tokens, se.cache_read_tokens, "
         "  se.source_path, "
         "  sf.session_id, sf.project_path, "
-        "  se.cost_usd_raw, se.usage_extra_json "
+        "  se.cost_usd_raw, se.speed "
         "FROM session_entries se "
         "LEFT JOIN session_files sf ON sf.path = se.source_path "
         "WHERE se.timestamp_utc >= ? AND se.timestamp_utc <= ?"
@@ -1458,7 +1458,10 @@ def get_claude_session_entries(
             session_id=row[7],
             project_path=row[8],
             cost_usd=row[9],
-            usage_extra=(json.loads(row[10]) if row[10] else None),
+            # speed materialized into its own column (#181); reconstruct the
+            # {"speed": …} shape _usage_entry_from_joined already merges, with
+            # zero JSON parsing. `is not None` so an empty-string speed surfaces.
+            usage_extra=({"speed": row[10]} if row[10] is not None else None),
         )
         for row in rows
     ]

package/bin/_cctally_dashboard.py

@@ -5341,6 +5341,10 @@ class DashboardHTTPHandler(BaseHTTPRequestHandler):
             self._handle_get_conversations()
         elif path == "/api/conversation/search":
             self._handle_get_conversation_search()
+        elif path.startswith("/api/conversation/") and path.endswith("/payload"):
+            # #178: on-demand load-full. Matched BEFORE the <id> reader
+            # catch-all (same precedence as /api/conversation/search).
+            self._handle_get_conversation_payload(path)
         elif path.startswith("/api/conversation/"):
             self._handle_get_conversation_detail(path)
         else:
@@ -7344,6 +7348,46 @@ class DashboardHTTPHandler(BaseHTTPRequestHandler):
             return
         self._respond_json(200, body)
 
+    def _handle_get_conversation_payload(self, path: str) -> None:
+        """``GET /api/conversation/<sid>/payload?tool_use_id=<id>&which=<result|input>``
+        — the #178 on-demand load-full route. Re-reads the source JSONL line so
+        a clipped result/input can be expanded without enlarging the cache.
+
+        Gated FIRST by the same loopback/Host transcript privacy predicate the
+        three other conversation routes use (fail-closed 403). ``locate_tool_payload``
+        runs against cache.db (via the shared 500-envelope scaffold); the actual
+        full body is re-read from disk by ``read_full_payload`` (no cache conn).
+        ``which`` is validated to ``result``/``input`` (else 400); a missing
+        tool_use_id is 400; an unknown id is 404; a gone/unparseable source line
+        is 410 (the documented consequence of storing only capped text).
+        """
+        if not self._require_transcripts_allowed():
+            return
+        import urllib.parse as _u
+        session_id = _u.unquote(
+            path[len("/api/conversation/"):-len("/payload")])
+        q = _u.parse_qs(self.path.partition("?")[2])
+        tool_use_id = _qs_str(q, "tool_use_id", "")
+        which = _qs_str(q, "which", "result")
+        if not session_id or which not in ("result", "input") or not tool_use_id:
+            self._respond_json(400, {"error": "bad request"})
+            return
+        cq = self._conversation_query()
+        ok, loc = self._run_conversation_query(
+            lambda conn: cq.locate_tool_payload(
+                conn, session_id, tool_use_id, which),
+            "/api/conversation/payload")
+        if not ok:
+            return
+        if loc is None:
+            self._respond_json(404, {"error": "not found"})
+            return
+        payload = cq.read_full_payload(loc[0], loc[1], tool_use_id, which)
+        if payload is None:
+            self._respond_json(410, {"error": "source no longer available"})
+            return
+        self._respond_json(200, payload)
+
     def _handle_get_project_detail(self) -> None:
         """Return ProjectDetail JSON for ``GET /api/project/<key>?weeks=N``
         (spec §5.3 / §6.5).

package/bin/_cctally_db.py

@@ -2304,7 +2304,8 @@ def _apply_cache_schema(conn: sqlite3.Connection) -> None:
             cache_create_tokens INTEGER NOT NULL DEFAULT 0,
             cache_read_tokens   INTEGER NOT NULL DEFAULT 0,
             usage_extra_json    TEXT,
-            cost_usd_raw        REAL
+            cost_usd_raw        REAL,
+            speed               TEXT
         );
         CREATE INDEX IF NOT EXISTS idx_entries_timestamp
             ON session_entries(timestamp_utc);
@@ -2384,6 +2385,14 @@ def _apply_cache_schema(conn: sqlite3.Connection) -> None:
     # populated lazily in sync_cache() / _ensure_session_files_row().
     add_column_if_missing(conn, "session_files", "session_id", "TEXT")
     add_column_if_missing(conn, "session_files", "project_path", "TEXT")
+    # #181: materialize the only-ever-consumed extra `usage` key (`speed`) into
+    # a real session_entries column so the hot cache read paths (iter_entries /
+    # get_claude_session_entries) stop json.loads-ing the deeply-nested
+    # usage_extra_json blob per row — that per-tick parse was pegging a core in
+    # the dashboard. Idempotent column-add (no marker, no version), appended
+    # after cost_usd_raw to match the CREATE TABLE order; cache migration 008
+    # then backfills it from the legacy blob on existing rows.
+    add_column_if_missing(conn, "session_entries", "speed", "TEXT")
     # Existing-DB guard for the skill-content fold link (cctally-dev
     # skill-content-nesting): the message-level sourceToolUseID. Idempotent
     # column-add (no marker, no version); cache migration 006 then re-ingests
@@ -3171,6 +3180,33 @@ def _007_conversation_reingest_enrichment(conn: sqlite3.Connection) -> None:
     conn.commit()
 
 
+@cache_migration("008_session_entries_speed_backfill")
+def _008_session_entries_speed_backfill(conn: sqlite3.Connection) -> None:
+    """Backfill the materialized ``session_entries.speed`` column from the
+    legacy ``usage_extra_json`` blob (#181). ``speed`` is the ONLY non-token
+    ``usage`` key any consumer reads (``<model>-fast`` rendering in
+    _lib_aggregators + the _should_replace dedup tiebreak); materializing it
+    lets the hot read paths (iter_entries / get_claude_session_entries) stop
+    ``json.loads``-ing the deeply-nested blob per row — the per-tick dashboard
+    rebuild was pegging a core on a ~261K-row cache.
+
+    The column is added by _apply_cache_schema's add_column_if_missing (which
+    runs before this dispatcher), so it always exists here. This handler only
+    backfills existing rows; new ingests write the column directly and write
+    NULL to usage_extra_json. ``WHERE speed IS NULL`` self-guards re-runs.
+    We do NOT rewrite/NULL usage_extra_json on existing rows or VACUUM — the
+    stale blob has no reader and is reclaimed on the next cache-sync --rebuild.
+    json_extract returns NULL when '$.speed' is absent, so speed-less rows stay
+    NULL. Central stamp via the dispatcher (#140); a fresh install stamps it
+    without effect (empty table)."""
+    conn.execute(
+        "UPDATE session_entries "
+        "SET speed = json_extract(usage_extra_json, '$.speed') "
+        "WHERE speed IS NULL AND usage_extra_json IS NOT NULL"
+    )
+    conn.commit()
+
+
 # === Region 7d: Stats migration 008_recompute_weekly_cost_snapshots_dedup_fix ===
 
 @stats_migration("008_recompute_weekly_cost_snapshots_dedup_fix")
@@ -3323,6 +3359,10 @@ def _008_recompute_weekly_cost_snapshots_dedup_fix(
                         "cache_creation_input_tokens": cc,
                         "cache_read_input_tokens": cr,
                     }
+                    # #181: usage_extra_json is cost-irrelevant (cost is
+                    # token-only); parsed here only for pre-008 rows that may
+                    # still carry the legacy blob — NOT a speed reader, so the
+                    # write-side NULL going forward is safe.
                     if extras_json:
                         usage.update(json.loads(extras_json))
                     total += _calculate_entry_cost(
@@ -3637,6 +3677,10 @@ def _009_recompute_five_hour_blocks_dedup_fix(
                         "cache_creation_input_tokens": cc_t,
                         "cache_read_input_tokens": cr_t,
                     }
+                    # #181: usage_extra_json is cost-irrelevant (cost is
+                    # token-only); parsed here only for pre-008 rows that may
+                    # still carry the legacy blob — NOT a speed reader, so the
+                    # write-side NULL going forward is safe.
                     if extras_json:
                         usage.update(json.loads(extras_json))
                     cost = _calculate_entry_cost(
@@ -3909,6 +3953,10 @@ def _010_recompute_percent_milestones_dedup_fix(
                         "cache_creation_input_tokens": cc,
                         "cache_read_input_tokens": cr,
                     }
+                    # #181: usage_extra_json is cost-irrelevant (cost is
+                    # token-only); parsed here only for pre-008 rows that may
+                    # still carry the legacy blob — NOT a speed reader, so the
+                    # write-side NULL going forward is safe.
                     if extras_json:
                         usage.update(json.loads(extras_json))
                     cumulative += _calculate_entry_cost(

package/bin/_cctally_statusline.py

@@ -345,7 +345,7 @@ def _build_statusline_injections(warn_once):
                 "SELECT se.timestamp_utc, se.model, "
                 "  se.input_tokens, se.output_tokens, "
                 "  se.cache_create_tokens, se.cache_read_tokens, "
-                "  se.cost_usd_raw, se.usage_extra_json "
+                "  se.cost_usd_raw "
                 "FROM session_entries se "
                 "LEFT JOIN session_files sf ON sf.path = se.source_path "
                 "WHERE sf.session_id = ?"
@@ -368,13 +368,10 @@ def _build_statusline_injections(warn_once):
                 "cache_creation_input_tokens": r[4] or 0,
                 "cache_read_input_tokens":     r[5] or 0,
             }
-            try:
-                if r[7]:
-                    extras = json.loads(r[7])
-                    if isinstance(extras, dict):
-                        usage.update(extras)
-            except Exception:
-                pass
+            # #181: cost is token-only (_calculate_entry_cost ignores `speed`),
+            # so the statusline cost path no longer selects or json.loads the
+            # usage_extra_json blob — output is byte-identical. r[6] is still
+            # cost_usd_raw (the dropped column was the trailing slot).
             try:
                 total += c._calculate_entry_cost(
                     r[1], usage, mode="auto", cost_usd=r[6],

package/bin/_lib_conversation.py

@@ -106,6 +106,7 @@ def _normalize(obj, t, offset):
         entry_type = TOOL_RESULT
         _attach_subagent_result(blocks, obj)   # #166: record-level toolUseResult
         _attach_ask_answers(blocks, obj)       # #177 S2: AskUserQuestion answers
+        _attach_bash_streams(blocks, obj)      # #177 S3: Bash stderr/interrupted
         _attach_task_meta(blocks, obj)         # task checklist identity
         # tool_result rows are stored but NOT indexed as prose (spec §2). A
         # user line that mixes a text block with a tool_result block must not
@@ -289,6 +290,35 @@ def _attach_ask_answers(blocks, obj):
         results[0]["ask_annotations"] = bounded_anno
 
 
+def _attach_bash_streams(blocks, obj):
+    """Stash a Bash tool_result's structured stderr + interrupted onto its single
+    tool_result block (#177 S3). Self-identifying: fires only when toolUseResult
+    is a dict carrying a ``stdout``/``stderr`` key — a shape distinctive to Bash —
+    so no cross-record tool-name lookup is needed (same posture as
+    _attach_ask_answers' ``answers`` gate). We do NOT store stdout: the existing
+    ``result.text`` already equals stdout+stderr (the merged Bash output), so
+    storing stdout would roughly double the at-rest payload; the stdout/stderr
+    split is derived client-side by stripping the stderr suffix.
+
+    Parser-private keys ``bash_stderr`` / ``bash_interrupted`` are popped in the
+    query layer's Phase 1 so they never leak into emitted/orphan blocks. stderr is
+    bounded with the same cap as result.text (_TOOL_RESULT_CAP). Empty stderr +
+    not-interrupted is a no-op (the common case — stderr is empty in ~99% of
+    results), keeping the additive contract: absent on every non-Bash result and
+    on old rows. Same exactly-one-result-block guard as _attach_subagent_result."""
+    tur = obj.get("toolUseResult")
+    if not isinstance(tur, dict) or ("stdout" not in tur and "stderr" not in tur):
+        return
+    results = [b for b in blocks if b.get("kind") == "tool_result"]
+    if len(results) != 1:
+        return
+    stderr = tur.get("stderr")
+    if isinstance(stderr, str) and stderr:
+        results[0]["bash_stderr"] = stderr[:_TOOL_RESULT_CAP]
+    if bool(tur.get("interrupted")):
+        results[0]["bash_interrupted"] = True
+
+
 # Subagent Task tools record toolUseResult=null and put the identity in the
 # human-readable result text instead; the id is the only thing the fold needs
 # from the result (subject/status come from the call input). Anchored to the

package/bin/_lib_conversation_query.py

@@ -20,6 +20,10 @@ import sqlite3
 # mirror — imported by the dashboard's conversation endpoints at runtime.
 
 from _lib_pricing import _calculate_entry_cost
+# #178: the on-demand load-full re-read helper re-stringifies a raw tool_result
+# content block the same way the parser does at ingest — reuse the parser's
+# _stringify so the full (un-capped) result text matches the cached/capped one.
+from _lib_conversation import _stringify
 
 
 # Mirror of dashboard/web/src/conversations/systemMarkers.ts::MARKER_RE — anchored
@@ -484,6 +488,7 @@ def get_conversation(conn, session_id, *, after=None, limit=500):
     spawn_kind = {}     # tool_use id -> subagent_type
     agent_link = {}     # tool_use id -> (agent_id, raw_meta)
     ask_link = {}       # tool_use id -> (answers, annotations)  (#177 S2)
+    bash_link = {}      # tool_use id -> (stderr, interrupted)   (#177 S3)
     task_link = {}      # tool_use id -> {"task_id", "task_list"}  (Task* checklist)
     for it in items:
         for b in it["blocks"]:
@@ -501,6 +506,10 @@ def get_conversation(conn, session_id, *, after=None, limit=500):
                 anno = b.pop("ask_annotations", None)
                 if ans is not None and b.get("tool_use_id") is not None:
                     ask_link[b["tool_use_id"]] = (ans, anno)
+                bstderr = b.pop("bash_stderr", None)        # #177 S3
+                bintr = b.pop("bash_interrupted", None)
+                if b.get("tool_use_id") is not None and (bstderr is not None or bintr):
+                    bash_link[b["tool_use_id"]] = (bstderr, bool(bintr))
                 tid_ = b.pop("task_id", None)               # Task* checklist
                 tlist_ = b.pop("task_list", None)
                 if b.get("tool_use_id") is not None and (tid_ is not None or tlist_ is not None):
@@ -568,6 +577,12 @@ def get_conversation(conn, session_id, *, after=None, limit=500):
                         b["answers"] = link[0]
                         if link[1]:
                             b["annotations"] = link[1]
+                    blink = bash_link.get(b["tool_use_id"])  # #177 S3
+                    if blink is not None:
+                        if blink[0] is not None:
+                            b["stderr"] = blink[0]
+                        if blink[1]:
+                            b["interrupted"] = True
 
     # ---- Phase 3b: fold the Task* op stream into per-run checklist snapshots ----
     _fold_task_runs(items, task_link)
@@ -1127,3 +1142,178 @@ def _manual_snippet(text, q, width=80):
     end = min(len(text), lo + len(q) + width // 2)
     s = text[start:end]
     return ("… " if start else "") + s + (" …" if end < len(text) else "")
+
+
+# ---------------------------------------------------------------------------
+# #178: on-demand "load full result/input" kernels. Back the dashboard's
+# /api/conversation/<sid>/payload route. The cache stores only CAPPED tool text
+# (result clipped to _TOOL_RESULT_CAP, input leaves clipped to _INPUT_LEAF_CAP),
+# so the full body is re-derived from the source JSONL line here — the cache at
+# rest never grows. Pure (sqlite3.Connection + filesystem); no clock/network.
+# ---------------------------------------------------------------------------
+_FULL_PAYLOAD_CEILING = 1_000_000   # serve up to ~1 MB; protects the HTTP server / browser
+
+
+def locate_tool_payload(conn, session_id, tool_use_id, which):
+    """``(source_path, byte_offset)`` for the JSONL line holding the tool_use
+    (``which='input'``) or tool_result (``which='result'``) carrying this
+    ``tool_use_id`` in this session, else ``None``.
+
+    The prefilter uses ``instr(blocks_json, ?) > 0`` — NOT ``LIKE`` (Codex P1.4):
+    tool_use_ids contain ``_`` (e.g. ``toolu_01SEQ…``), which ``LIKE`` treats as a
+    single-char wildcard, so a near-miss id would false-match. ``instr`` is a
+    literal substring test. It is also NOT a ``≤2 rows`` situation — the DB holds
+    duplicate physical rows for one logical message (``get_conversation`` dedups
+    by uuid) — so every candidate is parsed and EXACT-matched on the block id
+    (``tool_use.id`` for input / ``tool_result.tool_use_id`` for result), under the
+    same deterministic ``ORDER BY timestamp_utc, id`` as ``get_conversation``. The
+    SELECT runs here (not via ``get_conversation``) because that reader omits
+    ``byte_offset`` (Codex P2.5)."""
+    rows = conn.execute(
+        "SELECT source_path, byte_offset, blocks_json FROM conversation_messages "
+        "WHERE session_id=? AND instr(blocks_json, ?) > 0 "
+        "ORDER BY timestamp_utc, id", (session_id, tool_use_id)).fetchall()
+    for source_path, byte_offset, blocks_json in rows:
+        try:
+            blocks = _json.loads(blocks_json)
+        except (ValueError, TypeError):
+            continue
+        if not isinstance(blocks, list):
+            continue
+        for b in blocks:
+            if not isinstance(b, dict):
+                continue
+            k = b.get("kind")
+            if which == "input" and k == "tool_use" and b.get("id") == tool_use_id:
+                return source_path, byte_offset
+            if which == "result" and k == "tool_result" and b.get("tool_use_id") == tool_use_id:
+                return source_path, byte_offset
+    return None
+
+
+def _clip_payload_input(inp, ceiling):
+    """Clip a structured input so the returned dict serializes to ``ceiling`` chars
+    or fewer, and report whether anything was clipped — the input-side analogue of
+    the result-side ceiling. A degenerate multi-MB input (one giant leaf OR many
+    sub-ceiling leaves that sum past the ceiling) is bounded so the HTTP server /
+    browser is protected, while every real payload returns whole.
+
+    The guarantee is AGGREGATE, not merely per-leaf: a shared remaining-char budget
+    is threaded through the walk (mirroring ``_bound_input``'s total-size backstop) —
+    each string leaf is clipped against the running budget and the budget is
+    decremented as we go, so once it is exhausted later leaves clip to ''.
+    ``truncated`` is True iff any leaf was clipped (or the post-walk serialized size
+    still exceeds ``ceiling``, the structural-overhead backstop). Post-condition:
+    ``len(json.dumps(clipped, ensure_ascii=False)) <= ceiling`` always."""
+    truncated = False
+    remaining = [ceiling]   # boxed so the nested walk can decrement it
+
+    def walk(v):
+        nonlocal truncated
+        if isinstance(v, str):
+            if len(v) > remaining[0]:
+                truncated = True
+                v = v[:remaining[0]]
+            remaining[0] -= len(v)
+            return v
+        if isinstance(v, dict):
+            return {k: walk(x) for k, x in v.items()}
+        if isinstance(v, list):
+            return [walk(x) for x in v]
+        return v
+
+    clipped = walk(inp)
+    # Backstop: structural JSON overhead (braces/quotes/keys) can push a
+    # budget-exact payload a few chars past the ceiling. Hard-clip the largest
+    # remaining string leaf(s) until the whole dict serializes within the ceiling.
+    while len(_json.dumps(clipped, ensure_ascii=False)) > ceiling:
+        truncated = True
+        if not _shrink_largest_leaf(clipped):
+            break   # no string leaf left to shrink (e.g. pure numeric/structural)
+    return clipped, truncated
+
+
+def _shrink_largest_leaf(obj):
+    """Halve the longest string leaf reachable in ``obj`` (a dict/list/scalar),
+    in place, and return True if one was shrunk — the post-walk backstop for the
+    rare structural-overhead overshoot. A leaf already at length 1 is truncated to
+    ''. Returns False when no non-empty string leaf exists."""
+    best = {"len": 0, "container": None, "key": None}
+
+    def scan(v, container, key):
+        if isinstance(v, str):
+            if len(v) > best["len"]:
+                best.update(len=len(v), container=container, key=key)
+        elif isinstance(v, dict):
+            for k, x in v.items():
+                scan(x, v, k)
+        elif isinstance(v, list):
+            for i, x in enumerate(v):
+                scan(x, v, i)
+
+    scan(obj, None, None)
+    if best["container"] is None or best["len"] == 0:
+        return False
+    s = best["container"][best["key"]]
+    best["container"][best["key"]] = s[: len(s) // 2]
+    return True
+
+
+def read_full_payload(source_path, byte_offset, tool_use_id, which):
+    """Re-read the raw JSONL line at ``(source_path, byte_offset)`` and return the
+    FULL (un-capped) payload for ``tool_use_id``:
+
+    - ``which='input'`` -> ``{"which":"input", "tool_use_id", "input", "full_length",
+      "truncated"}`` — the matching tool_use block's complete ``input`` dict, so the
+      DiffCard can pull old/new strings straight into computeDiff.
+    - ``which='result'`` -> ``{"which":"result", "tool_use_id", "text", "full_length",
+      "truncated", "is_error", [stderr]}`` — the full ``_stringify(content)`` plus,
+      for Bash, the full ``toolUseResult.stderr``.
+
+    ``None`` when the source is gone / the line is unparseable (rotated or deleted
+    JSONL — the documented 410 path) or the id is no longer present in that line.
+    ``full_length``/``truncated`` describe the payload against ``_FULL_PAYLOAD_CEILING``
+    — honoring #178's "un-capped" spirit for real payloads while bounding the
+    degenerate multi-MB case."""
+    try:
+        with open(source_path, "rb") as fh:
+            fh.seek(byte_offset)
+            line = fh.readline()
+        obj = _json.loads(line)
+    except (OSError, ValueError):
+        return None
+    if not isinstance(obj, dict):
+        return None
+    content = (obj.get("message") or {}).get("content")
+    if not isinstance(content, list):
+        return None
+    if which == "input":
+        for b in content:
+            if (isinstance(b, dict) and b.get("type") == "tool_use"
+                    and b.get("id") == tool_use_id):
+                inp = b.get("input") if isinstance(b.get("input"), dict) else {}
+                full_length = len(_json.dumps(inp, ensure_ascii=False))
+                clipped, truncated = _clip_payload_input(inp, _FULL_PAYLOAD_CEILING)
+                return {"which": "input", "tool_use_id": tool_use_id,
+                        "input": clipped, "full_length": full_length,
+                        "truncated": truncated}
+        return None
+    for b in content:
+        if (isinstance(b, dict) and b.get("type") == "tool_result"
+                and b.get("tool_use_id") == tool_use_id):
+            raw = _stringify(b.get("content"))
+            # The bound is PER-STREAM by design: `text` and the Bash `stderr`
+            # below are each clipped to _FULL_PAYLOAD_CEILING independently, so a
+            # result carrying both can serialize to ~2× the ceiling. That is
+            # intentional — they are distinct streams the DiffCard renders side by
+            # side, and each is individually bounded against the HTTP/browser DoS.
+            resp = {"which": "result", "tool_use_id": tool_use_id,
+                    "text": raw[:_FULL_PAYLOAD_CEILING], "full_length": len(raw),
+                    "truncated": len(raw) > _FULL_PAYLOAD_CEILING,
+                    "is_error": bool(b.get("is_error"))}
+            tur = obj.get("toolUseResult")
+            if (isinstance(tur, dict) and isinstance(tur.get("stderr"), str)
+                    and tur.get("stderr")):
+                resp["stderr"] = tur["stderr"][:_FULL_PAYLOAD_CEILING]   # per-stream bound (see above)
+            return resp
+    return None

package/bin/_lib_jsonl.py

@@ -85,8 +85,10 @@ def _should_replace(
     finalization row carries `speed`; streaming intermediates don't).
 
     The `usage.get("speed") is not None` check matches the SQL UPDATE WHERE
-    clause's `json_extract(..., '$.speed') IS NOT NULL` in `sync_cache`'s
-    INSERT … ON CONFLICT … DO UPDATE, keeping the direct-parse fallback and
+    clause's `excluded.speed IS NOT NULL` in `sync_cache`'s INSERT … ON
+    CONFLICT … DO UPDATE — `speed` is materialized into its own
+    `session_entries.speed` column (#181), so the tiebreak no longer
+    `json_extract`s the blob — keeping the direct-parse fallback and
     cache-ingest paths in lockstep on the rare-but-possible "explicit JSON
     null" payload.
     """

package/bin/cctally

@@ -1614,13 +1614,14 @@ def _usage_entry_from_joined(je) -> "UsageEntry":
     and the per-token integers; this adapter is pure shape conversion
     with no cache re-read.
 
-    Non-token ``usage`` extras (``je.usage_extra``) — notably ``speed`` —
+    Non-token ``usage`` extras (``je.usage_extra``) — now just ``speed`` —
     are merged AFTER the four token keys, mirroring ``iter_entries``'
-    ``usage.update(json.loads(...))``. Without this, the project-axis
+    reconstruction of ``usage["speed"]`` from the materialized
+    ``session_entries.speed`` column (#181). Without this, the project-axis
     ``daily`` path (and the diff/report joined-entry consumers) would
     drop the fast-tier flag and render ``<model>`` where the normal path
-    renders ``<model>-fast``. The write-side strips the four token keys
-    from ``usage_extra_json`` so the merge never shadows the integers.
+    renders ``<model>-fast``. ``je.usage_extra`` is ``{"speed": …}`` or
+    ``None``, so the merge never shadows the token integers.
     """
     usage = {
         "input_tokens": je.input_tokens,