cctally 1.37.2 → 1.38.0

package/CHANGELOG.md

@@ -5,6 +5,13 @@ based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 
 ## [Unreleased]
 
+## [1.38.0] - 2026-06-12
+
+### Added
+- Conversation viewer: Edit, MultiEdit, and Write now render as a unified red/green word-diff card — a `+N −M` stat header with the file's basename and dir, intra-line word-level highlighting on the changed text, per-language syntax coloring on the unchanged context lines, one hunk per edit for MultiEdit, and an all-added "wrote N lines" view for Write — with a collapsed `result · cat -n` sub-panel beneath carrying the real file line numbers, instead of a raw JSON tool chip (#177 S3).
+- Conversation viewer: Bash now renders as a terminal — a `$ <command>` prompt over the output, with `stderr` split into a red block, an `error`/`interrupted` status badge, and ANSI foreground colors honored — instead of an undifferentiated output blob; legacy transcripts captured before the stream split degrade to a merged terminal view (#177 S3).
+- Conversation viewer: a truncated tool input or result can now be expanded on demand — a "load full input"/"load full output" affordance re-reads the original session JSONL behind the existing transcript privacy gate to recompute the diff or re-render the terminal, without enlarging the on-disk cache; a rotated or deleted source surfaces a clear "source no longer available" note (#178).
+
 ## [1.37.2] - 2026-06-12
 
 ### Fixed

package/bin/_cctally_dashboard.py

@@ -5341,6 +5341,10 @@ class DashboardHTTPHandler(BaseHTTPRequestHandler):
             self._handle_get_conversations()
         elif path == "/api/conversation/search":
             self._handle_get_conversation_search()
+        elif path.startswith("/api/conversation/") and path.endswith("/payload"):
+            # #178: on-demand load-full. Matched BEFORE the <id> reader
+            # catch-all (same precedence as /api/conversation/search).
+            self._handle_get_conversation_payload(path)
         elif path.startswith("/api/conversation/"):
             self._handle_get_conversation_detail(path)
         else:
@@ -7344,6 +7348,46 @@ class DashboardHTTPHandler(BaseHTTPRequestHandler):
             return
         self._respond_json(200, body)
 
+    def _handle_get_conversation_payload(self, path: str) -> None:
+        """``GET /api/conversation/<sid>/payload?tool_use_id=<id>&which=<result|input>``
+        — the #178 on-demand load-full route. Re-reads the source JSONL line so
+        a clipped result/input can be expanded without enlarging the cache.
+
+        Gated FIRST by the same loopback/Host transcript privacy predicate the
+        three other conversation routes use (fail-closed 403). ``locate_tool_payload``
+        runs against cache.db (via the shared 500-envelope scaffold); the actual
+        full body is re-read from disk by ``read_full_payload`` (no cache conn).
+        ``which`` is validated to ``result``/``input`` (else 400); a missing
+        tool_use_id is 400; an unknown id is 404; a gone/unparseable source line
+        is 410 (the documented consequence of storing only capped text).
+        """
+        if not self._require_transcripts_allowed():
+            return
+        import urllib.parse as _u
+        session_id = _u.unquote(
+            path[len("/api/conversation/"):-len("/payload")])
+        q = _u.parse_qs(self.path.partition("?")[2])
+        tool_use_id = _qs_str(q, "tool_use_id", "")
+        which = _qs_str(q, "which", "result")
+        if not session_id or which not in ("result", "input") or not tool_use_id:
+            self._respond_json(400, {"error": "bad request"})
+            return
+        cq = self._conversation_query()
+        ok, loc = self._run_conversation_query(
+            lambda conn: cq.locate_tool_payload(
+                conn, session_id, tool_use_id, which),
+            "/api/conversation/payload")
+        if not ok:
+            return
+        if loc is None:
+            self._respond_json(404, {"error": "not found"})
+            return
+        payload = cq.read_full_payload(loc[0], loc[1], tool_use_id, which)
+        if payload is None:
+            self._respond_json(410, {"error": "source no longer available"})
+            return
+        self._respond_json(200, payload)
+
     def _handle_get_project_detail(self) -> None:
         """Return ProjectDetail JSON for ``GET /api/project/<key>?weeks=N``
         (spec §5.3 / §6.5).

package/bin/_lib_conversation.py

@@ -106,6 +106,7 @@ def _normalize(obj, t, offset):
         entry_type = TOOL_RESULT
         _attach_subagent_result(blocks, obj)   # #166: record-level toolUseResult
         _attach_ask_answers(blocks, obj)       # #177 S2: AskUserQuestion answers
+        _attach_bash_streams(blocks, obj)      # #177 S3: Bash stderr/interrupted
         _attach_task_meta(blocks, obj)         # task checklist identity
         # tool_result rows are stored but NOT indexed as prose (spec §2). A
         # user line that mixes a text block with a tool_result block must not
@@ -289,6 +290,35 @@ def _attach_ask_answers(blocks, obj):
         results[0]["ask_annotations"] = bounded_anno
 
 
+def _attach_bash_streams(blocks, obj):
+    """Stash a Bash tool_result's structured stderr + interrupted onto its single
+    tool_result block (#177 S3). Self-identifying: fires only when toolUseResult
+    is a dict carrying a ``stdout``/``stderr`` key — a shape distinctive to Bash —
+    so no cross-record tool-name lookup is needed (same posture as
+    _attach_ask_answers' ``answers`` gate). We do NOT store stdout: the existing
+    ``result.text`` already equals stdout+stderr (the merged Bash output), so
+    storing stdout would roughly double the at-rest payload; the stdout/stderr
+    split is derived client-side by stripping the stderr suffix.
+
+    Parser-private keys ``bash_stderr`` / ``bash_interrupted`` are popped in the
+    query layer's Phase 1 so they never leak into emitted/orphan blocks. stderr is
+    bounded with the same cap as result.text (_TOOL_RESULT_CAP). Empty stderr +
+    not-interrupted is a no-op (the common case — stderr is empty in ~99% of
+    results), keeping the additive contract: absent on every non-Bash result and
+    on old rows. Same exactly-one-result-block guard as _attach_subagent_result."""
+    tur = obj.get("toolUseResult")
+    if not isinstance(tur, dict) or ("stdout" not in tur and "stderr" not in tur):
+        return
+    results = [b for b in blocks if b.get("kind") == "tool_result"]
+    if len(results) != 1:
+        return
+    stderr = tur.get("stderr")
+    if isinstance(stderr, str) and stderr:
+        results[0]["bash_stderr"] = stderr[:_TOOL_RESULT_CAP]
+    if bool(tur.get("interrupted")):
+        results[0]["bash_interrupted"] = True
+
+
 # Subagent Task tools record toolUseResult=null and put the identity in the
 # human-readable result text instead; the id is the only thing the fold needs
 # from the result (subject/status come from the call input). Anchored to the

package/bin/_lib_conversation_query.py

@@ -20,6 +20,10 @@ import sqlite3
 # mirror — imported by the dashboard's conversation endpoints at runtime.
 
 from _lib_pricing import _calculate_entry_cost
+# #178: the on-demand load-full re-read helper re-stringifies a raw tool_result
+# content block the same way the parser does at ingest — reuse the parser's
+# _stringify so the full (un-capped) result text matches the cached/capped one.
+from _lib_conversation import _stringify
 
 
 # Mirror of dashboard/web/src/conversations/systemMarkers.ts::MARKER_RE — anchored
@@ -484,6 +488,7 @@ def get_conversation(conn, session_id, *, after=None, limit=500):
     spawn_kind = {}     # tool_use id -> subagent_type
     agent_link = {}     # tool_use id -> (agent_id, raw_meta)
     ask_link = {}       # tool_use id -> (answers, annotations)  (#177 S2)
+    bash_link = {}      # tool_use id -> (stderr, interrupted)   (#177 S3)
     task_link = {}      # tool_use id -> {"task_id", "task_list"}  (Task* checklist)
     for it in items:
         for b in it["blocks"]:
@@ -501,6 +506,10 @@ def get_conversation(conn, session_id, *, after=None, limit=500):
                 anno = b.pop("ask_annotations", None)
                 if ans is not None and b.get("tool_use_id") is not None:
                     ask_link[b["tool_use_id"]] = (ans, anno)
+                bstderr = b.pop("bash_stderr", None)        # #177 S3
+                bintr = b.pop("bash_interrupted", None)
+                if b.get("tool_use_id") is not None and (bstderr is not None or bintr):
+                    bash_link[b["tool_use_id"]] = (bstderr, bool(bintr))
                 tid_ = b.pop("task_id", None)               # Task* checklist
                 tlist_ = b.pop("task_list", None)
                 if b.get("tool_use_id") is not None and (tid_ is not None or tlist_ is not None):
@@ -568,6 +577,12 @@ def get_conversation(conn, session_id, *, after=None, limit=500):
                         b["answers"] = link[0]
                         if link[1]:
                             b["annotations"] = link[1]
+                    blink = bash_link.get(b["tool_use_id"])  # #177 S3
+                    if blink is not None:
+                        if blink[0] is not None:
+                            b["stderr"] = blink[0]
+                        if blink[1]:
+                            b["interrupted"] = True
 
     # ---- Phase 3b: fold the Task* op stream into per-run checklist snapshots ----
     _fold_task_runs(items, task_link)
@@ -1127,3 +1142,178 @@ def _manual_snippet(text, q, width=80):
     end = min(len(text), lo + len(q) + width // 2)
     s = text[start:end]
     return ("… " if start else "") + s + (" …" if end < len(text) else "")
+
+
+# ---------------------------------------------------------------------------
+# #178: on-demand "load full result/input" kernels. Back the dashboard's
+# /api/conversation/<sid>/payload route. The cache stores only CAPPED tool text
+# (result clipped to _TOOL_RESULT_CAP, input leaves clipped to _INPUT_LEAF_CAP),
+# so the full body is re-derived from the source JSONL line here — the cache at
+# rest never grows. Pure (sqlite3.Connection + filesystem); no clock/network.
+# ---------------------------------------------------------------------------
+_FULL_PAYLOAD_CEILING = 1_000_000   # serve up to ~1 MB; protects the HTTP server / browser
+
+
+def locate_tool_payload(conn, session_id, tool_use_id, which):
+    """``(source_path, byte_offset)`` for the JSONL line holding the tool_use
+    (``which='input'``) or tool_result (``which='result'``) carrying this
+    ``tool_use_id`` in this session, else ``None``.
+
+    The prefilter uses ``instr(blocks_json, ?) > 0`` — NOT ``LIKE`` (Codex P1.4):
+    tool_use_ids contain ``_`` (e.g. ``toolu_01SEQ…``), which ``LIKE`` treats as a
+    single-char wildcard, so a near-miss id would false-match. ``instr`` is a
+    literal substring test. It is also NOT a ``≤2 rows`` situation — the DB holds
+    duplicate physical rows for one logical message (``get_conversation`` dedups
+    by uuid) — so every candidate is parsed and EXACT-matched on the block id
+    (``tool_use.id`` for input / ``tool_result.tool_use_id`` for result), under the
+    same deterministic ``ORDER BY timestamp_utc, id`` as ``get_conversation``. The
+    SELECT runs here (not via ``get_conversation``) because that reader omits
+    ``byte_offset`` (Codex P2.5)."""
+    rows = conn.execute(
+        "SELECT source_path, byte_offset, blocks_json FROM conversation_messages "
+        "WHERE session_id=? AND instr(blocks_json, ?) > 0 "
+        "ORDER BY timestamp_utc, id", (session_id, tool_use_id)).fetchall()
+    for source_path, byte_offset, blocks_json in rows:
+        try:
+            blocks = _json.loads(blocks_json)
+        except (ValueError, TypeError):
+            continue
+        if not isinstance(blocks, list):
+            continue
+        for b in blocks:
+            if not isinstance(b, dict):
+                continue
+            k = b.get("kind")
+            if which == "input" and k == "tool_use" and b.get("id") == tool_use_id:
+                return source_path, byte_offset
+            if which == "result" and k == "tool_result" and b.get("tool_use_id") == tool_use_id:
+                return source_path, byte_offset
+    return None
+
+
+def _clip_payload_input(inp, ceiling):
+    """Clip a structured input so the returned dict serializes to ``ceiling`` chars
+    or fewer, and report whether anything was clipped — the input-side analogue of
+    the result-side ceiling. A degenerate multi-MB input (one giant leaf OR many
+    sub-ceiling leaves that sum past the ceiling) is bounded so the HTTP server /
+    browser is protected, while every real payload returns whole.
+
+    The guarantee is AGGREGATE, not merely per-leaf: a shared remaining-char budget
+    is threaded through the walk (mirroring ``_bound_input``'s total-size backstop) —
+    each string leaf is clipped against the running budget and the budget is
+    decremented as we go, so once it is exhausted later leaves clip to ''.
+    ``truncated`` is True iff any leaf was clipped (or the post-walk serialized size
+    still exceeds ``ceiling``, the structural-overhead backstop). Post-condition:
+    ``len(json.dumps(clipped, ensure_ascii=False)) <= ceiling`` always."""
+    truncated = False
+    remaining = [ceiling]   # boxed so the nested walk can decrement it
+
+    def walk(v):
+        nonlocal truncated
+        if isinstance(v, str):
+            if len(v) > remaining[0]:
+                truncated = True
+                v = v[:remaining[0]]
+            remaining[0] -= len(v)
+            return v
+        if isinstance(v, dict):
+            return {k: walk(x) for k, x in v.items()}
+        if isinstance(v, list):
+            return [walk(x) for x in v]
+        return v
+
+    clipped = walk(inp)
+    # Backstop: structural JSON overhead (braces/quotes/keys) can push a
+    # budget-exact payload a few chars past the ceiling. Hard-clip the largest
+    # remaining string leaf(s) until the whole dict serializes within the ceiling.
+    while len(_json.dumps(clipped, ensure_ascii=False)) > ceiling:
+        truncated = True
+        if not _shrink_largest_leaf(clipped):
+            break   # no string leaf left to shrink (e.g. pure numeric/structural)
+    return clipped, truncated
+
+
+def _shrink_largest_leaf(obj):
+    """Halve the longest string leaf reachable in ``obj`` (a dict/list/scalar),
+    in place, and return True if one was shrunk — the post-walk backstop for the
+    rare structural-overhead overshoot. A leaf already at length 1 is truncated to
+    ''. Returns False when no non-empty string leaf exists."""
+    best = {"len": 0, "container": None, "key": None}
+
+    def scan(v, container, key):
+        if isinstance(v, str):
+            if len(v) > best["len"]:
+                best.update(len=len(v), container=container, key=key)
+        elif isinstance(v, dict):
+            for k, x in v.items():
+                scan(x, v, k)
+        elif isinstance(v, list):
+            for i, x in enumerate(v):
+                scan(x, v, i)
+
+    scan(obj, None, None)
+    if best["container"] is None or best["len"] == 0:
+        return False
+    s = best["container"][best["key"]]
+    best["container"][best["key"]] = s[: len(s) // 2]
+    return True
+
+
+def read_full_payload(source_path, byte_offset, tool_use_id, which):
+    """Re-read the raw JSONL line at ``(source_path, byte_offset)`` and return the
+    FULL (un-capped) payload for ``tool_use_id``:
+
+    - ``which='input'`` -> ``{"which":"input", "tool_use_id", "input", "full_length",
+      "truncated"}`` — the matching tool_use block's complete ``input`` dict, so the
+      DiffCard can pull old/new strings straight into computeDiff.
+    - ``which='result'`` -> ``{"which":"result", "tool_use_id", "text", "full_length",
+      "truncated", "is_error", [stderr]}`` — the full ``_stringify(content)`` plus,
+      for Bash, the full ``toolUseResult.stderr``.
+
+    ``None`` when the source is gone / the line is unparseable (rotated or deleted
+    JSONL — the documented 410 path) or the id is no longer present in that line.
+    ``full_length``/``truncated`` describe the payload against ``_FULL_PAYLOAD_CEILING``
+    — honoring #178's "un-capped" spirit for real payloads while bounding the
+    degenerate multi-MB case."""
+    try:
+        with open(source_path, "rb") as fh:
+            fh.seek(byte_offset)
+            line = fh.readline()
+        obj = _json.loads(line)
+    except (OSError, ValueError):
+        return None
+    if not isinstance(obj, dict):
+        return None
+    content = (obj.get("message") or {}).get("content")
+    if not isinstance(content, list):
+        return None
+    if which == "input":
+        for b in content:
+            if (isinstance(b, dict) and b.get("type") == "tool_use"
+                    and b.get("id") == tool_use_id):
+                inp = b.get("input") if isinstance(b.get("input"), dict) else {}
+                full_length = len(_json.dumps(inp, ensure_ascii=False))
+                clipped, truncated = _clip_payload_input(inp, _FULL_PAYLOAD_CEILING)
+                return {"which": "input", "tool_use_id": tool_use_id,
+                        "input": clipped, "full_length": full_length,
+                        "truncated": truncated}
+        return None
+    for b in content:
+        if (isinstance(b, dict) and b.get("type") == "tool_result"
+                and b.get("tool_use_id") == tool_use_id):
+            raw = _stringify(b.get("content"))
+            # The bound is PER-STREAM by design: `text` and the Bash `stderr`
+            # below are each clipped to _FULL_PAYLOAD_CEILING independently, so a
+            # result carrying both can serialize to ~2× the ceiling. That is
+            # intentional — they are distinct streams the DiffCard renders side by
+            # side, and each is individually bounded against the HTTP/browser DoS.
+            resp = {"which": "result", "tool_use_id": tool_use_id,
+                    "text": raw[:_FULL_PAYLOAD_CEILING], "full_length": len(raw),
+                    "truncated": len(raw) > _FULL_PAYLOAD_CEILING,
+                    "is_error": bool(b.get("is_error"))}
+            tur = obj.get("toolUseResult")
+            if (isinstance(tur, dict) and isinstance(tur.get("stderr"), str)
+                    and tur.get("stderr")):
+                resp["stderr"] = tur["stderr"][:_FULL_PAYLOAD_CEILING]   # per-stream bound (see above)
+            return resp
+    return None