cctally 1.52.1 → 1.53.0

package/bin/_lib_conversation.py

@@ -214,12 +214,12 @@ class MessageRow:
     attribution_skill: "str | None" = None
     attribution_plugin: "str | None" = None
     # #177 S6: split non-prose search columns, derived by _derive_search_columns
-    # on the FINAL post-augment blocks (see _normalize). ``search_aux`` is kept
-    # physically (legacy column) but NEVER assigned a non-empty value — the
-    # consolidated multi-column FTS reads search_tool/search_thinking instead.
+    # on the FINAL post-augment blocks (see _normalize). The consolidated
+    # multi-column FTS reads search_tool/search_thinking. (#217 S1 / U7a: the
+    # legacy ``search_aux`` field was removed — the column is dropped from the
+    # live schema by migration 016 and the writer never carried a value anyway.)
     search_tool: str = ""
     search_thinking: str = ""
-    search_aux: str = ""   # documented-dead (#177 S6); always "" on new rows
 
 
 def iter_message_rows(fh, path_str):
@@ -462,16 +462,15 @@ def _normalize(obj, t, offset):
         is_sidechain=1 if obj.get("isSidechain") else 0,
         source_tool_use_id=obj.get("sourceToolUseID"),
         # #177: message-level enrichment. stop_reason is assistant-only;
-        # attribution is top-level on the JSONL object. search_aux is kept even
-        # for tool_result/meta rows — only `text` is zeroed for prose FTS;
-        # search_aux is the non-prose index (tool content stays searchable).
+        # attribution is top-level on the JSONL object. The non-prose index is
+        # the split search_tool/search_thinking columns — tool/thinking content
+        # stays searchable even though `text` is zeroed for prose FTS on
+        # tool_result/meta rows.
         stop_reason=msg.get("stop_reason") if is_asst else None,
         attribution_skill=obj.get("attributionSkill"),
         attribution_plugin=obj.get("attributionPlugin"),
         search_tool=search_tool,
         search_thinking=search_thinking,
-        # search_aux stays "" (documented-dead, #177 S6) — the consolidated
-        # multi-column FTS reads search_tool/search_thinking instead.
     )
 
 
@@ -536,6 +535,18 @@ def _blocks_and_text(content):
                          "full_length": len(raw),
                          "is_error": bool(b.get("is_error")),
                          "tool_use_id": b.get("tool_use_id")}
+                # #217 S1 / U6: a nested (grandchild) subagent result carries its
+                # `agentId:` (+ optional <usage>) in the result STRING, not a
+                # record-level toolUseResult. Parse them from the FULL raw HERE,
+                # BEFORE the cap clips the text — so a >16 KB result whose agentId:
+                # trailer lands past the cut still links (the read-time regex over
+                # the clipped text would miss it). Stamp the structured keys the
+                # kernel's id consumer already reads (b.pop("agent_id")); the
+                # read-time regex stays a fallback for un-reingested rows.
+                _nested = _nested_agent_stamp_from_text(raw)
+                if _nested is not None:
+                    block["agent_id"] = _nested[0]
+                    block["subagent_meta"] = _nested[1]
                 # #177 S4: media placeholders for image/document items inside the
                 # tool_result content array (where every MCP screenshot lives) —
                 # ordinals from the shared iter_media_items chokepoint.
@@ -561,6 +572,45 @@ _SUBAGENT_META_KEYS = (
 )
 
 
+# #217 S1 / U6 — nested (grandchild) subagent result parse. The new Claude Code
+# format emits a grandchild's spawn result as STRING content (no structured
+# record-level toolUseResult.agentId), with a trailing `agentId: <hash> (use
+# SendMessage …)` line and an OPTIONAL `<usage>` totals wrapper. These constants
+# live HERE (the parser) so the INGEST stamp runs them against the FULL raw
+# content BEFORE the _TOOL_RESULT_CAP clip — a result whose agentId: trailer
+# lands past the 16 KB cut would otherwise survive only as flat text and the
+# read-time regex (over the clipped text) would miss it. _lib_conversation_query
+# re-exports both for its read-time fallback over un-reingested rows.
+_NESTED_AGENT_ID_RE = re.compile(r"agentId:\s*([0-9a-f]+)")
+_NESTED_USAGE_RE = re.compile(
+    r"<usage>\s*subagent_tokens:\s*(\d+)\s*tool_uses:\s*(\d+)\s*duration_ms:\s*(\d+)\s*</usage>",
+    re.DOTALL,
+)
+
+
+def _nested_agent_stamp_from_text(raw):
+    """(agent_id, subagent_meta) parsed from a tool_result's FULL string content,
+    or None when no `agentId:` line is present (an ordinary result, NOT a spawn
+    grandchild). subagent_meta carries the completed `<usage>` totals when the
+    wrapper is present + well-formed, else {} — linking on agentId ALONE is
+    sufficient, so a missing/clipped/malformed usage degrades usage to {}, never
+    the whole link. Keys mirror the snake_case shape _attach_subagent_result /
+    the query kernel's structured-id consumer expect (Codex P1-B)."""
+    if not raw:
+        return None
+    m = _NESTED_AGENT_ID_RE.search(raw)
+    if not m:
+        return None
+    meta = {}
+    u = _NESTED_USAGE_RE.search(raw)
+    if u:
+        meta["total_tokens"] = int(u.group(1))
+        meta["total_tool_use_count"] = int(u.group(2))
+        meta["total_duration_ms"] = int(u.group(3))
+        meta["status"] = "completed"
+    return m.group(1), meta
+
+
 def _attach_subagent_result(blocks, obj):
     """Attach the record-level ``toolUseResult`` agentId + meta (#166) onto the
     tool_result block, but ONLY when the record carries exactly one tool_result
@@ -580,6 +630,9 @@ def _attach_subagent_result(blocks, obj):
     if len(results) != 1:
         return
     block = results[0]
+    # The record-level toolUseResult is the AUTHORITATIVE source — it overwrites
+    # any nested-text stamp (#217 S1 / U6) for the same id (the two shapes do not
+    # co-occur in real data, so this is only a deterministic-precedence guard).
     block["agent_id"] = agent_id
     meta = {}
     for src, dst in _SUBAGENT_META_KEYS:
@@ -977,11 +1030,142 @@ def _bound_input(inp):
         return v
 
     bounded = walk(inp, 0)
+    # #217 S1 / U5 (data-contract honesty): the total-cap backstop must actually
+    # CLIP, not merely flag. The four per-axis caps above bound any SINGLE leaf /
+    # key / node-count / depth, but many sub-leaf-cap leaves can still sum past
+    # _INPUT_TOTAL_CAP — and the old code only set truncated=True there, serving
+    # an over-cap payload. _clip_payload_input threads a shared remaining-WIRE-char
+    # budget through the (already per-axis-bounded) dict — measured in the SAME
+    # json.dumps(separators=(",",":")) wire form this check uses (and blocks_json
+    # stores) — so the SERVED payload satisfies
+    # len(json.dumps(bounded, separators=(",",":"))) <= _INPUT_TOTAL_CAP whenever it
+    # clips. Non-ASCII leaves escape to \uXXXX (6 wire chars/char), so a plain char
+    # budget would clip ~6x too late on the wire (round-2 P1 fix).
     if len(json.dumps(bounded, separators=(",", ":"))) > _INPUT_TOTAL_CAP:
+        bounded, _ = _clip_payload_input(bounded, _INPUT_TOTAL_CAP)
         state["truncated"] = True
     return (bounded, state["truncated"])
 
 
+# #217 S1 / U5: the payload-clip helpers live HERE (the parser/ingest module, low
+# in the dependency graph) so BOTH _bound_input's total-cap backstop (above) and
+# read_full_payload's on-demand input clip (the query kernel, which imports this
+# module) share one bound-guaranteeing implementation. The query layer never
+# imports back into the parser's callers, so this is the import-safe home (query
+# depends on the parser, not vice-versa — verified at #217 implementation).
+def _wire_len(s):
+    """Serialized char-length of a string leaf in the WIRE form — the SAME
+    ``json.dumps(separators=(",",":"))`` (default ``ensure_ascii=True``) used both
+    by ``_bound_input``'s total-cap check and the ``blocks_json`` storage at the
+    top of this module. For ASCII this equals ``len(s)``; for non-ASCII each char
+    escapes to ``\\uXXXX`` (6 chars) so the wire cost is larger — measuring against
+    it (not ``len(s)``) is what makes the clip's post-condition hold on the wire.
+    The surrounding quotes are part of the per-leaf structural cost the post-walk
+    backstop absorbs, so this counts only the escaped CONTENT (no quotes)."""
+    # json.dumps wraps the string in quotes; strip them to get the content cost.
+    return len(json.dumps(s, separators=(",", ":"))) - 2
+
+
+def _clip_string_to_wire_budget(s, budget):
+    """Return the longest prefix of ``s`` whose WIRE length is ``<= budget`` (>= 0).
+    ASCII is a fast 1:1 slice; for non-ASCII we walk char-by-char accumulating the
+    per-char wire cost so a 6-chars-per-char escape never overshoots the budget."""
+    if budget <= 0:
+        return ""
+    if s.isascii():
+        return s[:budget]
+    out, used = [], 0
+    for ch in s:
+        c = _wire_len(ch)
+        if used + c > budget:
+            break
+        out.append(ch)
+        used += c
+    return "".join(out)
+
+
+def _clip_payload_input(inp, ceiling):
+    """Clip a structured input so the returned dict serializes to ``ceiling`` chars
+    or fewer IN THE WIRE FORM, and report whether anything was clipped — the
+    input-side analogue of the result-side ceiling. A degenerate multi-MB input
+    (one giant leaf OR many sub-ceiling leaves that sum past the ceiling) is bounded
+    so the HTTP server / browser is protected, while every real payload returns
+    whole.
+
+    The guarantee is AGGREGATE, not merely per-leaf: a shared remaining-WIRE-char
+    budget is threaded through the walk (mirroring ``_bound_input``'s total-size
+    backstop) — each string leaf is clipped against the running budget, measured in
+    the wire form, and the budget is decremented by the leaf's wire cost as we go,
+    so once it is exhausted later leaves clip to ''. ``truncated`` is True iff any
+    leaf was clipped (or the post-walk serialized size still exceeds ``ceiling``,
+    the structural-overhead backstop).
+
+    The measurement is the WIRE form ``json.dumps(separators=(",",":"))`` (default
+    ``ensure_ascii=True``) — the SAME serialization ``_bound_input`` checks against
+    and the ``blocks_json`` storage uses — NOT ``ensure_ascii=False``. For non-ASCII
+    payloads each char escapes to ``\\uXXXX`` (6 wire chars), so an ``ensure_ascii=
+    False`` measurement clips ~6x too late on the wire; measuring in the wire form
+    is what makes the post-condition real. Post-condition:
+    ``len(json.dumps(clipped, separators=(",",":"))) <= ceiling`` always."""
+    truncated = False
+    remaining = [ceiling]   # boxed so the nested walk can decrement it (WIRE chars)
+
+    def walk(v):
+        nonlocal truncated
+        if isinstance(v, str):
+            if _wire_len(v) > remaining[0]:
+                truncated = True
+                v = _clip_string_to_wire_budget(v, remaining[0])
+            remaining[0] -= _wire_len(v)
+            return v
+        if isinstance(v, dict):
+            return {k: walk(x) for k, x in v.items()}
+        if isinstance(v, list):
+            return [walk(x) for x in v]
+        return v
+
+    clipped = walk(inp)
+    # Backstop: structural JSON overhead (braces/quotes/keys) can push a
+    # budget-exact payload a few chars past the ceiling. Hard-clip the largest
+    # remaining string leaf(s) until the whole dict serializes within the ceiling —
+    # measured in the wire form so the loop terminates against the same size the
+    # post-condition guards.
+    while len(json.dumps(clipped, separators=(",", ":"))) > ceiling:
+        truncated = True
+        if not _shrink_largest_leaf(clipped):
+            break   # no string leaf left to shrink (e.g. pure numeric/structural)
+    return clipped, truncated
+
+
+def _shrink_largest_leaf(obj):
+    """Halve the wire-longest string leaf reachable in ``obj`` (a dict/list/scalar),
+    in place, and return True if one was shrunk — the post-walk backstop for the
+    rare structural-overhead overshoot. "Longest" is measured in the WIRE form (so
+    a non-ASCII leaf, 6 wire chars per char, is correctly preferred over a longer
+    ASCII leaf with fewer wire chars). A leaf already at length 1 is truncated to
+    ''. Returns False when no non-empty string leaf exists."""
+    best = {"len": 0, "container": None, "key": None}
+
+    def scan(v, container, key):
+        if isinstance(v, str):
+            wl = _wire_len(v)
+            if wl > best["len"]:
+                best.update(len=wl, container=container, key=key)
+        elif isinstance(v, dict):
+            for k, x in v.items():
+                scan(x, v, k)
+        elif isinstance(v, list):
+            for i, x in enumerate(v):
+                scan(x, v, i)
+
+    scan(obj, None, None)
+    if best["container"] is None or best["len"] == 0:
+        return False
+    s = best["container"][best["key"]]
+    best["container"][best["key"]] = s[: len(s) // 2]
+    return True
+
+
 def _derive_search_columns(blocks):
     """(search_tool, search_thinking) from the FINAL normalized blocks list.
 
@@ -1038,6 +1222,38 @@ def _aux_strings(v):
             yield from _aux_strings(vv)
 
 
+# #217 S2 / I-3: the WRITE-class file tools — the only tools that actually mutate
+# a file on disk. Read / Bash / Grep / Glob are deliberately EXCLUDED: a file
+# "touch" for the file-path search axis means an edit/create, not a read or a
+# shell that merely names a path. The axis answers "which sessions modified this
+# file."
+_FILE_TOUCH_TOOLS = {"Edit", "MultiEdit", "Write", "NotebookEdit"}
+
+
+def _derive_file_touches(blocks):
+    """Yield ``(file_path, tool)`` for every WRITE-class tool_use block (#217 S2 /
+    I-3). Reads the same bounded ``input.file_path`` the renderer reads (file paths
+    are well under the leaf cap, so they survive _bound_input intact).
+
+    Tolerates BOTH block shapes: the STORED ``blocks_json`` form keys the kind on
+    ``kind`` (the _normalize output that _fill_file_touches reparses), while raw
+    API content blocks key it on ``type``. We accept either so the same chokepoint
+    derives touches from the persisted rows AND from a raw-shape unit fixture."""
+    out = []
+    for b in (blocks or []):
+        if not isinstance(b, dict):
+            continue
+        if (b.get("kind") or b.get("type")) != "tool_use":
+            continue
+        if b.get("name") not in _FILE_TOUCH_TOOLS:
+            continue
+        inp = b.get("input")
+        fp = inp.get("file_path") if isinstance(inp, dict) else None
+        if isinstance(fp, str) and fp:
+            out.append((fp, b["name"]))
+    return out
+
+
 _PREVIEW_FIELDS = {
     "Read": "file_path", "Write": "file_path", "Edit": "file_path",
     "MultiEdit": "file_path", "NotebookEdit": "file_path",