cctally 1.34.1 → 1.35.0

package/CHANGELOG.md

@@ -5,6 +5,11 @@ based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 
 ## [Unreleased]
 
+## [1.35.0] - 2026-06-11
+
+### Added
+- Dashboard conversation reader: the transcript data contract now carries structured tool input (with an `input_truncated` flag), tool results with honest length accounting (`full_length` plus a raised result cap), per-turn token usage drawn from the same deduped session-entry row as the cost, and each turn's stop-reason and skill/plugin attribution — all additive (the existing reader renders unchanged), with a parallel tool-content search index built as groundwork. The new fields are re-derived across existing transcripts by a one-time cache migration (#177).
+
 ## [1.34.1] - 2026-06-11
 
 ### Fixed

package/bin/_cctally_cache.py

@@ -183,18 +183,25 @@ _CONV_INSERT_SQL = (
     "INSERT OR IGNORE INTO conversation_messages"
     "(session_id,uuid,parent_uuid,source_path,byte_offset,"
     " timestamp_utc,entry_type,text,blocks_json,model,msg_id,"
-    " req_id,cwd,git_branch,is_sidechain,source_tool_use_id)"
-    " VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
+    " req_id,cwd,git_branch,is_sidechain,source_tool_use_id,"
+    " stop_reason,attribution_skill,attribution_plugin,search_aux)"
+    " VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
 )
 
 
 def _conv_row_tuple(m, path_str):
-    """Flatten a ``MessageRow`` into the ``_CONV_INSERT_SQL`` column order."""
+    """Flatten a ``MessageRow`` into the ``_CONV_INSERT_SQL`` column order.
+
+    The #177 enrichment fields (stop_reason / attribution_skill /
+    attribution_plugin / search_aux) are TAIL-APPENDED after source_tool_use_id
+    — same order as the SQL column list — so both ingest paths (fused per-file
+    walk + backfill_conversation_messages) carry them through this one tuple."""
     return (
         m.session_id, m.uuid, m.parent_uuid, path_str, m.byte_offset,
         m.timestamp_utc, m.entry_type, m.text, m.blocks_json, m.model,
         m.msg_id, m.req_id, m.cwd, m.git_branch, m.is_sidechain,
         m.source_tool_use_id,
+        m.stop_reason, m.attribution_skill, m.attribution_plugin, m.search_aux,
     )
 
 
@@ -622,11 +629,18 @@ def sync_cache(
             # dropping the flag here covers the 004 reingest too. Migration 006
             # sets the DISTINCT conversation_source_tool_use_reingest_pending
             # flag (to land source_tool_use_id); the same offset-0 walk re-derives
-            # it, so drop that flag here as well to avoid a redundant pass.
+            # it, so drop that flag here as well to avoid a redundant pass. #177
+            # migration 007 sets the DISTINCT
+            # conversation_reingest_enrichment_pending flag (to land structured
+            # input / full_length / stop_reason / attribution / search_aux); the
+            # same offset-0 walk re-derives those through the enriched parser, so
+            # drop that flag here too — MISSING this site re-arms the flag on
+            # every cache-sync --rebuild.
             conn.execute(
                 "DELETE FROM cache_meta WHERE key IN "
                 "('conversation_reingest_pending',"
-                " 'conversation_source_tool_use_reingest_pending')")
+                " 'conversation_source_tool_use_reingest_pending',"
+                " 'conversation_reingest_enrichment_pending')")
             conn.commit()
             eprint("[cache-sync] rebuild: cleared Claude cached entries")
 
@@ -687,13 +701,21 @@ def sync_cache(
             # shared one) to land the message-level ``source_tool_use_id`` — the
             # shared flag also gates the kernel's 005 human-fallback, so re-arming
             # it for 006 could misclassify a genuine human prompt during the
-            # pre-reingest window. We trigger the SAME clear + offset-0 backfill on
-            # EITHER flag and clear BOTH atomically here under the held flock.
+            # pre-reingest window. #177 migration 007 uses ANOTHER distinct flag
+            # ``conversation_reingest_enrichment_pending`` (for the same shared-flag
+            # reason) to land the enriched data contract (structured input +
+            # input_truncated, the raised result cap + full_length, stop_reason /
+            # attribution_skill / attribution_plugin, and the search_aux FTS-aux
+            # blob); the offset-0 re-parse through the enriched parser lands them
+            # all with zero new consumption code. We trigger the SAME clear +
+            # offset-0 backfill on ANY of these flags and clear them ALL atomically
+            # here under the held flock.
             try:
                 _reingest = conn.execute(
                     "SELECT 1 FROM cache_meta WHERE key IN "
                     "('conversation_reingest_pending',"
-                    " 'conversation_source_tool_use_reingest_pending')"
+                    " 'conversation_source_tool_use_reingest_pending',"
+                    " 'conversation_reingest_enrichment_pending')"
                 ).fetchone() is not None
             except sqlite3.OperationalError:
                 _reingest = False
@@ -703,7 +725,8 @@ def sync_cache(
                 conn.execute(
                     "DELETE FROM cache_meta WHERE key IN "
                     "('conversation_reingest_pending',"
-                    " 'conversation_source_tool_use_reingest_pending')"
+                    " 'conversation_source_tool_use_reingest_pending',"
+                    " 'conversation_reingest_enrichment_pending')"
                 )
                 conn.commit()
 

package/bin/_cctally_db.py

@@ -2389,6 +2389,15 @@ def _apply_cache_schema(conn: sqlite3.Connection) -> None:
     # column-add (no marker, no version); cache migration 006 then re-ingests
     # so the value actually lands on historical rows.
     add_column_if_missing(conn, "conversation_messages", "source_tool_use_id", "TEXT")
+    # #177 Session 1: enriched data-contract columns. Idempotent column-adds (no
+    # marker, no version — exactly like source_tool_use_id); cache migration 007
+    # then re-ingests so the values actually land on historical rows. search_aux
+    # is the parser-populated non-prose blob the conversation_fts_aux index reads.
+    add_column_if_missing(conn, "conversation_messages", "stop_reason", "TEXT")
+    add_column_if_missing(conn, "conversation_messages", "attribution_skill", "TEXT")
+    add_column_if_missing(conn, "conversation_messages", "attribution_plugin", "TEXT")
+    add_column_if_missing(
+        conn, "conversation_messages", "search_aux", "TEXT NOT NULL DEFAULT ''")
     conn.execute(
         "CREATE INDEX IF NOT EXISTS idx_session_files_session_id "
         "ON session_files(session_id)"
@@ -2414,23 +2423,40 @@ def _apply_cache_schema(conn: sqlite3.Connection) -> None:
             conn.execute(
                 "CREATE VIRTUAL TABLE IF NOT EXISTS conversation_fts "
                 "USING fts5(text, content='conversation_messages', content_rowid='id')")
-            # Trigger DDL lives in ONE place (_CONV_FTS_TRIGGER_DDL) so this
-            # initial create and the #138 storm-free full-clear
-            # (clear_conversation_messages, which drops + recreates the
-            # triggers) can never drift.
+            # #177: the aux external-content index is ALL-OR-NOTHING with the
+            # prose FTS under the SINGLE fts5_unavailable flag — created in this
+            # SAME try-envelope (via the _create_conversation_fts_aux_table seam,
+            # which tests monkeypatch to simulate an aux-create failure AFTER the
+            # prose create succeeded), so ANY OperationalError below drops BOTH
+            # tables + BOTH trigger sets (the except arm). A live aux trigger over
+            # a missing conversation_fts_aux would roll back the shared
+            # conversation_messages write transaction (which also carries cost
+            # ingest into session_entries). It is NOT given its own guarded block
+            # or its own flag.
+            _create_conversation_fts_aux_table(conn)
+            # Trigger DDL lives in ONE place (_CONV_FTS_TRIGGER_DDL +
+            # _CONV_FTS_AUX_TRIGGER_DDL) so this initial create and the #138
+            # storm-free full-clear (clear_conversation_messages, which drops +
+            # recreates BOTH trigger sets) can never drift.
             _create_conversation_fts_triggers(conn)
             if recovering:
-                # Repopulate the freshly-(re)created index from the base table
+                # Repopulate the freshly-(re)created indexes from the base table
                 # so pre-recovery history is searchable. Cheap no-op when
                 # conversation_messages is empty.
                 conn.execute(
                     "INSERT INTO conversation_fts(conversation_fts) VALUES('rebuild')")
+                conn.execute(
+                    "INSERT INTO conversation_fts_aux(conversation_fts_aux) VALUES('rebuild')")
             conn.execute("DELETE FROM cache_meta WHERE key='fts5_unavailable'")
         except sqlite3.OperationalError:
-            # partial create cleanup, then mark unavailable
+            # partial create cleanup, then mark unavailable. _drop drops BOTH
+            # prose + aux trigger sets (#177), and we drop BOTH vtables, so a
+            # failed aux create can't leave a live aux trigger over a missing
+            # table.
             _drop_conversation_fts_triggers(conn)
             try:
                 conn.execute("DROP TABLE IF EXISTS conversation_fts")
+                conn.execute("DROP TABLE IF EXISTS conversation_fts_aux")
             except sqlite3.OperationalError:
                 pass
             _set_cache_meta(conn, "fts5_unavailable", "1")
@@ -2472,6 +2498,28 @@ def _set_cache_meta(conn: sqlite3.Connection, key: str, value: str) -> None:
                  "ON CONFLICT(key) DO UPDATE SET value=excluded.value", (key, value))
 
 
+def _create_conversation_fts_aux_table(conn: sqlite3.Connection) -> None:
+    """Create the #177 aux external-content FTS5 index over the ``search_aux``
+    blob. A standalone module-level seam (NOT inlined in ``_apply_cache_schema``)
+    so the all-or-nothing regression test can monkeypatch it to raise
+    ``OperationalError`` AFTER the prose ``conversation_fts`` create succeeded —
+    proving the shared try-envelope drops BOTH indexes + BOTH trigger sets and a
+    later ``conversation_messages`` INSERT still commits (the cost write txn is
+    not rolled back). Must run inside that envelope; idempotent
+    (``IF NOT EXISTS``)."""
+    # The FTS column name MUST match the content table column (``search_aux``),
+    # exactly as the prose ``conversation_fts`` uses ``text``. An external-content
+    # FTS5 table resolves its columns through the content table by NAME, so a
+    # mismatched column (e.g. ``aux`` against a ``search_aux`` content column)
+    # leaves the index functional for MATCH but breaks any content-backed read of
+    # the column — notably ``sqlite3.Connection.iterdump`` (the dump emits
+    # ``SELECT quote(aux) FROM conversation_fts_aux`` → "no such column"). Aligning
+    # the names keeps the index dumpable and is the documented FTS5 posture.
+    conn.execute(
+        "CREATE VIRTUAL TABLE IF NOT EXISTS conversation_fts_aux "
+        "USING fts5(search_aux, content='conversation_messages', content_rowid='id')")
+
+
 # Conversation FTS sync triggers (external-content FTS5). Defined ONCE here so
 # the initial create in _apply_cache_schema and the #138 storm-free full-clear
 # in clear_conversation_messages (which drops + recreates them) can never drift.
@@ -2491,21 +2539,47 @@ _CONV_FTS_TRIGGER_DDL = (
 # cosmetic — order doesn't matter for independent triggers.
 _CONV_FTS_TRIGGER_NAMES = ("conv_fts_au", "conv_fts_ad", "conv_fts_ai")
 
+# #177: the parallel aux index (conversation_fts_aux) over the search_aux blob.
+# All-or-nothing with the prose FTS under the single fts5_unavailable flag — its
+# DDL lives beside the prose set and the SAME create/drop chokepoints handle
+# both, so the two trigger sets can never drift. The AU trigger is keyed on
+# ``AFTER UPDATE OF search_aux`` (a text-only update doesn't fire it, and the
+# prose AU's ``AFTER UPDATE OF text`` doesn't fire this one).
+_CONV_FTS_AUX_TRIGGER_DDL = (
+    "CREATE TRIGGER IF NOT EXISTS conv_fts_aux_ai AFTER INSERT ON conversation_messages "
+    "BEGIN INSERT INTO conversation_fts_aux(rowid, search_aux) VALUES (new.id, new.search_aux); END",
+    "CREATE TRIGGER IF NOT EXISTS conv_fts_aux_ad AFTER DELETE ON conversation_messages "
+    "BEGIN INSERT INTO conversation_fts_aux(conversation_fts_aux, rowid, search_aux) "
+    "VALUES('delete', old.id, old.search_aux); END",
+    "CREATE TRIGGER IF NOT EXISTS conv_fts_aux_au AFTER UPDATE OF search_aux ON conversation_messages "
+    "BEGIN INSERT INTO conversation_fts_aux(conversation_fts_aux, rowid, search_aux) "
+    "VALUES('delete', old.id, old.search_aux); "
+    "INSERT INTO conversation_fts_aux(rowid, search_aux) VALUES (new.id, new.search_aux); END",
+)
+_CONV_FTS_AUX_TRIGGER_NAMES = ("conv_fts_aux_au", "conv_fts_aux_ad", "conv_fts_aux_ai")
+
 
 def _create_conversation_fts_triggers(conn: sqlite3.Connection) -> None:
-    """Create the three external-content FTS5 sync triggers (idempotent —
-    each is ``IF NOT EXISTS``). Single source of truth for the trigger DDL,
-    shared by ``_apply_cache_schema`` and ``clear_conversation_messages``
-    (#138). The caller must have already created ``conversation_fts``."""
+    """Create BOTH external-content FTS5 sync trigger sets — prose
+    (conversation_fts) + aux (conversation_fts_aux, #177) — idempotent (each is
+    ``IF NOT EXISTS``). Single source of truth for the trigger DDL, shared by
+    ``_apply_cache_schema`` and ``clear_conversation_messages`` (#138). The
+    caller must have already created both ``conversation_fts`` and
+    ``conversation_fts_aux``; one call site creates both sets so they can never
+    drift."""
     for stmt in _CONV_FTS_TRIGGER_DDL:
         conn.execute(stmt)
+    for stmt in _CONV_FTS_AUX_TRIGGER_DDL:
+        conn.execute(stmt)
 
 
 def _drop_conversation_fts_triggers(conn: sqlite3.Connection) -> None:
-    """Drop the three FTS5 sync triggers (idempotent — ``IF EXISTS``). Swallows
-    ``OperationalError`` per statement so a partial/absent trigger set (e.g. an
-    FTS-unavailable build) is tolerated."""
-    for name in _CONV_FTS_TRIGGER_NAMES:
+    """Drop BOTH FTS5 sync trigger sets — prose + aux (#177) — idempotent
+    (``IF EXISTS``). Swallows ``OperationalError`` per statement so a
+    partial/absent trigger set (e.g. an FTS-unavailable build) is tolerated. One
+    call site drops both sets so a failed aux create can't strand a live aux
+    trigger over a missing conversation_fts_aux."""
+    for name in _CONV_FTS_TRIGGER_NAMES + _CONV_FTS_AUX_TRIGGER_NAMES:
         try:
             conn.execute(f"DROP TRIGGER IF EXISTS {name}")
         except sqlite3.OperationalError:
@@ -2556,6 +2630,10 @@ def clear_conversation_messages(conn: sqlite3.Connection) -> None:
     _drop_conversation_fts_triggers(conn)
     conn.execute("DELETE FROM conversation_messages")
     conn.execute("INSERT INTO conversation_fts(conversation_fts) VALUES('delete-all')")
+    # #177: the aux index is created all-or-nothing with the prose FTS, so when
+    # fts_unavailable is False BOTH vtables exist — reset BOTH the same storm-free
+    # way (triggers already dropped above, so the base DELETE touched neither).
+    conn.execute("INSERT INTO conversation_fts_aux(conversation_fts_aux) VALUES('delete-all')")
     _create_conversation_fts_triggers(conn)
 
 
@@ -3074,6 +3152,25 @@ def _006_conversation_reingest_source_tool_use_id(conn: sqlite3.Connection) -> N
     conn.commit()
 
 
+@cache_migration("007_conversation_reingest_enrichment")
+def _007_conversation_reingest_enrichment(conn: sqlite3.Connection) -> None:
+    """Flag-only re-ingest so the enriched data contract — structured tool
+    ``input`` + ``input_truncated``, the raised result cap + ``full_length``,
+    ``stop_reason``/``attribution_skill``/``attribution_plugin``, and the
+    ``search_aux`` FTS-aux blob — lands on existing history. Sets the DISTINCT
+    ``conversation_reingest_enrichment_pending`` flag (NOT the shared
+    ``conversation_reingest_pending``, which also gates migration 005's read-time
+    human-fallback in the query kernel — re-arming it could misclassify a genuine
+    human prompt during the pre-reingest window). sync_cache consumes the flag
+    under the cache.db.lock flock (clear + offset-0 backfill); the offset-0 walk
+    re-parses every JSONL through the enriched parser, so the new fields/columns
+    land with zero new consumption code. Central stamp via the dispatcher (#140);
+    a fresh install stamps it without running (empty table -> the flag, if ever
+    set, is a harmless no-op)."""
+    _set_cache_meta(conn, "conversation_reingest_enrichment_pending", "1")
+    conn.commit()
+
+
 # === Region 7d: Stats migration 008_recompute_weekly_cost_snapshots_dedup_fix ===
 
 @stats_migration("008_recompute_weekly_cost_snapshots_dedup_fix")

package/bin/_lib_conversation.py

@@ -15,7 +15,13 @@ ASSISTANT = "assistant"
 TOOL_RESULT = "tool_result"
 META = "meta"
 
-_TOOL_RESULT_CAP = 4000  # chars; full text always re-derivable from JSONL
+_TOOL_RESULT_CAP = 16000   # was 4000; full text always re-derivable from JSONL
+_INPUT_LEAF_CAP = 8000     # max chars per string leaf in a bounded tool input
+_INPUT_TOTAL_CAP = 32000   # honesty backstop on the serialized bounded input
+_INPUT_MAX_NODES = 2000    # max dict-values + list-elements kept before tail elision
+_INPUT_MAX_DEPTH = 12      # max nesting depth before subtree elision (RecursionError guard)
+_INPUT_KEY_CAP = 512       # max chars per dict key (else keys are stored verbatim, unbounded)
+_INPUT_ELISION = "…"       # sentinel for elided leaves / subtrees
 
 
 @dataclass
@@ -35,6 +41,10 @@ class MessageRow:
     git_branch: "str | None"
     is_sidechain: int
     source_tool_use_id: "str | None" = None
+    stop_reason: "str | None" = None
+    attribution_skill: "str | None" = None
+    attribution_plugin: "str | None" = None
+    search_aux: str = ""
 
 
 def iter_message_rows(fh, path_str):
@@ -88,7 +98,7 @@ def _normalize(obj, t, offset):
     msg = obj.get("message")
     if not isinstance(msg, dict):
         msg = {}
-    blocks, text = _blocks_and_text(msg.get("content"))
+    blocks, text, aux = _blocks_and_text(msg.get("content"))
     if t == "assistant":
         entry_type = ASSISTANT
     elif any(b["kind"] == "tool_result" for b in blocks):
@@ -132,15 +142,33 @@ def _normalize(obj, t, offset):
         git_branch=obj.get("gitBranch"),
         is_sidechain=1 if obj.get("isSidechain") else 0,
         source_tool_use_id=obj.get("sourceToolUseID"),
+        # #177: message-level enrichment. stop_reason is assistant-only;
+        # attribution is top-level on the JSONL object. search_aux is kept even
+        # for tool_result/meta rows — only `text` is zeroed for prose FTS;
+        # search_aux is the non-prose index (tool content stays searchable).
+        stop_reason=msg.get("stop_reason") if is_asst else None,
+        attribution_skill=obj.get("attributionSkill"),
+        attribution_plugin=obj.get("attributionPlugin"),
+        search_aux=aux,
     )
 
 
 def _blocks_and_text(content):
-    """Return (normalized blocks list, indexed-prose string). Prose = joined
-    `text` blocks only (thinking / tool_use / tool_result excluded)."""
+    """Return (normalized blocks list, indexed-prose string, search_aux string).
+
+    Prose (``text``) = joined ``text`` blocks only (thinking / tool_use /
+    tool_result excluded — those go to the prose FTS via the ``text`` column).
+    ``search_aux`` (#177) = the non-prose searchable content: bounded tool-input
+    string leaves, the (capped) tool_result ``text``, and the thinking text
+    capped at ``_TOOL_RESULT_CAP`` (code-review I2 — the FULL thinking still
+    lives in ``blocks_json`` for rendering; only this aux index entry is capped
+    so the second FTS index doesn't double the at-rest cost of large thinking) —
+    indexed by the parallel ``conversation_fts_aux`` so tool content stays
+    searchable without polluting prose FTS. Prose is deliberately excluded from
+    aux (it is already in ``text``)."""
     if isinstance(content, str):
-        return ([{"kind": "text", "text": content}] if content else []), content
-    blocks, texts = [], []
+        return (([{"kind": "text", "text": content}] if content else []), content, "")
+    blocks, texts, aux_parts = [], [], []
     if isinstance(content, list):
         for b in content:
             if not isinstance(b, dict):
@@ -151,28 +179,37 @@ def _blocks_and_text(content):
                 blocks.append({"kind": "text", "text": txt})
                 texts.append(txt)
             elif bt == "thinking":
-                blocks.append({"kind": "thinking", "text": b.get("thinking", "") or ""})
+                think = b.get("thinking", "") or ""
+                blocks.append({"kind": "thinking", "text": think})  # FULL text for render
+                aux_parts.append(think[:_TOOL_RESULT_CAP])          # aux index capped (I2)
             elif bt == "tool_use":
+                bounded, input_trunc = _bound_input(b.get("input"))
                 block = {"kind": "tool_use", "name": b.get("name"),
                          "input_summary": _summarize(b.get("input")),
+                         "input": bounded, "input_truncated": input_trunc,
                          "id": b.get("id"),
                          "preview": tool_preview(b.get("name"), b.get("input"))}
                 inp = b.get("input")
                 st = inp.get("subagent_type") if isinstance(inp, dict) else None
                 if isinstance(st, str) and st:        # #166: spawn kind (Agent/Task)
                     block["subagent_type"] = st
+                aux_parts.extend(_aux_strings(bounded))
                 blocks.append(block)
             elif bt == "tool_result":
                 raw = _stringify(b.get("content"))
-                blocks.append({"kind": "tool_result", "text": raw[:_TOOL_RESULT_CAP],
+                clipped = raw[:_TOOL_RESULT_CAP]
+                blocks.append({"kind": "tool_result", "text": clipped,
                                "truncated": len(raw) > _TOOL_RESULT_CAP,
+                               "full_length": len(raw),
                                "is_error": bool(b.get("is_error")),
                                "tool_use_id": b.get("tool_use_id")})
+                aux_parts.append(clipped)
             elif bt in ("image", "document"):
                 blocks.append({"kind": bt, **_media(b.get("source"))})
             elif bt == "tool_reference":
                 blocks.append({"kind": "tool_reference", "name": b.get("name")})
-    return blocks, "\n".join(t for t in texts if t)
+    return (blocks, "\n".join(t for t in texts if t),
+            "\n".join(a for a in aux_parts if a))
 
 
 _SUBAGENT_META_KEYS = (
@@ -233,6 +270,77 @@ def _summarize(inp):
     return s[:200]
 
 
+def _bound_input(inp):
+    """Return (bounded_structured_input, truncated) for a tool_use input dict, or
+    (None, False) for a non-dict (the same non-dict contract as _summarize /
+    tool_preview). Hard-bounds the result on five axes so a pathological input
+    can't bloat blocks_json (Codex P1 + code-review I1): string leaves clip to
+    _INPUT_LEAF_CAP; dict keys clip to _INPUT_KEY_CAP (keys were stored verbatim
+    otherwise — the last unbounded axis); non-string scalars pass through; once
+    _INPUT_MAX_NODES dict-values/list-elements are kept the remainder elides to
+    _INPUT_ELISION; recursion past _INPUT_MAX_DEPTH elides the subtree
+    (RecursionError guard). A final _INPUT_TOTAL_CAP serialized-size check is the
+    honesty backstop. Structure is preserved for the kept prefix so downstream
+    renderers can read tool params."""
+    if not isinstance(inp, dict):
+        return (None, False)
+    state = {"nodes": 0, "truncated": False}
+
+    def walk(v, depth):
+        if depth > _INPUT_MAX_DEPTH:
+            state["truncated"] = True
+            return _INPUT_ELISION
+        if isinstance(v, str):
+            if len(v) > _INPUT_LEAF_CAP:
+                state["truncated"] = True
+                return v[:_INPUT_LEAF_CAP]
+            return v
+        if isinstance(v, dict):
+            out = {}
+            for k, vv in v.items():
+                ks = str(k)
+                if len(ks) > _INPUT_KEY_CAP:   # clip pathological long keys (I1)
+                    state["truncated"] = True
+                    ks = ks[:_INPUT_KEY_CAP]
+                if state["nodes"] >= _INPUT_MAX_NODES:
+                    state["truncated"] = True
+                    out[ks] = _INPUT_ELISION
+                    break
+                state["nodes"] += 1
+                out[ks] = walk(vv, depth + 1)
+            return out
+        if isinstance(v, list):
+            out = []
+            for vv in v:
+                if state["nodes"] >= _INPUT_MAX_NODES:
+                    state["truncated"] = True
+                    out.append(_INPUT_ELISION)
+                    break
+                state["nodes"] += 1
+                out.append(walk(vv, depth + 1))
+            return out
+        # int / float / bool / None — bounded-width scalars, pass through
+        return v
+
+    bounded = walk(inp, 0)
+    if len(json.dumps(bounded, separators=(",", ":"))) > _INPUT_TOTAL_CAP:
+        state["truncated"] = True
+    return (bounded, state["truncated"])
+
+
+def _aux_strings(v):
+    """Yield string leaves from a bounded input value (for the search_aux blob)."""
+    if isinstance(v, str):
+        if v:
+            yield v
+    elif isinstance(v, dict):
+        for vv in v.values():
+            yield from _aux_strings(vv)
+    elif isinstance(v, list):
+        for vv in v:
+            yield from _aux_strings(vv)
+
+
 _PREVIEW_FIELDS = {
     "Read": "file_path", "Write": "file_path", "Edit": "file_path",
     "MultiEdit": "file_path", "NotebookEdit": "file_path",

package/bin/_lib_conversation_query.py

@@ -358,6 +358,33 @@ def _turn_cost_map(conn, turn_keys):
     return costs
 
 
+def _turn_usage_map(conn, turn_keys):
+    """{(msg_id, req_id): {"input","output","cache_creation","cache_read"}} for
+    the given non-null turn keys, read from the SAME deduped session_entries row
+    cost is computed from (#177). This is a SEPARATE sibling of _turn_cost_map —
+    that one returns a float and is also consumed by the search path
+    (_attach_costs), so its shape must NOT change. Tokens here come from the same
+    source row as the cost, but they are NOT arithmetically equal to it: cost may
+    be the vendor-provided cost_usd_raw override (token math bypassed), so the
+    contract is "same source row," never cost == f(tokens). Keys absent from
+    session_entries are simply not present (the turn omits ``tokens``)."""
+    usage = {}
+    keys = [(m, r) for (m, r) in turn_keys if m is not None and r is not None]
+    if not keys:
+        return usage
+    for i in range(0, len(keys), 400):
+        chunk = keys[i:i + 400]
+        cond = " OR ".join("(msg_id=? AND req_id=?)" for _ in chunk)
+        params = [v for pair in chunk for v in pair]
+        sql = ("SELECT msg_id, req_id, input_tokens, output_tokens, "
+               "cache_create_tokens, cache_read_tokens "
+               "FROM session_entries WHERE " + cond)
+        for m, r, inp, out, cc, cr in conn.execute(sql, params):
+            usage[(m, r)] = {"input": inp or 0, "output": out or 0,
+                             "cache_creation": cc or 0, "cache_read": cr or 0}
+    return usage
+
+
 def get_conversation(conn, session_id, *, after=None, limit=500):
     """Reader payload for one session (spec §3.2). Returns None for an unknown
     session. Dedups logical messages by (session_id, uuid) (canonical = earliest
@@ -375,10 +402,14 @@ def get_conversation(conn, session_id, *, after=None, limit=500):
     # Pull the session ordered; dedup logical messages by (session_id, uuid),
     # canonical row = earliest (timestamp_utc, id). Replays carry the original
     # uuid, so the first occurrence in ascending order is canonical.
+    # #177: stop_reason / attribution_* are TAIL-APPENDED (indices 15/16/17)
+    # AFTER source_tool_use_id so the existing positional reads (incl.
+    # _latest(logical, 10/11) for cwd/git_branch, r[6] for model, [2] for ts)
+    # are all unchanged. Every unpacker below extends its tail in lockstep.
     raw = conn.execute(
         "SELECT id, uuid, timestamp_utc, entry_type, text, blocks_json, model, "
         "       msg_id, req_id, is_sidechain, cwd, git_branch, source_path, parent_uuid, "
-        "       source_tool_use_id "
+        "       source_tool_use_id, stop_reason, attribution_skill, attribution_plugin "
         "FROM conversation_messages WHERE session_id=? "
         "ORDER BY timestamp_utc, id", (session_id,)).fetchall()
 
@@ -422,7 +453,8 @@ def get_conversation(conn, session_id, *, after=None, limit=500):
 
     for row in logical:
         (rid, u, ts, etype, text, blocks, model, msg_id, req_id,
-         is_sc, cwd, branch, source_path, parent_uuid, source_tool_use_id) = row
+         is_sc, cwd, branch, source_path, parent_uuid, source_tool_use_id,
+         stop_reason, attr_skill, attr_plugin) = row
         if etype == "assistant" and msg_id is not None:
             key = (msg_id, req_id)
             idx = turn_index.get(key)
@@ -495,8 +527,12 @@ def get_conversation(conn, session_id, *, after=None, limit=500):
         if owners and not non_result and len(owner_ids) == 1:
             owner = owners[0]
             for use_block, res_block in resolved:
+                # #177: full_length (pre-clip char count) rides through the fold
+                # for the "showing X of Y" affordance; None on pre-enrichment
+                # rows that lack it (the .get default — never KeyErrors).
                 use_block["result"] = {"text": res_block.get("text", ""),
                                        "truncated": bool(res_block.get("truncated")),
+                                       "full_length": res_block.get("full_length"),
                                        "is_error": bool(res_block.get("is_error"))}
             owner["member_uuids"].append(tr["anchor"]["uuid"])
             drop.add(id(tr))
@@ -570,6 +606,9 @@ def get_conversation(conn, session_id, *, after=None, limit=500):
         items = [it for it in items if id(it) not in _skill_drop]
 
     costs = _turn_cost_map(conn, list(turn_index))
+    # #177: per-turn token usage from the SAME deduped session_entries row cost
+    # uses (a separate map; _turn_cost_map is unchanged for the search path).
+    usage = _turn_usage_map(conn, list(turn_index))
     # Stamp per-item cost first, then derive the header from the SUM of the
     # ROUNDED per-item assistant costs (M2) — so the §6.5 invariant
     # sum(items.cost_usd) == header cost_usd holds EXACTLY to 1e-9 by
@@ -582,6 +621,11 @@ def get_conversation(conn, session_id, *, after=None, limit=500):
             turn_cost = round(costs.get((it["_msg_id"], it["_req_id"]), 0.0), 6)
             it["cost_usd"] = turn_cost
             header_cost += turn_cost
+            # #177: stamp tokens from the same source row; absent when the turn
+            # key has no session_entries row (omitted, not zero-filled).
+            tok = usage.get((it["_msg_id"], it["_req_id"]))
+            if tok is not None:
+                it["tokens"] = tok
             del it["_msg_id"]
             del it["_req_id"]
             it.pop("_has_prose", None)
@@ -682,6 +726,11 @@ def _build_turn(members):
         "_source_tool_use_id": first[14],
         "_has_prose": False,
     }
+    # #177: stop_reason / attribution_* (tail-appended cols 15/16/17) are seeded
+    # by the _fold_fragment(item, first) call below — the same seed-then-fold path
+    # model/text/is_sidechain already use. _fold_fragment applies the last-non-null
+    # guard, so a single-fragment turn is covered there and omitted keys (never
+    # None) preserve the absent-when-absent contract.
     _fold_fragment(item, first)
     for m in members[1:]:
         _extend_turn(item, m)
@@ -717,6 +766,16 @@ def _fold_fragment(item, row):
             item["text"] = frag_text
         else:
             item["text"] = item["text"] + "\n" + frag_text
+    # #177: stop_reason = last-non-null fragment value (the terminal fragment
+    # carries the real reason); attribution_* = last-non-null (turn-level
+    # constant). A later null fragment must NOT blank an earlier value — hence
+    # the `is not None` guard rather than an unconditional assign.
+    if row[15] is not None:
+        item["stop_reason"] = row[15]
+    if row[16] is not None:
+        item["attribution_skill"] = row[16]
+    if row[17] is not None:
+        item["attribution_plugin"] = row[17]
 
 
 def _build_simple(row):
@@ -726,7 +785,8 @@ def _build_simple(row):
     internal _msg_id/_req_id keys, so the cost loop's KeyError path can never fire
     (I2). The model is preserved for assistant rows."""
     (rid, u, ts, etype, text, blocks, model, msg_id, req_id, is_sc, cwd, branch,
-     source_path, parent_uuid, source_tool_use_id) = row
+     source_path, parent_uuid, source_tool_use_id,
+     stop_reason, attr_skill, attr_plugin) = row
     try:
         parsed = _json.loads(blocks or "[]")
     except (ValueError, TypeError):
@@ -749,6 +809,15 @@ def _build_simple(row):
     if etype == "assistant":
         item["model"] = model
         item["cost_usd"] = 0.0
+        # #177: stop_reason / attribution are assistant-only — a null-msg_id
+        # assistant turn still carries them. Omitted when null (absent-when-
+        # absent). Human / tool_result simple items never get these keys.
+        if stop_reason is not None:
+            item["stop_reason"] = stop_reason
+        if attr_skill is not None:
+            item["attribution_skill"] = attr_skill
+        if attr_plugin is not None:
+            item["attribution_plugin"] = attr_plugin
     return item
 
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cctally",
-  "version": "1.34.1",
+  "version": "1.35.0",
   "description": "Claude Code usage tracker and local dashboard for Pro/Max subscription limits - weekly cost-per-percent trend, quota forecasts, threshold alerts. ccusage-compatible.",
   "homepage": "https://github.com/omrikais/cctally",
   "repository": {