cctally 1.34.1 → 1.36.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +10 -0
- package/bin/_cctally_cache.py +32 -9
- package/bin/_cctally_db.py +111 -14
- package/bin/_lib_conversation.py +155 -9
- package/bin/_lib_conversation_query.py +82 -3
- package/dashboard/static/assets/{index-CXakskUV.js → index-DxQXelz7.js} +19 -17
- package/dashboard/static/assets/{index-Q7C0vruA.css → index-a_BOAOJ_.css} +1 -1
- package/dashboard/static/dashboard.html +2 -2
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -5,6 +5,16 @@ based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
|
|
|
5
5
|
|
|
6
6
|
## [Unreleased]
|
|
7
7
|
|
|
8
|
+
## [1.36.0] - 2026-06-11
|
|
9
|
+
|
|
10
|
+
### Added
|
|
11
|
+
- Conversation viewer: AskUserQuestion, TodoWrite, and ExitPlanMode now render as dedicated cards — the question-and-chosen-answer, a live checklist with progress, and the rendered plan with its approve/reject outcome — instead of raw JSON tool chips (#177 S2).
|
|
12
|
+
|
|
13
|
+
## [1.35.0] - 2026-06-11
|
|
14
|
+
|
|
15
|
+
### Added
|
|
16
|
+
- Dashboard conversation reader: the transcript data contract now carries structured tool input (with an `input_truncated` flag), tool results with honest length accounting (`full_length` plus a raised result cap), per-turn token usage drawn from the same deduped session-entry row as the cost, and each turn's stop-reason and skill/plugin attribution — all additive (the existing reader renders unchanged), with a parallel tool-content search index built as groundwork. The new fields are re-derived across existing transcripts by a one-time cache migration (#177).
|
|
17
|
+
|
|
8
18
|
## [1.34.1] - 2026-06-11
|
|
9
19
|
|
|
10
20
|
### Fixed
|
package/bin/_cctally_cache.py
CHANGED
|
@@ -183,18 +183,25 @@ _CONV_INSERT_SQL = (
|
|
|
183
183
|
"INSERT OR IGNORE INTO conversation_messages"
|
|
184
184
|
"(session_id,uuid,parent_uuid,source_path,byte_offset,"
|
|
185
185
|
" timestamp_utc,entry_type,text,blocks_json,model,msg_id,"
|
|
186
|
-
" req_id,cwd,git_branch,is_sidechain,source_tool_use_id
|
|
187
|
-
"
|
|
186
|
+
" req_id,cwd,git_branch,is_sidechain,source_tool_use_id,"
|
|
187
|
+
" stop_reason,attribution_skill,attribution_plugin,search_aux)"
|
|
188
|
+
" VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
|
|
188
189
|
)
|
|
189
190
|
|
|
190
191
|
|
|
191
192
|
def _conv_row_tuple(m, path_str):
|
|
192
|
-
"""Flatten a ``MessageRow`` into the ``_CONV_INSERT_SQL`` column order.
|
|
193
|
+
"""Flatten a ``MessageRow`` into the ``_CONV_INSERT_SQL`` column order.
|
|
194
|
+
|
|
195
|
+
The #177 enrichment fields (stop_reason / attribution_skill /
|
|
196
|
+
attribution_plugin / search_aux) are TAIL-APPENDED after source_tool_use_id
|
|
197
|
+
— same order as the SQL column list — so both ingest paths (fused per-file
|
|
198
|
+
walk + backfill_conversation_messages) carry them through this one tuple."""
|
|
193
199
|
return (
|
|
194
200
|
m.session_id, m.uuid, m.parent_uuid, path_str, m.byte_offset,
|
|
195
201
|
m.timestamp_utc, m.entry_type, m.text, m.blocks_json, m.model,
|
|
196
202
|
m.msg_id, m.req_id, m.cwd, m.git_branch, m.is_sidechain,
|
|
197
203
|
m.source_tool_use_id,
|
|
204
|
+
m.stop_reason, m.attribution_skill, m.attribution_plugin, m.search_aux,
|
|
198
205
|
)
|
|
199
206
|
|
|
200
207
|
|
|
@@ -622,11 +629,18 @@ def sync_cache(
|
|
|
622
629
|
# dropping the flag here covers the 004 reingest too. Migration 006
|
|
623
630
|
# sets the DISTINCT conversation_source_tool_use_reingest_pending
|
|
624
631
|
# flag (to land source_tool_use_id); the same offset-0 walk re-derives
|
|
625
|
-
# it, so drop that flag here as well to avoid a redundant pass.
|
|
632
|
+
# it, so drop that flag here as well to avoid a redundant pass. #177
|
|
633
|
+
# migration 007 sets the DISTINCT
|
|
634
|
+
# conversation_reingest_enrichment_pending flag (to land structured
|
|
635
|
+
# input / full_length / stop_reason / attribution / search_aux); the
|
|
636
|
+
# same offset-0 walk re-derives those through the enriched parser, so
|
|
637
|
+
# drop that flag here too — MISSING this site re-arms the flag on
|
|
638
|
+
# every cache-sync --rebuild.
|
|
626
639
|
conn.execute(
|
|
627
640
|
"DELETE FROM cache_meta WHERE key IN "
|
|
628
641
|
"('conversation_reingest_pending',"
|
|
629
|
-
" 'conversation_source_tool_use_reingest_pending'
|
|
642
|
+
" 'conversation_source_tool_use_reingest_pending',"
|
|
643
|
+
" 'conversation_reingest_enrichment_pending')")
|
|
630
644
|
conn.commit()
|
|
631
645
|
eprint("[cache-sync] rebuild: cleared Claude cached entries")
|
|
632
646
|
|
|
@@ -687,13 +701,21 @@ def sync_cache(
|
|
|
687
701
|
# shared one) to land the message-level ``source_tool_use_id`` — the
|
|
688
702
|
# shared flag also gates the kernel's 005 human-fallback, so re-arming
|
|
689
703
|
# it for 006 could misclassify a genuine human prompt during the
|
|
690
|
-
# pre-reingest window.
|
|
691
|
-
#
|
|
704
|
+
# pre-reingest window. #177 migration 007 uses ANOTHER distinct flag
|
|
705
|
+
# ``conversation_reingest_enrichment_pending`` (for the same shared-flag
|
|
706
|
+
# reason) to land the enriched data contract (structured input +
|
|
707
|
+
# input_truncated, the raised result cap + full_length, stop_reason /
|
|
708
|
+
# attribution_skill / attribution_plugin, and the search_aux FTS-aux
|
|
709
|
+
# blob); the offset-0 re-parse through the enriched parser lands them
|
|
710
|
+
# all with zero new consumption code. We trigger the SAME clear +
|
|
711
|
+
# offset-0 backfill on ANY of these flags and clear them ALL atomically
|
|
712
|
+
# here under the held flock.
|
|
692
713
|
try:
|
|
693
714
|
_reingest = conn.execute(
|
|
694
715
|
"SELECT 1 FROM cache_meta WHERE key IN "
|
|
695
716
|
"('conversation_reingest_pending',"
|
|
696
|
-
" 'conversation_source_tool_use_reingest_pending'
|
|
717
|
+
" 'conversation_source_tool_use_reingest_pending',"
|
|
718
|
+
" 'conversation_reingest_enrichment_pending')"
|
|
697
719
|
).fetchone() is not None
|
|
698
720
|
except sqlite3.OperationalError:
|
|
699
721
|
_reingest = False
|
|
@@ -703,7 +725,8 @@ def sync_cache(
|
|
|
703
725
|
conn.execute(
|
|
704
726
|
"DELETE FROM cache_meta WHERE key IN "
|
|
705
727
|
"('conversation_reingest_pending',"
|
|
706
|
-
" 'conversation_source_tool_use_reingest_pending'
|
|
728
|
+
" 'conversation_source_tool_use_reingest_pending',"
|
|
729
|
+
" 'conversation_reingest_enrichment_pending')"
|
|
707
730
|
)
|
|
708
731
|
conn.commit()
|
|
709
732
|
|
package/bin/_cctally_db.py
CHANGED
|
@@ -2389,6 +2389,15 @@ def _apply_cache_schema(conn: sqlite3.Connection) -> None:
|
|
|
2389
2389
|
# column-add (no marker, no version); cache migration 006 then re-ingests
|
|
2390
2390
|
# so the value actually lands on historical rows.
|
|
2391
2391
|
add_column_if_missing(conn, "conversation_messages", "source_tool_use_id", "TEXT")
|
|
2392
|
+
# #177 Session 1: enriched data-contract columns. Idempotent column-adds (no
|
|
2393
|
+
# marker, no version — exactly like source_tool_use_id); cache migration 007
|
|
2394
|
+
# then re-ingests so the values actually land on historical rows. search_aux
|
|
2395
|
+
# is the parser-populated non-prose blob the conversation_fts_aux index reads.
|
|
2396
|
+
add_column_if_missing(conn, "conversation_messages", "stop_reason", "TEXT")
|
|
2397
|
+
add_column_if_missing(conn, "conversation_messages", "attribution_skill", "TEXT")
|
|
2398
|
+
add_column_if_missing(conn, "conversation_messages", "attribution_plugin", "TEXT")
|
|
2399
|
+
add_column_if_missing(
|
|
2400
|
+
conn, "conversation_messages", "search_aux", "TEXT NOT NULL DEFAULT ''")
|
|
2392
2401
|
conn.execute(
|
|
2393
2402
|
"CREATE INDEX IF NOT EXISTS idx_session_files_session_id "
|
|
2394
2403
|
"ON session_files(session_id)"
|
|
@@ -2414,23 +2423,40 @@ def _apply_cache_schema(conn: sqlite3.Connection) -> None:
|
|
|
2414
2423
|
conn.execute(
|
|
2415
2424
|
"CREATE VIRTUAL TABLE IF NOT EXISTS conversation_fts "
|
|
2416
2425
|
"USING fts5(text, content='conversation_messages', content_rowid='id')")
|
|
2417
|
-
#
|
|
2418
|
-
#
|
|
2419
|
-
#
|
|
2420
|
-
#
|
|
2426
|
+
# #177: the aux external-content index is ALL-OR-NOTHING with the
|
|
2427
|
+
# prose FTS under the SINGLE fts5_unavailable flag — created in this
|
|
2428
|
+
# SAME try-envelope (via the _create_conversation_fts_aux_table seam,
|
|
2429
|
+
# which tests monkeypatch to simulate an aux-create failure AFTER the
|
|
2430
|
+
# prose create succeeded), so ANY OperationalError below drops BOTH
|
|
2431
|
+
# tables + BOTH trigger sets (the except arm). A live aux trigger over
|
|
2432
|
+
# a missing conversation_fts_aux would roll back the shared
|
|
2433
|
+
# conversation_messages write transaction (which also carries cost
|
|
2434
|
+
# ingest into session_entries). It is NOT given its own guarded block
|
|
2435
|
+
# or its own flag.
|
|
2436
|
+
_create_conversation_fts_aux_table(conn)
|
|
2437
|
+
# Trigger DDL lives in ONE place (_CONV_FTS_TRIGGER_DDL +
|
|
2438
|
+
# _CONV_FTS_AUX_TRIGGER_DDL) so this initial create and the #138
|
|
2439
|
+
# storm-free full-clear (clear_conversation_messages, which drops +
|
|
2440
|
+
# recreates BOTH trigger sets) can never drift.
|
|
2421
2441
|
_create_conversation_fts_triggers(conn)
|
|
2422
2442
|
if recovering:
|
|
2423
|
-
# Repopulate the freshly-(re)created
|
|
2443
|
+
# Repopulate the freshly-(re)created indexes from the base table
|
|
2424
2444
|
# so pre-recovery history is searchable. Cheap no-op when
|
|
2425
2445
|
# conversation_messages is empty.
|
|
2426
2446
|
conn.execute(
|
|
2427
2447
|
"INSERT INTO conversation_fts(conversation_fts) VALUES('rebuild')")
|
|
2448
|
+
conn.execute(
|
|
2449
|
+
"INSERT INTO conversation_fts_aux(conversation_fts_aux) VALUES('rebuild')")
|
|
2428
2450
|
conn.execute("DELETE FROM cache_meta WHERE key='fts5_unavailable'")
|
|
2429
2451
|
except sqlite3.OperationalError:
|
|
2430
|
-
# partial create cleanup, then mark unavailable
|
|
2452
|
+
# partial create cleanup, then mark unavailable. _drop drops BOTH
|
|
2453
|
+
# prose + aux trigger sets (#177), and we drop BOTH vtables, so a
|
|
2454
|
+
# failed aux create can't leave a live aux trigger over a missing
|
|
2455
|
+
# table.
|
|
2431
2456
|
_drop_conversation_fts_triggers(conn)
|
|
2432
2457
|
try:
|
|
2433
2458
|
conn.execute("DROP TABLE IF EXISTS conversation_fts")
|
|
2459
|
+
conn.execute("DROP TABLE IF EXISTS conversation_fts_aux")
|
|
2434
2460
|
except sqlite3.OperationalError:
|
|
2435
2461
|
pass
|
|
2436
2462
|
_set_cache_meta(conn, "fts5_unavailable", "1")
|
|
@@ -2472,6 +2498,28 @@ def _set_cache_meta(conn: sqlite3.Connection, key: str, value: str) -> None:
|
|
|
2472
2498
|
"ON CONFLICT(key) DO UPDATE SET value=excluded.value", (key, value))
|
|
2473
2499
|
|
|
2474
2500
|
|
|
2501
|
+
def _create_conversation_fts_aux_table(conn: sqlite3.Connection) -> None:
|
|
2502
|
+
"""Create the #177 aux external-content FTS5 index over the ``search_aux``
|
|
2503
|
+
blob. A standalone module-level seam (NOT inlined in ``_apply_cache_schema``)
|
|
2504
|
+
so the all-or-nothing regression test can monkeypatch it to raise
|
|
2505
|
+
``OperationalError`` AFTER the prose ``conversation_fts`` create succeeded —
|
|
2506
|
+
proving the shared try-envelope drops BOTH indexes + BOTH trigger sets and a
|
|
2507
|
+
later ``conversation_messages`` INSERT still commits (the cost write txn is
|
|
2508
|
+
not rolled back). Must run inside that envelope; idempotent
|
|
2509
|
+
(``IF NOT EXISTS``)."""
|
|
2510
|
+
# The FTS column name MUST match the content table column (``search_aux``),
|
|
2511
|
+
# exactly as the prose ``conversation_fts`` uses ``text``. An external-content
|
|
2512
|
+
# FTS5 table resolves its columns through the content table by NAME, so a
|
|
2513
|
+
# mismatched column (e.g. ``aux`` against a ``search_aux`` content column)
|
|
2514
|
+
# leaves the index functional for MATCH but breaks any content-backed read of
|
|
2515
|
+
# the column — notably ``sqlite3.Connection.iterdump`` (the dump emits
|
|
2516
|
+
# ``SELECT quote(aux) FROM conversation_fts_aux`` → "no such column"). Aligning
|
|
2517
|
+
# the names keeps the index dumpable and is the documented FTS5 posture.
|
|
2518
|
+
conn.execute(
|
|
2519
|
+
"CREATE VIRTUAL TABLE IF NOT EXISTS conversation_fts_aux "
|
|
2520
|
+
"USING fts5(search_aux, content='conversation_messages', content_rowid='id')")
|
|
2521
|
+
|
|
2522
|
+
|
|
2475
2523
|
# Conversation FTS sync triggers (external-content FTS5). Defined ONCE here so
|
|
2476
2524
|
# the initial create in _apply_cache_schema and the #138 storm-free full-clear
|
|
2477
2525
|
# in clear_conversation_messages (which drops + recreates them) can never drift.
|
|
@@ -2491,21 +2539,47 @@ _CONV_FTS_TRIGGER_DDL = (
|
|
|
2491
2539
|
# cosmetic — order doesn't matter for independent triggers.
|
|
2492
2540
|
_CONV_FTS_TRIGGER_NAMES = ("conv_fts_au", "conv_fts_ad", "conv_fts_ai")
|
|
2493
2541
|
|
|
2542
|
+
# #177: the parallel aux index (conversation_fts_aux) over the search_aux blob.
|
|
2543
|
+
# All-or-nothing with the prose FTS under the single fts5_unavailable flag — its
|
|
2544
|
+
# DDL lives beside the prose set and the SAME create/drop chokepoints handle
|
|
2545
|
+
# both, so the two trigger sets can never drift. The AU trigger is keyed on
|
|
2546
|
+
# ``AFTER UPDATE OF search_aux`` (a text-only update doesn't fire it, and the
|
|
2547
|
+
# prose AU's ``AFTER UPDATE OF text`` doesn't fire this one).
|
|
2548
|
+
_CONV_FTS_AUX_TRIGGER_DDL = (
|
|
2549
|
+
"CREATE TRIGGER IF NOT EXISTS conv_fts_aux_ai AFTER INSERT ON conversation_messages "
|
|
2550
|
+
"BEGIN INSERT INTO conversation_fts_aux(rowid, search_aux) VALUES (new.id, new.search_aux); END",
|
|
2551
|
+
"CREATE TRIGGER IF NOT EXISTS conv_fts_aux_ad AFTER DELETE ON conversation_messages "
|
|
2552
|
+
"BEGIN INSERT INTO conversation_fts_aux(conversation_fts_aux, rowid, search_aux) "
|
|
2553
|
+
"VALUES('delete', old.id, old.search_aux); END",
|
|
2554
|
+
"CREATE TRIGGER IF NOT EXISTS conv_fts_aux_au AFTER UPDATE OF search_aux ON conversation_messages "
|
|
2555
|
+
"BEGIN INSERT INTO conversation_fts_aux(conversation_fts_aux, rowid, search_aux) "
|
|
2556
|
+
"VALUES('delete', old.id, old.search_aux); "
|
|
2557
|
+
"INSERT INTO conversation_fts_aux(rowid, search_aux) VALUES (new.id, new.search_aux); END",
|
|
2558
|
+
)
|
|
2559
|
+
_CONV_FTS_AUX_TRIGGER_NAMES = ("conv_fts_aux_au", "conv_fts_aux_ad", "conv_fts_aux_ai")
|
|
2560
|
+
|
|
2494
2561
|
|
|
2495
2562
|
def _create_conversation_fts_triggers(conn: sqlite3.Connection) -> None:
|
|
2496
|
-
"""Create
|
|
2497
|
-
|
|
2498
|
-
|
|
2499
|
-
(#138). The
|
|
2563
|
+
"""Create BOTH external-content FTS5 sync trigger sets — prose
|
|
2564
|
+
(conversation_fts) + aux (conversation_fts_aux, #177) — idempotent (each is
|
|
2565
|
+
``IF NOT EXISTS``). Single source of truth for the trigger DDL, shared by
|
|
2566
|
+
``_apply_cache_schema`` and ``clear_conversation_messages`` (#138). The
|
|
2567
|
+
caller must have already created both ``conversation_fts`` and
|
|
2568
|
+
``conversation_fts_aux``; one call site creates both sets so they can never
|
|
2569
|
+
drift."""
|
|
2500
2570
|
for stmt in _CONV_FTS_TRIGGER_DDL:
|
|
2501
2571
|
conn.execute(stmt)
|
|
2572
|
+
for stmt in _CONV_FTS_AUX_TRIGGER_DDL:
|
|
2573
|
+
conn.execute(stmt)
|
|
2502
2574
|
|
|
2503
2575
|
|
|
2504
2576
|
def _drop_conversation_fts_triggers(conn: sqlite3.Connection) -> None:
|
|
2505
|
-
"""Drop
|
|
2506
|
-
``OperationalError`` per statement so a
|
|
2507
|
-
FTS-unavailable build) is tolerated.
|
|
2508
|
-
|
|
2577
|
+
"""Drop BOTH FTS5 sync trigger sets — prose + aux (#177) — idempotent
|
|
2578
|
+
(``IF EXISTS``). Swallows ``OperationalError`` per statement so a
|
|
2579
|
+
partial/absent trigger set (e.g. an FTS-unavailable build) is tolerated. One
|
|
2580
|
+
call site drops both sets so a failed aux create can't strand a live aux
|
|
2581
|
+
trigger over a missing conversation_fts_aux."""
|
|
2582
|
+
for name in _CONV_FTS_TRIGGER_NAMES + _CONV_FTS_AUX_TRIGGER_NAMES:
|
|
2509
2583
|
try:
|
|
2510
2584
|
conn.execute(f"DROP TRIGGER IF EXISTS {name}")
|
|
2511
2585
|
except sqlite3.OperationalError:
|
|
@@ -2556,6 +2630,10 @@ def clear_conversation_messages(conn: sqlite3.Connection) -> None:
|
|
|
2556
2630
|
_drop_conversation_fts_triggers(conn)
|
|
2557
2631
|
conn.execute("DELETE FROM conversation_messages")
|
|
2558
2632
|
conn.execute("INSERT INTO conversation_fts(conversation_fts) VALUES('delete-all')")
|
|
2633
|
+
# #177: the aux index is created all-or-nothing with the prose FTS, so when
|
|
2634
|
+
# fts_unavailable is False BOTH vtables exist — reset BOTH the same storm-free
|
|
2635
|
+
# way (triggers already dropped above, so the base DELETE touched neither).
|
|
2636
|
+
conn.execute("INSERT INTO conversation_fts_aux(conversation_fts_aux) VALUES('delete-all')")
|
|
2559
2637
|
_create_conversation_fts_triggers(conn)
|
|
2560
2638
|
|
|
2561
2639
|
|
|
@@ -3074,6 +3152,25 @@ def _006_conversation_reingest_source_tool_use_id(conn: sqlite3.Connection) -> N
|
|
|
3074
3152
|
conn.commit()
|
|
3075
3153
|
|
|
3076
3154
|
|
|
3155
|
+
@cache_migration("007_conversation_reingest_enrichment")
|
|
3156
|
+
def _007_conversation_reingest_enrichment(conn: sqlite3.Connection) -> None:
|
|
3157
|
+
"""Flag-only re-ingest so the enriched data contract — structured tool
|
|
3158
|
+
``input`` + ``input_truncated``, the raised result cap + ``full_length``,
|
|
3159
|
+
``stop_reason``/``attribution_skill``/``attribution_plugin``, and the
|
|
3160
|
+
``search_aux`` FTS-aux blob — lands on existing history. Sets the DISTINCT
|
|
3161
|
+
``conversation_reingest_enrichment_pending`` flag (NOT the shared
|
|
3162
|
+
``conversation_reingest_pending``, which also gates migration 005's read-time
|
|
3163
|
+
human-fallback in the query kernel — re-arming it could misclassify a genuine
|
|
3164
|
+
human prompt during the pre-reingest window). sync_cache consumes the flag
|
|
3165
|
+
under the cache.db.lock flock (clear + offset-0 backfill); the offset-0 walk
|
|
3166
|
+
re-parses every JSONL through the enriched parser, so the new fields/columns
|
|
3167
|
+
land with zero new consumption code. Central stamp via the dispatcher (#140);
|
|
3168
|
+
a fresh install stamps it without running (empty table -> the flag, if ever
|
|
3169
|
+
set, is a harmless no-op)."""
|
|
3170
|
+
_set_cache_meta(conn, "conversation_reingest_enrichment_pending", "1")
|
|
3171
|
+
conn.commit()
|
|
3172
|
+
|
|
3173
|
+
|
|
3077
3174
|
# === Region 7d: Stats migration 008_recompute_weekly_cost_snapshots_dedup_fix ===
|
|
3078
3175
|
|
|
3079
3176
|
@stats_migration("008_recompute_weekly_cost_snapshots_dedup_fix")
|
package/bin/_lib_conversation.py
CHANGED
|
@@ -15,7 +15,13 @@ ASSISTANT = "assistant"
|
|
|
15
15
|
TOOL_RESULT = "tool_result"
|
|
16
16
|
META = "meta"
|
|
17
17
|
|
|
18
|
-
_TOOL_RESULT_CAP =
|
|
18
|
+
_TOOL_RESULT_CAP = 16000 # was 4000; full text always re-derivable from JSONL
|
|
19
|
+
_INPUT_LEAF_CAP = 8000 # max chars per string leaf in a bounded tool input
|
|
20
|
+
_INPUT_TOTAL_CAP = 32000 # honesty backstop on the serialized bounded input
|
|
21
|
+
_INPUT_MAX_NODES = 2000 # max dict-values + list-elements kept before tail elision
|
|
22
|
+
_INPUT_MAX_DEPTH = 12 # max nesting depth before subtree elision (RecursionError guard)
|
|
23
|
+
_INPUT_KEY_CAP = 512 # max chars per dict key (else keys are stored verbatim, unbounded)
|
|
24
|
+
_INPUT_ELISION = "…" # sentinel for elided leaves / subtrees
|
|
19
25
|
|
|
20
26
|
|
|
21
27
|
@dataclass
|
|
@@ -35,6 +41,10 @@ class MessageRow:
|
|
|
35
41
|
git_branch: "str | None"
|
|
36
42
|
is_sidechain: int
|
|
37
43
|
source_tool_use_id: "str | None" = None
|
|
44
|
+
stop_reason: "str | None" = None
|
|
45
|
+
attribution_skill: "str | None" = None
|
|
46
|
+
attribution_plugin: "str | None" = None
|
|
47
|
+
search_aux: str = ""
|
|
38
48
|
|
|
39
49
|
|
|
40
50
|
def iter_message_rows(fh, path_str):
|
|
@@ -88,12 +98,13 @@ def _normalize(obj, t, offset):
|
|
|
88
98
|
msg = obj.get("message")
|
|
89
99
|
if not isinstance(msg, dict):
|
|
90
100
|
msg = {}
|
|
91
|
-
blocks, text = _blocks_and_text(msg.get("content"))
|
|
101
|
+
blocks, text, aux = _blocks_and_text(msg.get("content"))
|
|
92
102
|
if t == "assistant":
|
|
93
103
|
entry_type = ASSISTANT
|
|
94
104
|
elif any(b["kind"] == "tool_result" for b in blocks):
|
|
95
105
|
entry_type = TOOL_RESULT
|
|
96
106
|
_attach_subagent_result(blocks, obj) # #166: record-level toolUseResult
|
|
107
|
+
_attach_ask_answers(blocks, obj) # #177 S2: AskUserQuestion answers
|
|
97
108
|
# tool_result rows are stored but NOT indexed as prose (spec §2). A
|
|
98
109
|
# user line that mixes a text block with a tool_result block must not
|
|
99
110
|
# leak that text into the FTS index; the full content stays in
|
|
@@ -132,15 +143,33 @@ def _normalize(obj, t, offset):
|
|
|
132
143
|
git_branch=obj.get("gitBranch"),
|
|
133
144
|
is_sidechain=1 if obj.get("isSidechain") else 0,
|
|
134
145
|
source_tool_use_id=obj.get("sourceToolUseID"),
|
|
146
|
+
# #177: message-level enrichment. stop_reason is assistant-only;
|
|
147
|
+
# attribution is top-level on the JSONL object. search_aux is kept even
|
|
148
|
+
# for tool_result/meta rows — only `text` is zeroed for prose FTS;
|
|
149
|
+
# search_aux is the non-prose index (tool content stays searchable).
|
|
150
|
+
stop_reason=msg.get("stop_reason") if is_asst else None,
|
|
151
|
+
attribution_skill=obj.get("attributionSkill"),
|
|
152
|
+
attribution_plugin=obj.get("attributionPlugin"),
|
|
153
|
+
search_aux=aux,
|
|
135
154
|
)
|
|
136
155
|
|
|
137
156
|
|
|
138
157
|
def _blocks_and_text(content):
|
|
139
|
-
"""Return (normalized blocks list, indexed-prose string).
|
|
140
|
-
|
|
158
|
+
"""Return (normalized blocks list, indexed-prose string, search_aux string).
|
|
159
|
+
|
|
160
|
+
Prose (``text``) = joined ``text`` blocks only (thinking / tool_use /
|
|
161
|
+
tool_result excluded — those go to the prose FTS via the ``text`` column).
|
|
162
|
+
``search_aux`` (#177) = the non-prose searchable content: bounded tool-input
|
|
163
|
+
string leaves, the (capped) tool_result ``text``, and the thinking text
|
|
164
|
+
capped at ``_TOOL_RESULT_CAP`` (code-review I2 — the FULL thinking still
|
|
165
|
+
lives in ``blocks_json`` for rendering; only this aux index entry is capped
|
|
166
|
+
so the second FTS index doesn't double the at-rest cost of large thinking) —
|
|
167
|
+
indexed by the parallel ``conversation_fts_aux`` so tool content stays
|
|
168
|
+
searchable without polluting prose FTS. Prose is deliberately excluded from
|
|
169
|
+
aux (it is already in ``text``)."""
|
|
141
170
|
if isinstance(content, str):
|
|
142
|
-
return ([{"kind": "text", "text": content}] if content else []), content
|
|
143
|
-
blocks, texts = [], []
|
|
171
|
+
return (([{"kind": "text", "text": content}] if content else []), content, "")
|
|
172
|
+
blocks, texts, aux_parts = [], [], []
|
|
144
173
|
if isinstance(content, list):
|
|
145
174
|
for b in content:
|
|
146
175
|
if not isinstance(b, dict):
|
|
@@ -151,28 +180,37 @@ def _blocks_and_text(content):
|
|
|
151
180
|
blocks.append({"kind": "text", "text": txt})
|
|
152
181
|
texts.append(txt)
|
|
153
182
|
elif bt == "thinking":
|
|
154
|
-
|
|
183
|
+
think = b.get("thinking", "") or ""
|
|
184
|
+
blocks.append({"kind": "thinking", "text": think}) # FULL text for render
|
|
185
|
+
aux_parts.append(think[:_TOOL_RESULT_CAP]) # aux index capped (I2)
|
|
155
186
|
elif bt == "tool_use":
|
|
187
|
+
bounded, input_trunc = _bound_input(b.get("input"))
|
|
156
188
|
block = {"kind": "tool_use", "name": b.get("name"),
|
|
157
189
|
"input_summary": _summarize(b.get("input")),
|
|
190
|
+
"input": bounded, "input_truncated": input_trunc,
|
|
158
191
|
"id": b.get("id"),
|
|
159
192
|
"preview": tool_preview(b.get("name"), b.get("input"))}
|
|
160
193
|
inp = b.get("input")
|
|
161
194
|
st = inp.get("subagent_type") if isinstance(inp, dict) else None
|
|
162
195
|
if isinstance(st, str) and st: # #166: spawn kind (Agent/Task)
|
|
163
196
|
block["subagent_type"] = st
|
|
197
|
+
aux_parts.extend(_aux_strings(bounded))
|
|
164
198
|
blocks.append(block)
|
|
165
199
|
elif bt == "tool_result":
|
|
166
200
|
raw = _stringify(b.get("content"))
|
|
167
|
-
|
|
201
|
+
clipped = raw[:_TOOL_RESULT_CAP]
|
|
202
|
+
blocks.append({"kind": "tool_result", "text": clipped,
|
|
168
203
|
"truncated": len(raw) > _TOOL_RESULT_CAP,
|
|
204
|
+
"full_length": len(raw),
|
|
169
205
|
"is_error": bool(b.get("is_error")),
|
|
170
206
|
"tool_use_id": b.get("tool_use_id")})
|
|
207
|
+
aux_parts.append(clipped)
|
|
171
208
|
elif bt in ("image", "document"):
|
|
172
209
|
blocks.append({"kind": bt, **_media(b.get("source"))})
|
|
173
210
|
elif bt == "tool_reference":
|
|
174
211
|
blocks.append({"kind": "tool_reference", "name": b.get("name")})
|
|
175
|
-
return blocks, "\n".join(t for t in texts if t)
|
|
212
|
+
return (blocks, "\n".join(t for t in texts if t),
|
|
213
|
+
"\n".join(a for a in aux_parts if a))
|
|
176
214
|
|
|
177
215
|
|
|
178
216
|
_SUBAGENT_META_KEYS = (
|
|
@@ -212,6 +250,43 @@ def _attach_subagent_result(blocks, obj):
|
|
|
212
250
|
block["subagent_meta"] = meta
|
|
213
251
|
|
|
214
252
|
|
|
253
|
+
def _attach_ask_answers(blocks, obj):
|
|
254
|
+
"""Stash an AskUserQuestion's structured answers (+ annotations) onto its
|
|
255
|
+
single tool_result block (#177 S2), so the chosen option(s) have a robust
|
|
256
|
+
source the client can highlight without parsing the harness result string.
|
|
257
|
+
Self-identifying: fires only when toolUseResult carries an ``answers`` dict
|
|
258
|
+
(a key distinctive to AskUserQuestion), so no cross-record tool-name lookup
|
|
259
|
+
is needed. Same exactly-one-result-block guard as _attach_subagent_result.
|
|
260
|
+
|
|
261
|
+
answers/annotations are BOUNDED through _bound_input before storage — a
|
|
262
|
+
free-form "Other" answer or a long annotation note is attacker-controlled
|
|
263
|
+
free text, and every other free-text payload in this parser is capped
|
|
264
|
+
before it reaches blocks_json. Reusing _bound_input applies the same
|
|
265
|
+
five-axis cap (leaf/key/node/depth/total). The bound's truncation flag is
|
|
266
|
+
intentionally NOT surfaced (no answer-level "truncated" affordance — a
|
|
267
|
+
>8000-char option answer is not a realistic shape; the cap is a backstop).
|
|
268
|
+
|
|
269
|
+
An EMPTY answers dict is treated as no-capture (symmetric with the empty-
|
|
270
|
+
annotations drop below): emitting ``ask_answers={}`` would set a falsy
|
|
271
|
+
``answers`` on the tool_call and suppress the client's result-text
|
|
272
|
+
fallback, so a degenerate empty payload must no-op here instead."""
|
|
273
|
+
tur = obj.get("toolUseResult")
|
|
274
|
+
if not isinstance(tur, dict):
|
|
275
|
+
return
|
|
276
|
+
answers = tur.get("answers")
|
|
277
|
+
if not isinstance(answers, dict) or not answers: # require a non-empty dict
|
|
278
|
+
return
|
|
279
|
+
results = [b for b in blocks if b.get("kind") == "tool_result"]
|
|
280
|
+
if len(results) != 1:
|
|
281
|
+
return
|
|
282
|
+
bounded_ans, _ = _bound_input(answers)
|
|
283
|
+
results[0]["ask_answers"] = bounded_ans
|
|
284
|
+
anno = tur.get("annotations")
|
|
285
|
+
if isinstance(anno, dict) and anno:
|
|
286
|
+
bounded_anno, _ = _bound_input(anno)
|
|
287
|
+
results[0]["ask_annotations"] = bounded_anno
|
|
288
|
+
|
|
289
|
+
|
|
215
290
|
def _stringify(c):
|
|
216
291
|
if isinstance(c, str):
|
|
217
292
|
return c
|
|
@@ -233,6 +308,77 @@ def _summarize(inp):
|
|
|
233
308
|
return s[:200]
|
|
234
309
|
|
|
235
310
|
|
|
311
|
+
def _bound_input(inp):
|
|
312
|
+
"""Return (bounded_structured_input, truncated) for a tool_use input dict, or
|
|
313
|
+
(None, False) for a non-dict (the same non-dict contract as _summarize /
|
|
314
|
+
tool_preview). Hard-bounds the result on five axes so a pathological input
|
|
315
|
+
can't bloat blocks_json (Codex P1 + code-review I1): string leaves clip to
|
|
316
|
+
_INPUT_LEAF_CAP; dict keys clip to _INPUT_KEY_CAP (keys were stored verbatim
|
|
317
|
+
otherwise — the last unbounded axis); non-string scalars pass through; once
|
|
318
|
+
_INPUT_MAX_NODES dict-values/list-elements are kept the remainder elides to
|
|
319
|
+
_INPUT_ELISION; recursion past _INPUT_MAX_DEPTH elides the subtree
|
|
320
|
+
(RecursionError guard). A final _INPUT_TOTAL_CAP serialized-size check is the
|
|
321
|
+
honesty backstop. Structure is preserved for the kept prefix so downstream
|
|
322
|
+
renderers can read tool params."""
|
|
323
|
+
if not isinstance(inp, dict):
|
|
324
|
+
return (None, False)
|
|
325
|
+
state = {"nodes": 0, "truncated": False}
|
|
326
|
+
|
|
327
|
+
def walk(v, depth):
|
|
328
|
+
if depth > _INPUT_MAX_DEPTH:
|
|
329
|
+
state["truncated"] = True
|
|
330
|
+
return _INPUT_ELISION
|
|
331
|
+
if isinstance(v, str):
|
|
332
|
+
if len(v) > _INPUT_LEAF_CAP:
|
|
333
|
+
state["truncated"] = True
|
|
334
|
+
return v[:_INPUT_LEAF_CAP]
|
|
335
|
+
return v
|
|
336
|
+
if isinstance(v, dict):
|
|
337
|
+
out = {}
|
|
338
|
+
for k, vv in v.items():
|
|
339
|
+
ks = str(k)
|
|
340
|
+
if len(ks) > _INPUT_KEY_CAP: # clip pathological long keys (I1)
|
|
341
|
+
state["truncated"] = True
|
|
342
|
+
ks = ks[:_INPUT_KEY_CAP]
|
|
343
|
+
if state["nodes"] >= _INPUT_MAX_NODES:
|
|
344
|
+
state["truncated"] = True
|
|
345
|
+
out[ks] = _INPUT_ELISION
|
|
346
|
+
break
|
|
347
|
+
state["nodes"] += 1
|
|
348
|
+
out[ks] = walk(vv, depth + 1)
|
|
349
|
+
return out
|
|
350
|
+
if isinstance(v, list):
|
|
351
|
+
out = []
|
|
352
|
+
for vv in v:
|
|
353
|
+
if state["nodes"] >= _INPUT_MAX_NODES:
|
|
354
|
+
state["truncated"] = True
|
|
355
|
+
out.append(_INPUT_ELISION)
|
|
356
|
+
break
|
|
357
|
+
state["nodes"] += 1
|
|
358
|
+
out.append(walk(vv, depth + 1))
|
|
359
|
+
return out
|
|
360
|
+
# int / float / bool / None — bounded-width scalars, pass through
|
|
361
|
+
return v
|
|
362
|
+
|
|
363
|
+
bounded = walk(inp, 0)
|
|
364
|
+
if len(json.dumps(bounded, separators=(",", ":"))) > _INPUT_TOTAL_CAP:
|
|
365
|
+
state["truncated"] = True
|
|
366
|
+
return (bounded, state["truncated"])
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
def _aux_strings(v):
|
|
370
|
+
"""Yield string leaves from a bounded input value (for the search_aux blob)."""
|
|
371
|
+
if isinstance(v, str):
|
|
372
|
+
if v:
|
|
373
|
+
yield v
|
|
374
|
+
elif isinstance(v, dict):
|
|
375
|
+
for vv in v.values():
|
|
376
|
+
yield from _aux_strings(vv)
|
|
377
|
+
elif isinstance(v, list):
|
|
378
|
+
for vv in v:
|
|
379
|
+
yield from _aux_strings(vv)
|
|
380
|
+
|
|
381
|
+
|
|
236
382
|
_PREVIEW_FIELDS = {
|
|
237
383
|
"Read": "file_path", "Write": "file_path", "Edit": "file_path",
|
|
238
384
|
"MultiEdit": "file_path", "NotebookEdit": "file_path",
|