cctally 1.91.0 → 1.92.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/CHANGELOG.md +51 -0
  2. package/README.md +4 -2
  3. package/bin/_cctally_cache.py +903 -74
  4. package/bin/_cctally_config.py +57 -0
  5. package/bin/_cctally_core.py +94 -14
  6. package/bin/_cctally_dashboard.py +217 -19
  7. package/bin/_cctally_dashboard_conversation.py +170 -20
  8. package/bin/_cctally_dashboard_envelope.py +2 -0
  9. package/bin/_cctally_db.py +481 -19
  10. package/bin/_cctally_doctor.py +18 -1
  11. package/bin/_cctally_journal.py +1156 -21
  12. package/bin/_cctally_journal_repair.py +6 -0
  13. package/bin/_cctally_parser.py +26 -0
  14. package/bin/_cctally_quota.py +171 -55
  15. package/bin/_cctally_record.py +13 -1
  16. package/bin/_cctally_rederive.py +4 -0
  17. package/bin/_cctally_statusline.py +6 -6
  18. package/bin/_cctally_store.py +1061 -40
  19. package/bin/_cctally_transcript.py +32 -2
  20. package/bin/_cctally_tui.py +54 -6
  21. package/bin/_lib_cache_report.py +8 -3
  22. package/bin/_lib_codex_conversation.py +851 -81
  23. package/bin/_lib_codex_conversation_query.py +2031 -96
  24. package/bin/_lib_codex_find_projection.py +517 -0
  25. package/bin/_lib_codex_harness_preamble.py +176 -0
  26. package/bin/_lib_codex_hooks.py +5 -3
  27. package/bin/_lib_codex_js_scan.py +254 -0
  28. package/bin/_lib_codex_landmarks.py +309 -0
  29. package/bin/_lib_codex_title_clean.py +116 -0
  30. package/bin/_lib_conversation_dispatch.py +168 -22
  31. package/bin/_lib_conversation_query.py +62 -2
  32. package/bin/_lib_conversation_watch.py +4 -2
  33. package/bin/_lib_doctor.py +64 -0
  34. package/bin/_lib_quota_alert_axes.py +31 -34
  35. package/bin/_lib_stats_damage.py +523 -0
  36. package/bin/_lib_stats_publish.py +243 -0
  37. package/bin/cctally +17 -3
  38. package/dashboard/static/assets/index-Dat-mza6.js +97 -0
  39. package/dashboard/static/assets/{index-Dwirao3Y.css → index-DnWdv8um.css} +1 -1
  40. package/dashboard/static/dashboard.html +2 -2
  41. package/package.json +8 -1
  42. package/dashboard/static/assets/index-CILAoEja.js +0 -90
@@ -12,22 +12,44 @@ the caller at its I/O boundary, never here (§5.4).
12
12
  Public names imported verbatim by the S7 dispatch layer — do not rename:
13
13
  ``codex_normalization_authoritative``, ``codex_item_key``,
14
14
  ``get_codex_conversation``, ``get_codex_conversation_outline``,
15
- ``list_codex_conversations``, ``search_codex_conversations``,
15
+ ``list_codex_conversations``, ``list_codex_conversation_facets``,
16
+ ``search_codex_conversations``,
16
17
  ``CODEX_SEARCH_KINDS``.
17
18
  """
18
19
  from __future__ import annotations
19
20
 
21
+ import base64
22
+ import binascii
23
+ import collections
24
+ import contextlib
20
25
  import hashlib
21
26
  import json
22
27
  import os
23
28
  import re
24
29
  import sqlite3
30
+ import threading
31
+ from collections import deque
25
32
 
26
33
  import _lib_codex_conversation as kern
34
+ import _lib_codex_landmarks as landmarks
27
35
  import _lib_codex_segments as segkern
28
- from _lib_codex_reasoning_headings import decompose_reasoning_headings
36
+ from _lib_codex_find_projection import (
37
+ CODEX_FIND_PROJECTION_VERSION,
38
+ ProjectedLeaf,
39
+ RenderLeaf,
40
+ iter_literal_ranges,
41
+ iter_regex_ranges,
42
+ literal_ranges,
43
+ project_context,
44
+ project_markdown,
45
+ project_plain,
46
+ regex_ranges,
47
+ slice_range_to_leaves,
48
+ )
49
+ from _lib_codex_title_clean import clean_codex_title
29
50
  from _lib_conversation import _strip_ansi
30
- from _lib_conversation_query import _FULL_PAYLOAD_CEILING, _first_nonblank_line
51
+ from _lib_conversation_query import (
52
+ _FULL_PAYLOAD_CEILING, _first_nonblank_line, _parse_outline_ts)
31
53
  from _lib_pricing import _calculate_codex_entry_cost
32
54
 
33
55
  # ── constants ────────────────────────────────────────────────────────────────
@@ -389,6 +411,59 @@ def _load_rows_at_positions(
389
411
  return hydrated
390
412
 
391
413
 
414
+ def _iter_row_payloads(
415
+ conn: sqlite3.Connection, conversation_key: str, positions=None,
416
+ ):
417
+ """Yield ``(position, record_type, payload)`` for retained physical payloads.
418
+
419
+ ONE copy of the event-table read: the per-source-path chunking, the bound
420
+ ``IN (…)`` construction, the defensive re-filter and the
421
+ ``{"payload": …}`` unwrap. ``_load_row_payloads`` collects this into a dict
422
+ and ``_derive_outline_events`` consumes it one row at a time, which is the
423
+ only difference between them — duplicating the read to get the streaming
424
+ form would mean a later correction to the unwrap or the chunking has to be
425
+ made twice, and the second is easy to miss.
426
+
427
+ ``positions`` scopes the read to a known set (#463 S1, Phase C). Passing
428
+ ``None`` reads the whole conversation, which the export path still wants and
429
+ which #463 S4 §4.1 forbids on the outline route.
430
+
431
+ A row whose ``payload_json`` does not parse, or whose ``payload`` member is
432
+ not an object, yields nothing — the caller therefore sees no entry for that
433
+ position rather than an empty one.
434
+ """
435
+ def _batches():
436
+ """One cursor per bound chunk, opened only when its turn comes."""
437
+ if positions is None:
438
+ yield conn.execute(
439
+ "SELECT source_path,line_offset,record_type,payload_json "
440
+ "FROM codex_conversation_events WHERE conversation_key = ?",
441
+ (conversation_key,)), None
442
+ return
443
+ scope = set(positions)
444
+ for path, chunks in _chunk_positions(scope).items():
445
+ for chunk in chunks:
446
+ yield conn.execute(
447
+ "SELECT source_path,line_offset,record_type,payload_json "
448
+ "FROM codex_conversation_events "
449
+ "WHERE conversation_key = ? AND source_path = ? "
450
+ f"AND line_offset IN ({','.join('?' for _ in chunk)})",
451
+ (conversation_key, path, *chunk)), scope
452
+
453
+ for cursor, wanted in _batches():
454
+ for source_path, line_offset, record_type, payload_json in cursor:
455
+ position = (source_path, line_offset)
456
+ if wanted is not None and position not in wanted:
457
+ continue
458
+ try:
459
+ obj = json.loads(payload_json or "{}")
460
+ except (json.JSONDecodeError, TypeError):
461
+ continue
462
+ payload = obj.get("payload") if isinstance(obj, dict) else None
463
+ if isinstance(payload, dict):
464
+ yield position, record_type, payload
465
+
466
+
392
467
  def _load_row_payloads(
393
468
  conn: sqlite3.Connection, conversation_key: str, positions=None,
394
469
  ) -> dict[tuple[str, int], tuple[str | None, dict]]:
@@ -401,42 +476,198 @@ def _load_row_payloads(
401
476
  ``positions`` scopes the read to one page (#463 S1, Phase C). Passing None
402
477
  keeps the whole-conversation behaviour, which the export path still wants.
403
478
  """
404
- result: dict[tuple[str, int], tuple[str | None, dict]] = {}
405
- if positions is None:
406
- cursor = conn.execute(
407
- "SELECT source_path,line_offset,record_type,payload_json "
408
- "FROM codex_conversation_events WHERE conversation_key = ?",
409
- (conversation_key,))
410
- rows = list(cursor)
411
- else:
412
- wanted = set(positions)
413
- rows = []
414
- for path, chunks in _chunk_positions(wanted).items():
415
- for chunk in chunks:
416
- marks = ",".join("?" for _ in chunk)
417
- rows.extend(
418
- row for row in conn.execute(
419
- "SELECT source_path,line_offset,record_type,payload_json "
420
- "FROM codex_conversation_events "
421
- "WHERE conversation_key = ? AND source_path = ? "
422
- f"AND line_offset IN ({marks})",
423
- (conversation_key, path, *chunk))
424
- if (row[0], row[1]) in wanted)
425
- for source_path, line_offset, record_type, payload_json in rows:
426
- try:
427
- obj = json.loads(payload_json or "{}")
428
- except (json.JSONDecodeError, TypeError):
429
- continue
430
- payload = obj.get("payload") if isinstance(obj, dict) else None
431
- if isinstance(payload, dict):
432
- result[(source_path, line_offset)] = (record_type, payload)
433
- return result
479
+ return {
480
+ position: (record_type, payload)
481
+ for position, record_type, payload
482
+ in _iter_row_payloads(conn, conversation_key, positions)
483
+ }
434
484
 
435
485
 
436
486
  def _row_payload(row, payloads: dict) -> tuple[str | None, dict] | None:
437
487
  return payloads.get((row.source_path, row.line_offset))
438
488
 
439
489
 
490
+ # The three lifecycle events that carry an outcome the §6.4 classification reads.
491
+ # A `tool_output` row is the fourth source and is selected by `kind`, not by
492
+ # event type, because it is a response item rather than an event.
493
+ _S4_OUTCOME_EVENTS = frozenset(
494
+ {"patch_apply_end", "web_search_end", "mcp_tool_call_end"})
495
+
496
+
497
+ # ── the outline derivation cache (#463 S4, D2's fallback) ────────────────────
498
+ #
499
+ # Task 11's re-measurement breached §4.7's first ceiling. On the heaviest
500
+ # production conversation the warm outline costs 332 ms, of which the event
501
+ # payload pass is 242 ms, against 229 ms for the detail page that opens beside
502
+ # it — so the outline HAD become the critical path on open, and the route is
503
+ # refetched on every live-tail growth push. §4.7 says a breach escalates to D2's
504
+ # fallback inside the session rather than deferring it, and this is that.
505
+ #
506
+ # **Why a position-keyed cache is sound.** A derivation entry is keyed by
507
+ # ``(source_path, line_offset)`` — a byte offset into an append-only rollout
508
+ # file. A line at a given offset is immutable, so the verdict for a position
509
+ # cannot change while the row exists, and a re-ingest rewrites the same bytes at
510
+ # the same offsets. The cache therefore EXTENDS on append rather than only
511
+ # hitting or missing: a growth push decodes the newly-in-scope positions and
512
+ # reuses every earlier one.
513
+ #
514
+ # **What the watermark is for.** Immutability covers append; it does not cover
515
+ # DELETION, and a deleted event row is exactly the case where the outline must
516
+ # stop reporting a verdict it can no longer support. The stored watermark is the
517
+ # cached prefix's ``(row count, max id)``, and reuse requires the count of rows
518
+ # still at ``id <= max id`` to equal it. Ids are ``AUTOINCREMENT`` and only
519
+ # increase, so no later insert can land inside that prefix — a preserved count
520
+ # therefore proves no row of the prefix was deleted. The check is one covering
521
+ # index read, measured at 0.18 ms on a conversation with 5,950 event rows.
522
+ #
523
+ # **Absence stays absence.** A position that was in scope and produced no entry
524
+ # — payload gone, unparseable, or a shape no decoder recognises — is recorded as
525
+ # covered and is not retried. That is deliberate: ``_outline_error_count`` reads
526
+ # absence as the third state "could not classify", and a payload does not appear
527
+ # later at an offset it was missing from.
528
+ #
529
+ # In-process only, so no cross-version staleness is possible: the binary that
530
+ # filled an entry is the binary that reads it.
531
+ _OUTLINE_DERIVATION_CACHE_MAX = 4
532
+ _outline_derivation_cache: "collections.OrderedDict[str, dict]" = (
533
+ collections.OrderedDict())
534
+ _outline_derivation_lock = threading.Lock()
535
+
536
+
537
+ def reset_outline_derivation_cache() -> None:
538
+ """Drop every cached derivation. For tests and for measurement runs."""
539
+ with _outline_derivation_lock:
540
+ _outline_derivation_cache.clear()
541
+
542
+
543
+ def _outline_event_watermark(
544
+ conn: sqlite3.Connection, conversation_key: str, prefix_max_id: int | None,
545
+ ) -> tuple[int, int | None, int]:
546
+ """``(row count, max id, rows still at id <= prefix_max_id)`` in one read."""
547
+ count, max_id, prefix = conn.execute(
548
+ "SELECT COUNT(*), MAX(id), COALESCE(SUM(id <= ?), 0) "
549
+ "FROM codex_conversation_events WHERE conversation_key = ?",
550
+ (prefix_max_id if prefix_max_id is not None else -1, conversation_key),
551
+ ).fetchone()
552
+ return count, max_id, prefix
553
+
554
+
555
+ def _derive_outline_events(
556
+ conn: sqlite3.Connection, conversation_key: str, rows: list,
557
+ ) -> landmarks.EventDerivation:
558
+ """The outline's read-time pass over the retained payloads (#463 S4, §4.1).
559
+
560
+ SCOPED and STREAMING, and Task 1 measured both halves of that on the
561
+ production store rather than assuming them.
562
+
563
+ *Scoped*: the position set comes from ``rows`` — the wide
564
+ ``codex_conversation_messages`` read the outline has already performed — so
565
+ the pass never selects an event row no landmark can come from. That is the
566
+ idiom ``get_codex_conversation`` already uses two call sites above.
567
+ ``positions=None`` would decode every event row of the conversation, which on
568
+ the heaviest one is 92.1 MB of JSON. Measured, scoping is worth 24% of that
569
+ read (109-122 ms and 183.1 MB of Python heap against 160 ms and 211.7 MB) —
570
+ real, but far less than the spec's framing implies, because 93.7% of that
571
+ conversation's payload bytes ARE the rows S4 needs. Do not treat scoping as
572
+ having solved the cost.
573
+
574
+ *Streaming*: each payload is decoded to the small fact the outline wants and
575
+ then dropped, rather than being retained in a dict the way
576
+ ``_load_row_payloads`` retains it. Both go through the one
577
+ ``_iter_row_payloads`` read; retaining is the only thing that differs.
578
+ Measured like for like — load AND decode,
579
+ to completion — the retaining form costs 254 ms and 182.9 MB of Python heap
580
+ on the heaviest conversation and this one costs 232 ms and 22.7 MB. That is
581
+ 9% faster and 8x less memory, and peak RSS is in scope for §4.7's gate
582
+ precisely because this route is refetched on every live-tail growth push
583
+ rather than once per open.
584
+
585
+ Read-time only. Every decoder call takes ``for_storage=False`` and nothing
586
+ here is written back.
587
+ """
588
+ outcomes: dict[tuple[str, int], object] = {}
589
+ headings_wanted: dict[tuple[str, int], object] = {}
590
+ for row in rows:
591
+ position = (row.source_path, row.line_offset)
592
+ if row.kind == "tool_output":
593
+ outcomes[position] = "output"
594
+ elif row.kind == "event" and row.event_type in _S4_OUTCOME_EVENTS:
595
+ outcomes[position] = row.event_type
596
+ elif row.kind == "reasoning":
597
+ # The same stored-detail gate `_reasoning_headings` applies, so the
598
+ # outline's headings are the set the reader route publishes and not
599
+ # a second, wider one.
600
+ detail = _parse_detail(row.detail_json)
601
+ if isinstance(detail, dict) and isinstance(detail.get("reasoning"), dict):
602
+ headings_wanted[position] = True
603
+ wanted = set(outcomes) | set(headings_wanted)
604
+ derivation = landmarks.EventDerivation()
605
+ if not wanted:
606
+ return derivation
607
+
608
+ with _outline_derivation_lock:
609
+ entry = _outline_derivation_cache.get(conversation_key)
610
+ count, max_id, prefix = _outline_event_watermark(
611
+ conn, conversation_key, entry["max_id"] if entry else None)
612
+ reusable = (entry is not None
613
+ and entry["count"] == prefix
614
+ and count >= entry["count"])
615
+ covered: set = set()
616
+ if reusable:
617
+ covered = entry["covered"]
618
+ derivation.errors_by_position.update(entry["derivation"].errors_by_position)
619
+ derivation.patch_files_by_position.update(
620
+ entry["derivation"].patch_files_by_position)
621
+ derivation.headings_by_position.update(
622
+ entry["derivation"].headings_by_position)
623
+ missing = wanted - covered
624
+
625
+ for position, _record_type, payload in (
626
+ _iter_row_payloads(conn, conversation_key, missing) if missing else ()):
627
+ kind = outcomes.get(position)
628
+ if kind == "output":
629
+ decoded = kern.decode_tool_output_card(payload, for_storage=False)
630
+ if decoded is not None:
631
+ derivation.errors_by_position[position] = (
632
+ landmarks.classify_tool_failure(
633
+ {"terminal_output": decoded[0]}))
634
+ elif kind == "patch_apply_end":
635
+ card = kern.decode_patch_event_card(payload, for_storage=False)
636
+ if card is not None:
637
+ derivation.errors_by_position[position] = (
638
+ landmarks.classify_tool_failure({"patch": card}))
639
+ # Counted from the UNBOUNDED raw `changes`, not off the card above,
640
+ # whose shared 16,000-character budget silently undercounts a large
641
+ # diff (§4.5).
642
+ derivation.patch_files_by_position[position] = (
643
+ landmarks.patch_file_touches(payload))
644
+ elif kind in _S4_OUTCOME_EVENTS:
645
+ card = kern.decode_secondary_event_card(payload)
646
+ if card is not None:
647
+ family = "web" if kind == "web_search_end" else "mcp"
648
+ derivation.errors_by_position[position] = (
649
+ landmarks.classify_tool_failure(
650
+ {family: {"completion": card}}))
651
+ if position in headings_wanted:
652
+ texts = landmarks.reasoning_heading_texts(payload)
653
+ if texts:
654
+ derivation.headings_by_position[position] = texts
655
+
656
+ # Stored AFTER the pass, so a raising derivation leaves the previous entry
657
+ # rather than a half-filled one. The value is the object just returned: it
658
+ # is never mutated again, because the next extension copies its three maps
659
+ # into a fresh `EventDerivation` above rather than adding to this one.
660
+ with _outline_derivation_lock:
661
+ _outline_derivation_cache[conversation_key] = {
662
+ "count": count, "max_id": max_id,
663
+ "covered": covered | wanted, "derivation": derivation,
664
+ }
665
+ _outline_derivation_cache.move_to_end(conversation_key)
666
+ while len(_outline_derivation_cache) > _OUTLINE_DERIVATION_CACHE_MAX:
667
+ _outline_derivation_cache.popitem(last=False)
668
+ return derivation
669
+
670
+
440
671
  def _row_display(row) -> str:
441
672
  """The row's display/search text from whichever column carries it."""
442
673
  return row.text or row.search_thinking or row.search_tool or ""
@@ -470,6 +701,50 @@ def _item_kind(item: dict) -> str:
470
701
  return item["anchor_row"].kind # unturned: the row's own provider kind
471
702
 
472
703
 
704
+ def _item_model(item: dict) -> str | None:
705
+ """The model a canonical tier-1 item states, or ``None`` (§4.2).
706
+
707
+ The anchor row first, then the item's own rows in order. Reading the anchor
708
+ row ALONE under-reported Codex model usage about sixfold: most Codex
709
+ response items anchor on a ``reasoning`` row, which carries no model, so a
710
+ conversation with 13 outline turns holding assistant rows and 82 assistant
711
+ rows in total rendered ``gpt-5.6-sol x2``, and 182 of 200 Codex
712
+ conversations in the production store reported a model total under a third
713
+ of their turn count. §4.2 defines the counting unit as the canonical tier-1
714
+ assistant TURN, mirroring the Claude side, whose histogram sums to exactly
715
+ ``stats.turns.assistant``; the anchor row is one row of that turn.
716
+ """
717
+ anchor = item["anchor_row"].model
718
+ if anchor:
719
+ return anchor
720
+ for row in item["rows"]:
721
+ if row.model:
722
+ return row.model
723
+ return None
724
+
725
+
726
+ def _outline_outcome_positions(rows: list) -> set:
727
+ """Every position whose row could carry an outcome verdict this request."""
728
+ return {(row.source_path, row.line_offset) for row in rows
729
+ if row.kind == "tool_output"
730
+ or (row.kind == "event" and row.event_type in _S4_OUTCOME_EVENTS)}
731
+
732
+
733
+ def _outline_failing_calls(derivation, outcome_positions: set,
734
+ call_by_position: dict) -> set:
735
+ """The calls this request found failing, charged to the call they fold into.
736
+
737
+ Intersected with ``outcome_positions`` because ``errors_by_position`` is
738
+ CACHED: the derivation is retained per conversation and EXTENDED on a
739
+ growth push, so the map can hold verdicts for positions the current request
740
+ never read, while every other consumer indexes it by a current position.
741
+ Defensive rather than a reproduction of an observed miscount.
742
+ """
743
+ return {call_by_position.get(position, position)
744
+ for position in derivation.failing_positions()
745
+ if position in outcome_positions}
746
+
747
+
473
748
  def _item_meta(item: dict) -> dict | None:
474
749
  if item["klass"] != "meta":
475
750
  return None
@@ -518,37 +793,163 @@ def _reasoning_headings(detail, payload, block_key: str):
518
793
  returns ``None`` and the caller omits the field entirely, so the client falls
519
794
  back to today's ``title``/``summary`` rendering. Decomposition never fails the
520
795
  request and never partially populates.
796
+
797
+ #463 S4 — the summary parse itself moved to
798
+ ``_lib_codex_landmarks.reasoning_heading_texts``, so the reader route here
799
+ and the outline's landmark derivation decompose by ONE rule. This function
800
+ keeps the two things the outline does not want: the stored-detail gate, and
801
+ the ``<block_key>#<ordinal>`` identity, which the outline mints from its own
802
+ block keys rather than from the reader's.
521
803
  """
522
804
  if not isinstance(detail, dict) or not isinstance(detail.get("reasoning"), dict):
523
805
  return None
524
- if not isinstance(payload, dict):
525
- return None
526
- summary = payload.get("summary")
527
- if not isinstance(summary, list) or not summary:
528
- return None
529
- entries = []
530
- for entry in summary:
531
- if not isinstance(entry, dict):
532
- return None
533
- text = entry.get("text")
534
- # Mirror `_join_content_texts`, which is what produced the stored
535
- # summary: it keeps non-empty string `text` leaves and ignores the rest.
536
- if text is None:
537
- continue
538
- if not isinstance(text, str):
539
- return None
540
- if text:
541
- entries.append(text)
542
- headings = decompose_reasoning_headings(entries)
806
+ headings = landmarks.reasoning_heading_texts(payload)
543
807
  if not headings:
544
808
  return None
545
809
  return [{"key": f"{block_key}#{ordinal}", "text": text}
546
810
  for ordinal, text in enumerate(headings)]
547
811
 
548
812
 
813
+ # ── the conversation-level session index (#463 S3, spec section 3.2) ─────────
814
+ #
815
+ # Page-local adaptation cannot decide whether a session label is unique across
816
+ # the conversation or whether an opener exists, because later pages adapt
817
+ # independently and live-tail can append. So the server publishes a bounded
818
+ # conversation-scoped index and the client never computes either fact itself.
819
+ #
820
+ # 870 sessions across 223 conversations, roughly four per conversation, so this
821
+ # cap is generous. A conversation past it publishes what fits and marks itself
822
+ # truncated rather than publishing a partial map that looks complete.
823
+ _SESSION_INDEX_MAX = 64
824
+
825
+
826
+ def _stored_write_stdin_session(detail) -> str | None:
827
+ """The session id a `write_stdin` call names, from its STORED arguments.
828
+
829
+ Phase A must not load event payloads, and it does not need to: `detail.args`
830
+ is in the narrow index, and it is the provider's own argument JSON.
831
+ """
832
+ if not isinstance(detail, dict) or detail.get("name") != "write_stdin":
833
+ return None
834
+ args = detail.get("args")
835
+ if not isinstance(args, str) or not args:
836
+ return None
837
+ try:
838
+ parsed = json.loads(args)
839
+ except (json.JSONDecodeError, TypeError, ValueError):
840
+ return None
841
+ raw = parsed.get("session_id") if isinstance(parsed, dict) else None
842
+ if isinstance(raw, bool) or not isinstance(raw, (str, int)):
843
+ return None
844
+ return str(raw)
845
+
846
+
847
+ def _stored_session_announcement(detail) -> str | None:
848
+ """The session id a tool output ANNOUNCES, from its stored card.
849
+
850
+ `Process running with session ID <id>` is the evidence linking a shell
851
+ session to the call that opened it — 698 of 870 sessions, 80.2%. The line is
852
+ read through the same anchored preamble reader the card path uses rather than
853
+ by searching the text, so a user's own output cannot be mistaken for one.
854
+
855
+ Spec section 3.2 names `search_tool` as the column this comes from. It is
856
+ read from the stored card in `detail_json` instead, which carries the same
857
+ bytes at the head of its first part and IS in the narrow index —
858
+ `_load_conversation_index_rows` excludes `search_tool` along with the other
859
+ two bulk columns, and adding it back would undo S1's Phase A saving.
860
+ """
861
+ card = detail.get("card") if isinstance(detail, dict) else None
862
+ if not isinstance(card, dict) or card.get("type") != "terminal_output":
863
+ return None
864
+ parts = card.get("parts")
865
+ if not isinstance(parts, list) or not parts:
866
+ return None
867
+ head = parts[0]
868
+ text = head.get("text") if isinstance(head, dict) else None
869
+ if not isinstance(text, str):
870
+ return None
871
+ parsed = kern.parse_harness_preamble(text)
872
+ return parsed[0]["session_announcement"] if parsed is not None else None
873
+
874
+
875
+ def _build_session_index(rows) -> tuple[dict, dict[str, str]]:
876
+ """``(envelope, ordinal_by_provider_session_id)`` over the WHOLE conversation.
877
+
878
+ Ordinals are assigned in first-appearance order over the conversation's
879
+ physical row order, so they are stable across pages and live-tail appends and
880
+ no client-side uniqueness decision is made from a partial window.
881
+
882
+ The envelope's `sessions` map is keyed by the ordinal in decimal, which is
883
+ exactly what a `session_ref` card's `ref` carries, so the client's lookup is
884
+ direct. Nothing in the envelope is derived from the provider's own session
885
+ id — that token is removed rather than scrubbed (spec section 4.3).
886
+ """
887
+ owners: dict[str, list] = {}
888
+ for row in rows:
889
+ if row.kind == "tool_call" and row.call_id:
890
+ owners.setdefault(row.call_id, []).append(row)
891
+ ordinals: dict[str, int] = {}
892
+ openers: dict[str, str | None] = {}
893
+ truncated = False
894
+ for row in rows:
895
+ session = None
896
+ opener_row = None
897
+ detail = _parse_detail(row.detail_json)
898
+ if row.kind == "tool_call":
899
+ session = _stored_write_stdin_session(detail)
900
+ elif row.kind == "tool_output":
901
+ session = _stored_session_announcement(detail)
902
+ if session is not None:
903
+ # The opener is the CALL that owns the announcing output, not the
904
+ # output row: a uniquely-owned output folds into its call and has
905
+ # no block of its own, so its key would name nothing on the page.
906
+ owning = owners.get(row.call_id or "", [])
907
+ opener_row = owning[0] if len(owning) == 1 else row
908
+ if session is None:
909
+ continue
910
+ if session not in ordinals:
911
+ if len(ordinals) >= _SESSION_INDEX_MAX:
912
+ truncated = True
913
+ continue
914
+ ordinals[session] = len(ordinals) + 1
915
+ openers[session] = None
916
+ if opener_row is not None and openers.get(session) is None:
917
+ openers[session] = _block_key_for_row(opener_row)
918
+ envelope = {
919
+ "sessions": {
920
+ str(ordinal): {"ordinal": ordinal,
921
+ "opener_block_key": openers.get(session)}
922
+ for session, ordinal in ordinals.items()
923
+ },
924
+ "truncated": truncated,
925
+ }
926
+ return envelope, {session: str(ordinal) for session, ordinal in ordinals.items()}
927
+
928
+
929
+ def _apply_session_ordinals(card, ordinals: dict[str, str]) -> None:
930
+ """Replace every SHELL session reference with its conversation-local ordinal.
931
+
932
+ Fails closed: a reference the index does not know becomes ``None`` rather
933
+ than falling back to the provider's id. `cell` scope is left alone — a cell
934
+ id is a small per-conversation sandbox ordinal that identifies nothing
935
+ outside the sandbox, and it is never presented as a shell session.
936
+ """
937
+ if not isinstance(card, dict):
938
+ return
939
+ if card.get("type") == "session_ref" and card.get("scope") == "shell":
940
+ card["ref"] = ordinals.get(card.get("ref"))
941
+ return
942
+ if card.get("type") == "program":
943
+ for entry in card.get("invocations") or []:
944
+ if (isinstance(entry, dict) and entry.get("kind") == "session"
945
+ and entry.get("scope") == "shell"):
946
+ entry["ref"] = ordinals.get(entry.get("ref"))
947
+
948
+
549
949
  def _item_blocks_with_rows(
550
950
  item: dict, payloads: dict | None = None, *, preserve_marker_text: bool = False,
551
951
  call_owner_count: dict | None = None, decompose_headings: bool = False,
952
+ session_ordinals: dict[str, str] | None = None,
552
953
  ) -> list[list]:
553
954
  """Assemble an item's blocks (the historical ``_build_item_blocks`` behaviour)
554
955
  AND expose each block's underlying rows, so the detail renderer and the payload
@@ -597,11 +998,28 @@ def _item_blocks_with_rows(
597
998
  text = kern._join_content_texts(payload.get("content"))
598
999
  elif retained[0] == "event_msg":
599
1000
  text = kern._stringify(payload.get("message"))
1001
+ if r.kind == "assistant":
1002
+ # #463 S3 section 5.5. Read-time detection over the row's own stored
1003
+ # text, so it reaches every historical marker with no payload load.
1004
+ # It is written to `external_call` and never to `markers`, which
1005
+ # selects export payload hydration.
1006
+ external = kern._external_call_from_text(text)
1007
+ # Fail closed on the span: it is published as offsets into the very
1008
+ # string served as `block["text"]`, and a span that does not resolve
1009
+ # would make the client hide the wrong run of prose. `text` is the
1010
+ # same object the block below carries, including the
1011
+ # `preserve_marker_text` replacement, so the check is against what is
1012
+ # actually served rather than against what was read.
1013
+ if external is not None and kern.external_call_span_resolves(
1014
+ text, external):
1015
+ detail = dict(detail) if isinstance(detail, dict) else {}
1016
+ detail["external_call"] = external
600
1017
  if r.kind == "tool_call" and isinstance(payload, dict):
601
1018
  card = kern.decode_tool_call_card(payload)
602
1019
  if card is None:
603
1020
  card = kern.decode_secondary_tool_call_card(payload)
604
1021
  if card is not None:
1022
+ _apply_session_ordinals(card, session_ordinals or {})
605
1023
  detail = dict(detail) if isinstance(detail, dict) else {}
606
1024
  detail["card"] = card
607
1025
  output_card = (kern.decode_tool_output_card(payload)
@@ -792,6 +1210,7 @@ def _item_blocks_with_rows(
792
1210
  def _build_item_blocks(
793
1211
  item: dict, payloads: dict | None = None, *, preserve_marker_text: bool = False,
794
1212
  call_owner_count: dict | None = None, decompose_headings: bool = False,
1213
+ session_ordinals: dict[str, str] | None = None,
795
1214
  ) -> list[dict]:
796
1215
  """Assemble an item's blocks, folding each ``tool_output`` into its
797
1216
  ``tool_call`` block via ``call_id`` when that call_id has exactly one owner
@@ -800,7 +1219,8 @@ def _build_item_blocks(
800
1219
  return [entry[0] for entry in _item_blocks_with_rows(
801
1220
  item, payloads, preserve_marker_text=preserve_marker_text,
802
1221
  call_owner_count=call_owner_count,
803
- decompose_headings=decompose_headings)]
1222
+ decompose_headings=decompose_headings,
1223
+ session_ordinals=session_ordinals)]
804
1224
 
805
1225
 
806
1226
  def _item_lifecycle(item: dict) -> dict | None:
@@ -912,9 +1332,32 @@ def _attribute_costs(conn: sqlite3.Connection, conversation_key: str, effective_
912
1332
  return turn_cost, turn_tokens, unattr_cost, unattr_tokens, total, conv_tokens
913
1333
 
914
1334
 
1335
+ def _conversation_totals(
1336
+ conn: sqlite3.Connection, conversation_key: str, effective_speed: str,
1337
+ ) -> tuple[float, dict]:
1338
+ """Lean priced and token totals over one conversation's accounting rows.
1339
+
1340
+ Unlike ``_attribute_costs``, this does not reconstruct the event-to-turn map:
1341
+ callers that need only conversation totals (outline, browse, child summaries)
1342
+ can sum the compact accounting rows directly. The row order and pricing
1343
+ primitive stay identical to the detail envelope's whole-conversation pass.
1344
+ """
1345
+ total = 0.0
1346
+ tokens = _zero_tokens()
1347
+ for model, inp, cin, out, rout in conn.execute(
1348
+ "SELECT model, input_tokens, cached_input_tokens, output_tokens, "
1349
+ "reasoning_output_tokens FROM codex_session_entries WHERE conversation_key = ? "
1350
+ "ORDER BY source_path, line_offset",
1351
+ (conversation_key,),
1352
+ ):
1353
+ total += _calculate_codex_entry_cost(
1354
+ model or "", inp or 0, cin or 0, out or 0, rout or 0, speed=effective_speed)
1355
+ _add_tokens(tokens, inp, out, cin, rout)
1356
+ return total, _tokens_union(tokens)
1357
+
1358
+
915
1359
  def _conversation_total_cost(conn: sqlite3.Connection, conversation_key: str, effective_speed: str) -> float:
916
- """Lean priced total over a conversation's accounting rows (browse rows,
917
- child summaries) — same primitive as ``_attribute_costs`` (§5.4)."""
1360
+ """Lean priced total for browse rows and child summaries (§5.4)."""
918
1361
  total = 0.0
919
1362
  for model, inp, cin, out, rout in conn.execute(
920
1363
  "SELECT model, input_tokens, cached_input_tokens, output_tokens, "
@@ -997,9 +1440,19 @@ def _short_native(native: str | None) -> str:
997
1440
 
998
1441
  def _display_chain(fields: dict) -> str:
999
1442
  """Read-time display fallback (§4.3): stored title → project_label → short
1000
- native-thread-id prefix."""
1001
- return fields.get("title") or fields.get("project_label") or _short_native(
1002
- fields.get("native_thread_id")) or ""
1443
+ native-thread-id prefix.
1444
+
1445
+ #463 S4 §5 — the stored title is cleaned of recognized harness markup here.
1446
+ This is ONE of three read paths that need it, not the universal chokepoint
1447
+ the first draft assumed: the outline turn label is built independently from
1448
+ anchor-row text and the `kind=title` search path reads rollup titles
1449
+ directly, so both clean through the same helper rather than through this
1450
+ call. A construct that strips to nothing falls through the chain below on
1451
+ its own, which is what makes `strip` expressible at read time at all.
1452
+ """
1453
+ return (clean_codex_title(fields.get("title"))
1454
+ or fields.get("project_label")
1455
+ or _short_native(fields.get("native_thread_id")) or "")
1003
1456
 
1004
1457
 
1005
1458
  def _conversation_display_title(conn: sqlite3.Connection, conversation_key: str, rows: list | None = None) -> str:
@@ -1538,6 +1991,7 @@ def _fold_groups_for_item(item: dict, call_owner_count: dict,
1538
1991
  def _build_segment_index(
1539
1992
  conversation_key: str, items: list[dict], detail_bytes: dict, *,
1540
1993
  segmented: bool, block_budget: int | None = None,
1994
+ fold_groups: bool = False,
1541
1995
  ) -> list[dict]:
1542
1996
  """Phase A's output: an ordered index of segments, with no block content.
1543
1997
 
@@ -1554,6 +2008,12 @@ def _build_segment_index(
1554
2008
  when omitted, never as a default argument value: a default argument binds
1555
2009
  once at import, so a test that lowered the budget would silently keep the
1556
2010
  imported figure and pass vacuously.
2011
+
2012
+ ``fold_groups`` publishes the fold-group membership as ``_fold_groups``. Only
2013
+ the outline reads it (#463 S4 — a ``tool_error`` landmark anchors on the call
2014
+ a failure folds into); the detail route S1 bounded and the search position
2015
+ map do not, and building it for them costs a tuple per row per request for a
2016
+ value nobody reads.
1557
2017
  """
1558
2018
  index: list[dict] = []
1559
2019
  for item_index, item in enumerate(items):
@@ -1605,6 +2065,16 @@ def _build_segment_index(
1605
2065
  "_turn_id": item["turn_id"],
1606
2066
  "_anchor_row": anchor,
1607
2067
  "_rows": segment_rows,
2068
+ # #463 S4 — the fold-group membership, as physical positions.
2069
+ # `_fold_groups_for_item` computes it payload-free and this
2070
+ # index discarded it, so nothing downstream could say WHICH
2071
+ # `tool_call` a failing `tool_output` belongs to — only that the
2072
+ # segment contained one. A `tool_error` landmark anchors on the
2073
+ # call, so the membership has to survive Phase A.
2074
+ "_fold_groups": [
2075
+ [(row.source_path, row.line_offset) for row in group.rows]
2076
+ for group in segment.groups
2077
+ ] if fold_groups else (),
1608
2078
  "_call_owner_count": call_owner_count,
1609
2079
  "_meta": _item_meta(item) if head else None,
1610
2080
  "_lifecycle": _item_lifecycle(item) if head else None,
@@ -1690,6 +2160,11 @@ def get_codex_conversation(
1690
2160
  # byte-identical to what the export golden already pins.
1691
2161
  index = _build_segment_index(
1692
2162
  conversation_key, items, detail_bytes, segmented=not legacy_export)
2163
+ # #463 S3 section 3.2. Built here, in Phase A, from the narrow index that is
2164
+ # already loaded for the whole conversation: it needs no extra payload read
2165
+ # and no extra column, and it must be whole-conversation because ordinals and
2166
+ # opener presence are global facts a page cannot decide.
2167
+ session_index, session_ordinals = _build_session_index(rows)
1693
2168
 
1694
2169
  # ── Phase B: paginate the index ──────────────────────────────────────────
1695
2170
  page_index, page = _paginate_items(
@@ -1761,7 +2236,8 @@ def get_codex_conversation(
1761
2236
  # marker-bearing payloads, so populating `headings` there would
1762
2237
  # force a whole-conversation payload read for a field the
1763
2238
  # exporter never reads.
1764
- decompose_headings=not legacy_export),
2239
+ decompose_headings=not legacy_export,
2240
+ session_ordinals=session_ordinals),
1765
2241
  "cost_usd": cost,
1766
2242
  "tokens": tokens,
1767
2243
  }
@@ -1782,6 +2258,7 @@ def get_codex_conversation(
1782
2258
  "title": _conversation_display_title(conn, conversation_key),
1783
2259
  "items": page_items,
1784
2260
  "page": page,
2261
+ "session_index": session_index,
1785
2262
  "children": _children_of(conn, conversation_key, effective_speed),
1786
2263
  "parent": _parent_of(conn, conversation_key),
1787
2264
  "total_cost_usd": total,
@@ -1793,15 +2270,299 @@ def get_codex_conversation(
1793
2270
  # ── outline assembly (§5.6) ───────────────────────────────────────────────────
1794
2271
 
1795
2272
 
1796
- def _conversation_files(conn: sqlite3.Connection, conversation_key: str) -> list[dict]:
1797
- return [
1798
- {"file_path": fp, "tool": tool, "count": count}
1799
- for fp, tool, count in conn.execute(
1800
- "SELECT file_path, tool, COUNT(*) FROM codex_conversation_file_touches "
1801
- "WHERE conversation_key = ? GROUP BY file_path, tool ORDER BY file_path, tool",
1802
- (conversation_key,),
1803
- )
1804
- ]
2273
+ def _tool_call_name(row) -> str | None:
2274
+ """The tool a ``tool_call`` row invoked, from its STORED detail.
2275
+
2276
+ Ingest writes ``{"name": payload["name"] or <record type>, …}`` for every
2277
+ tool call, so this needs no payload and stays outside the scoped pass.
2278
+ """
2279
+ detail = _parse_detail(row.detail_json)
2280
+ name = detail.get("name") if isinstance(detail, dict) else None
2281
+ return name if isinstance(name, str) and name else None
2282
+
2283
+
2284
+ def _conversation_duration_seconds(rows) -> int | None:
2285
+ """Wall span of the conversation, as MIN to MAX over row timestamps (§4.2).
2286
+
2287
+ Never last minus first. §3.4 records that ``timestamp_utc`` is monotone
2288
+ within a turn only — item and segment emission is physical order, not
2289
+ timestamp order — and Task 1 found five decreases across turns in the
2290
+ corpus, on which the naive form returns a negative duration.
2291
+
2292
+ The outline's own caller cannot exhibit that today, because
2293
+ ``_load_conversation_rows`` reads ``ORDER BY timestamp_utc, source_path,
2294
+ line_offset`` and the two forms therefore coincide on the rows it passes.
2295
+ The rule is stated for the next caller, which is much more likely to hand
2296
+ over an item-anchor list.
2297
+ """
2298
+ stamps = [row.timestamp_utc for row in rows if row.timestamp_utc]
2299
+ if not stamps:
2300
+ return None
2301
+ first = _parse_outline_ts(min(stamps))
2302
+ last = _parse_outline_ts(max(stamps))
2303
+ if first is None or last is None:
2304
+ return None
2305
+ return int((last - first).total_seconds())
2306
+
2307
+
2308
+ def _outline_error_count(
2309
+ failed_calls: set, outcome_positions: set,
2310
+ derivation: landmarks.EventDerivation,
2311
+ ) -> int | None:
2312
+ """How many calls failed, or ``None`` when that cannot be answered (D3).
2313
+
2314
+ Three states, because 0 and null are different claims. A conversation with
2315
+ no outcome-bearing row at all reports 0: nothing failed because nothing ran,
2316
+ and that is determinable. A conversation whose outcome rows produced no
2317
+ verdict at all — every retained payload gone, unparseable, or of a shape no
2318
+ decoder recognises — reports null, because the stored projection answers
2319
+ nothing here (Task 1 measured stored ``is_error`` true for 0 of 63,150
2320
+ production ``tool_output`` rows) and 0 would assert an absence nobody proved.
2321
+
2322
+ A PARTIAL read reports what it found rather than declining. The alternative
2323
+ would suppress real failures the pass did see because of one unreadable
2324
+ neighbour, which is the worse error of the two.
2325
+ """
2326
+ if not outcome_positions:
2327
+ return 0
2328
+ if not (outcome_positions & set(derivation.errors_by_position)):
2329
+ return None
2330
+ return len(failed_calls)
2331
+
2332
+
2333
+ # The Codex tool names that open the `plan` landmark family. Codex's decoded
2334
+ # plan card is named `update_plan`, and both existing CLIENT plan predicates
2335
+ # recognise only Claude's `ExitPlanMode` and `AskUserQuestion` — so publishing
2336
+ # raw Codex tool names into tier-1 `tools` would NOT have made the plan jump
2337
+ # work, it would have been a silent no-op (§3.2). The mapping is explicit here
2338
+ # and the outline target derivation reads landmark KINDS rather than inferring
2339
+ # from names.
2340
+ _S4_PLAN_TOOLS = frozenset({"update_plan"})
2341
+
2342
+
2343
+ def _landmark_label(row) -> str:
2344
+ """What a landmark row says. Never a raw provider identifier (§8).
2345
+
2346
+ §3.6 enumerates exactly TWO sources for a landmark label — reasoning heading
2347
+ text, or a tool name — and rests the decision not to scrub these labels on
2348
+ that enumeration. So a row that is neither a named ``tool_call`` nor a typed
2349
+ ``event`` falls back to its own KIND, which is normalizer vocabulary, and
2350
+ never to the row's stored text.
2351
+
2352
+ That branch is reachable: a failing ``tool_output`` whose ``call_id`` is
2353
+ owned by two or more ``tool_call`` rows in its turn does not fold, becomes
2354
+ its own group head, and enters ``failed_calls`` directly. Its ``text`` column
2355
+ is the harness preamble that ``decode_tool_output_card(for_storage=False)``
2356
+ exists to remove, and ``test_s3_no_raw_session_id_reaches_any_served_route``
2357
+ documents that preamble as carrying the provider ``session_id``.
2358
+ """
2359
+ if row.kind == "tool_call":
2360
+ name = _tool_call_name(row)
2361
+ if name is not None:
2362
+ return name
2363
+ elif row.kind == "event" and row.event_type:
2364
+ return row.event_type
2365
+ return row.kind or ""
2366
+
2367
+
2368
+ def _clean_outline_label(text: str) -> str:
2369
+ """An outline turn label, cleaned, and never cleaned away to nothing (§5.1).
2370
+
2371
+ Unlike ``_display_chain``, which §5.3 leans on as "already a fallback chain"
2372
+ when it justifies the ``strip`` disposition, this path has no chain: it
2373
+ cleans the anchor row's first non-blank line and publishes the result. Two
2374
+ allowlisted grammars can consume the whole string — ``<recommended_plugins>``
2375
+ (6 of the census's 438 titles) and ``<command-name>`` with no sibling tag —
2376
+ and the client's ``cleanQualifiedTitle(turn.label) ?? turn.label`` passes an
2377
+ empty string straight through, so the reader would get a row with no text at
2378
+ all. The uncleaned line is the pre-S4 label, which is legible.
2379
+ """
2380
+ cleaned = clean_codex_title(text)
2381
+ return cleaned if cleaned.strip() else text
2382
+
2383
+
2384
+ def _build_landmarks(
2385
+ index: list[dict], derivation: landmarks.EventDerivation,
2386
+ failed_calls: set,
2387
+ ) -> list[dict]:
2388
+ """Tier 2 — the landmarks a jump can reach (§3.2).
2389
+
2390
+ Three kinds and deliberately NOT one entry per tool call: a 523-call turn
2391
+ would contribute 523 rows, which is noise rather than navigation.
2392
+
2393
+ Emission is PHYSICAL order — the segment index in order, and each segment's
2394
+ rows in order — because §3.4 records that ``timestamp_utc`` is monotone
2395
+ within a turn only and no consumer sorts by it.
2396
+
2397
+ ``landmark_key`` is always COMPOUND — ``<block_key>#<discriminator>`` — and
2398
+ unique across every kind. A reasoning heading discriminates by its zero-based
2399
+ ordinal, which is the identity the reader route already mints for the same
2400
+ heading, because one block yields several headings and ``block_key`` alone
2401
+ would collide. Every other kind discriminates by the kind itself.
2402
+
2403
+ That is what lets one ``tool_call`` block carry BOTH a ``tool_error`` and a
2404
+ ``plan`` landmark. It has to: §3.2 gives the plan kind one entry per plan
2405
+ call, and a failed plan call is one, so filing it only as the error made the
2406
+ jump cluster's plan family report zero — asserting no plan activity in a
2407
+ conversation that has some, which is the claim the spec's own "0 is a claim,
2408
+ hiding is not" rule forbids. The error is emitted first, because it is the
2409
+ more urgent of the two claims about the same call.
2410
+
2411
+ A block carrying ``detail.external_call`` produces no landmark of any kind
2412
+ (§3.2). That holds by construction rather than by a filter here: the marker
2413
+ is published on ``assistant`` blocks only, and no kind below comes from an
2414
+ assistant row. ``test_external_call_block_produces_no_landmark`` pins it.
2415
+ """
2416
+ out: list[dict] = []
2417
+ for entry in index:
2418
+ for row in entry["_rows"]:
2419
+ position = (row.source_path, row.line_offset)
2420
+ block_key = _block_key_for_row(row)
2421
+ common = {
2422
+ "block_key": block_key,
2423
+ "item_key": entry["item_key"],
2424
+ "parent_item_key": entry["turn_item_key"],
2425
+ "timestamp_utc": row.timestamp_utc,
2426
+ }
2427
+ if row.kind == "reasoning":
2428
+ for ordinal, text in enumerate(
2429
+ derivation.headings_by_position.get(position, ())):
2430
+ out.append({"landmark_key": f"{block_key}#{ordinal}",
2431
+ "kind": "reasoning", "label": text, **common})
2432
+ continue
2433
+ if position in failed_calls:
2434
+ out.append({"landmark_key": f"{block_key}#tool_error",
2435
+ "kind": "tool_error",
2436
+ "label": _landmark_label(row), **common})
2437
+ if (row.kind == "tool_call"
2438
+ and _tool_call_name(row) in _S4_PLAN_TOOLS):
2439
+ out.append({"landmark_key": f"{block_key}#plan", "kind": "plan",
2440
+ "label": _landmark_label(row), **common})
2441
+ return out
2442
+
2443
+
2444
+ # The literal ingest writes into `codex_conversation_file_touches.tool`, kept so
2445
+ # the outline wire field and the file-search projection mean the same thing.
2446
+ # Every touch S4 derives comes from a `patch_apply_end`, which is the completion
2447
+ # of an `apply_patch` call.
2448
+ _PATCH_TOUCH_TOOL = "apply_patch"
2449
+
2450
+
2451
+ def _conversation_files(
2452
+ segment_index: list[dict], derivation: landmarks.EventDerivation,
2453
+ ) -> list[dict]:
2454
+ """The whole-conversation file list, DERIVED read-time (§1.2, §4.3).
2455
+
2456
+ The stored ``codex_conversation_file_touches`` table is the source for
2457
+ cross-conversation ``kind=files`` search after #489 repaired dict-shaped
2458
+ ingest and backfilled retained history. It is deliberately not the outline
2459
+ source: this payload pass has the richer evidence the outline contract needs.
2460
+
2461
+ Deriving it instead buys three things the table could not have supplied: a
2462
+ real segment anchor per touch, so a file row jumps to its change rather than
2463
+ to the top of a turn; the true per-file count; and first-touch DOCUMENT
2464
+ order, which is what ``OutlineFile`` has always promised while the SQL
2465
+ ordered alphabetically by path.
2466
+
2467
+ ``added``/``removed`` are summed over the touches, and go ``None`` as soon as
2468
+ ONE touch of that file cannot be counted. Summing only the countable touches
2469
+ would publish a number for a file that changed more — a file edited once with
2470
+ a real diff and then moved by a count-free ``update`` would report the first
2471
+ figure — and nothing in ``touches[]`` marks such a total as partial. §4.5 is
2472
+ explicit that an undeterminable count is null and the badge renders nothing
2473
+ rather than an understated number; that rule has to reach the aggregate, not
2474
+ only the individual touch. The per-touch counts themselves come from the
2475
+ UNBOUNDED raw ``changes`` entry (§4.5) — see ``landmarks.patch_file_touches``.
2476
+ """
2477
+ files: dict[str, dict] = {}
2478
+ undetermined: dict[str, set[str]] = {}
2479
+ for entry in segment_index:
2480
+ for row in entry["_rows"]:
2481
+ position = (row.source_path, row.line_offset)
2482
+ for touch in derivation.patch_files_by_position.get(position, ()):
2483
+ record = files.get(touch["path"])
2484
+ if record is None:
2485
+ record = files[touch["path"]] = {
2486
+ "file_path": touch["path"], "tool": _PATCH_TOUCH_TOOL,
2487
+ "count": 0, "touches": [],
2488
+ "added": None, "removed": None}
2489
+ record["count"] += 1
2490
+ record["touches"].append({
2491
+ "item_key": entry["item_key"],
2492
+ "timestamp_utc": row.timestamp_utc,
2493
+ # The raw change KIND — `add`/`delete`/`update` from the dict
2494
+ # shape, `modified` from the list one — never the tool name.
2495
+ "op": touch["op"],
2496
+ })
2497
+ for field in ("added", "removed"):
2498
+ if touch[field] is None:
2499
+ undetermined.setdefault(touch["path"], set()).add(field)
2500
+ else:
2501
+ record[field] = (record[field] or 0) + touch[field]
2502
+ for path, fields in undetermined.items():
2503
+ for field in fields:
2504
+ files[path][field] = None
2505
+ return list(files.values())
2506
+
2507
+
2508
+ # Connections whose read snapshot THIS module opened, by identity. A
2509
+ # ``sqlite3.Connection`` supports neither attribute assignment nor a weak
2510
+ # reference, so ownership cannot be recorded on the object; ``id`` is unique
2511
+ # among live objects and the connection is alive for the whole ``with`` body, so
2512
+ # the token cannot be confused with another connection's while it is registered.
2513
+ _OWNED_READ_SNAPSHOTS: set[int] = set()
2514
+
2515
+
2516
+ @contextlib.contextmanager
2517
+ def _read_snapshot(conn: sqlite3.Connection):
2518
+ """One consistent read snapshot across a multi-query envelope (#463 S4 §4.1).
2519
+
2520
+ The outline route uses one connection but opened no explicit read
2521
+ transaction, so its several queries each took their own snapshot. Concurrent
2522
+ APPEND is benign there — extra raw event rows have no normalized mapping yet
2523
+ — but a concurrent delete or truncation between the wide message read and the
2524
+ payload read can expose a message row whose payload is already gone, and the
2525
+ derivation would then report an absence that never existed.
2526
+
2527
+ A deferred ``BEGIN`` takes the snapshot on the first read and holds it for
2528
+ every later one. It is released with ``rollback``, which is the honest end of
2529
+ a transaction that wrote nothing.
2530
+
2531
+ **A transaction this module did not open is refused, not inherited.**
2532
+ ``conn.in_transaction`` is true for an outer WRITE transaction exactly as it
2533
+ is for an outer read snapshot, and Python's ``sqlite3`` exposes no
2534
+ ``txn_state``, so the two cannot be told apart here. Only one of them is safe
2535
+ to borrow: inside a write, the envelope would read that writer's uncommitted
2536
+ and possibly half-applied state — a message row whose events are already
2537
+ deleted — with no snapshot of its own and no way to notice. Treating both
2538
+ alike is silent; refusing is not. A caller that wants several envelopes on
2539
+ one snapshot opens it through this same helper, which nests without issuing
2540
+ the second ``BEGIN`` SQLite would refuse.
2541
+
2542
+ The caller sweep behind that decision, pinned by
2543
+ ``test_every_outline_caller_arrives_outside_a_transaction``: the three call
2544
+ paths into ``get_codex_conversation_outline`` are
2545
+ ``_lib_conversation_dispatch.neutral_outline`` (the dashboard route, on a
2546
+ connection ``open_conversations_db`` returns fresh per request and closes
2547
+ after), ``bin/build-codex-reader-fixtures.py``, and the tests. None holds a
2548
+ transaction at the call.
2549
+ """
2550
+ token = id(conn)
2551
+ if token in _OWNED_READ_SNAPSHOTS:
2552
+ yield
2553
+ return
2554
+ if conn.in_transaction:
2555
+ raise RuntimeError(
2556
+ "this envelope needs its own read snapshot, and the connection is "
2557
+ "already inside a transaction it did not open; wrap the outer "
2558
+ "scope in _read_snapshot instead")
2559
+ conn.execute("BEGIN")
2560
+ _OWNED_READ_SNAPSHOTS.add(token)
2561
+ try:
2562
+ yield
2563
+ finally:
2564
+ _OWNED_READ_SNAPSHOTS.discard(token)
2565
+ conn.rollback()
1805
2566
 
1806
2567
 
1807
2568
  def get_codex_conversation_outline(
@@ -1827,7 +2588,24 @@ def get_codex_conversation_outline(
1827
2588
  report true for a segment that has not been fetched, so the drain would never
1828
2589
  run and the jump would land nowhere. Membership for navigation and membership
1829
2590
  for "this item subsumes that key" are different relations.
2591
+
2592
+ #463 S4 — the route now makes TWO conversation reads under one snapshot: the
2593
+ wide message read below, and a scoped read-time pass over the retained event
2594
+ payloads (``_derive_outline_events``) whose position set comes from that
2595
+ first read. The pass is what gives the outline a failure verdict per call,
2596
+ the authored reasoning headings, and the per-file patch touches; §1.2 records
2597
+ why the stored ``codex_conversation_file_touches`` search projection is not
2598
+ the OUTLINE source, and Task 1 measured that the stored card carries 0 of the
2599
+ corpus's 896 tool failures, so read-time is not a preference here.
1830
2600
  """
2601
+ with _read_snapshot(conn):
2602
+ return _outline_envelope(
2603
+ conn, conversation_key, effective_speed=effective_speed)
2604
+
2605
+
2606
+ def _outline_envelope(
2607
+ conn: sqlite3.Connection, conversation_key: str, *, effective_speed: str
2608
+ ) -> dict:
1831
2609
  if not codex_normalization_authoritative(conn):
1832
2610
  return {"status": "normalization_pending", "conversation_key": conversation_key,
1833
2611
  "turns": [], "files": [], "children": []}
@@ -1841,11 +2619,26 @@ def get_codex_conversation_outline(
1841
2619
  kept, _suppressed = kern.pair_mirrors(rows)
1842
2620
  items = kern.canonical_items(kept)
1843
2621
  segment_keys: dict[int, list[str]] = {}
1844
- for entry in _build_segment_index(
1845
- conversation_key, items, detail_bytes, segmented=True):
2622
+ fold_groups: list[list[tuple[str, int]]] = []
2623
+ segment_index = _build_segment_index(
2624
+ conversation_key, items, detail_bytes, segmented=True, fold_groups=True)
2625
+ for entry in segment_index:
1846
2626
  segment_keys.setdefault(entry["_item_index"], []).append(entry["item_key"])
2627
+ fold_groups.extend(entry["_fold_groups"])
2628
+ # `rows` here is the wide read directly above, and that is what makes the
2629
+ # payload pass SCOPED rather than a second whole-conversation decode (§4.1).
2630
+ derivation = _derive_outline_events(conn, conversation_key, rows)
2631
+ call_by_position = landmarks.fold_owner_by_position(fold_groups)
2632
+ outcome_positions = _outline_outcome_positions(rows)
2633
+ # A failing outcome row is charged to the `tool_call` it folds into, so the
2634
+ # same failure cannot be counted twice when a call and its output both carry
2635
+ # one, and so a turn's `tools` entry can say WHICH call failed.
2636
+ failed_calls = _outline_failing_calls(
2637
+ derivation, outcome_positions, call_by_position)
1847
2638
  turns: list[dict] = []
1848
2639
  kind_totals: dict[str, int] = {}
2640
+ tool_counts: dict[str, int] = {}
2641
+ models: dict[str, int] = {}
1849
2642
  # Keyed on the ITEM index, which is what _build_segment_index records. Using
1850
2643
  # ``len(turns)`` would be correct only for as long as this loop appends a
1851
2644
  # turn for every item without exception; a later ``continue`` would misalign
@@ -1857,11 +2650,38 @@ def get_codex_conversation_outline(
1857
2650
  if meta is not None:
1858
2651
  label = _META_LABEL_TEXT.get(meta["meta_label"], "Harness context")
1859
2652
  else:
1860
- label = _first_nonblank_line(_strip_ansi(anchor_text)) if anchor_text else ""
2653
+ # Built from anchor-row TEXT, which is why it does not reach
2654
+ # `_display_chain` and has to clean through the shared helper here
2655
+ # (§5.1). A label with no recognized markup passes through byte for
2656
+ # byte, so this cannot move an ordinary prose label.
2657
+ label = _clean_outline_label(
2658
+ _first_nonblank_line(_strip_ansi(anchor_text))) if anchor_text else ""
1861
2659
  kinds: dict[str, int] = {}
2660
+ tools: list[dict] = []
2661
+ tool_slot: dict[str | None, int] = {}
2662
+ tool_call_count = 0
2663
+ first_failure_name: str | None = None
2664
+ thinking: list[str] = []
1862
2665
  for r in it["rows"]:
1863
2666
  kinds[r.kind] = kinds.get(r.kind, 0) + 1
1864
2667
  kind_totals[r.kind] = kind_totals.get(r.kind, 0) + 1
2668
+ position = (r.source_path, r.line_offset)
2669
+ if r.kind == "tool_call":
2670
+ tool_call_count += 1
2671
+ name = _tool_call_name(r)
2672
+ failed = position in failed_calls
2673
+ if failed and first_failure_name is None:
2674
+ first_failure_name = name
2675
+ if name is not None:
2676
+ tool_counts[name] = tool_counts.get(name, 0) + 1
2677
+ slot = tool_slot.get(name)
2678
+ if slot is None:
2679
+ tool_slot[name] = len(tools)
2680
+ tools.append({"name": name, "is_error": failed})
2681
+ elif failed:
2682
+ tools[slot]["is_error"] = True
2683
+ elif r.kind == "reasoning":
2684
+ thinking.extend(derivation.headings_by_position.get(position, ()))
1865
2685
  item_key = _item_key_for_item(conversation_key, it)
1866
2686
  turn = {
1867
2687
  "item_key": item_key,
@@ -1871,15 +2691,43 @@ def get_codex_conversation_outline(
1871
2691
  "timestamp_utc": it["anchor_row"].timestamp_utc,
1872
2692
  "kinds": kinds,
1873
2693
  }
2694
+ # Additive, and only where there is something to say: a turn with no
2695
+ # calls publishes neither an empty array nor a zero count, matching how
2696
+ # the Claude outline omits `tools` and `thinking`.
2697
+ if tools:
2698
+ turn["tools"] = tools
2699
+ turn["tool_call_count"] = tool_call_count
2700
+ turn["first_failure_name"] = first_failure_name
2701
+ if thinking:
2702
+ turn["thinking"] = thinking
2703
+ item_model = _item_model(it) if _item_kind(it) == "assistant" else None
2704
+ if item_model:
2705
+ turn["model"] = item_model
2706
+ models[item_model] = models.get(item_model, 0) + 1
1874
2707
  if meta is not None:
1875
2708
  turn.update(meta)
1876
2709
  turns.append(turn)
2710
+ total_cost, tokens = _conversation_totals(
2711
+ conn, conversation_key, effective_speed)
1877
2712
  return {
1878
2713
  "status": "ok",
1879
2714
  "conversation_key": conversation_key,
1880
2715
  "turns": turns,
1881
- "stats": {"items": len(items), "kinds": kind_totals},
1882
- "files": _conversation_files(conn, conversation_key),
2716
+ # Tier 2, deliberately a SEPARATE array (§3.3): `adaptQualifiedOutline`
2717
+ # derives `stats.turns.{human,assistant,tool_result,meta}` by filtering
2718
+ # `turns[]` on kind, so putting landmarks there would inflate counts
2719
+ # meant to describe the conversation's structure.
2720
+ "landmarks": _build_landmarks(
2721
+ segment_index, derivation, failed_calls),
2722
+ "stats": {
2723
+ "items": len(items), "kinds": kind_totals,
2724
+ "cost_usd": total_cost, "tokens": tokens,
2725
+ "tool_counts": tool_counts, "models": models,
2726
+ "duration_seconds": _conversation_duration_seconds(rows),
2727
+ "error_count": _outline_error_count(
2728
+ failed_calls, outcome_positions, derivation),
2729
+ },
2730
+ "files": _conversation_files(segment_index, derivation),
1883
2731
  "children": _children_of(conn, conversation_key, effective_speed),
1884
2732
  }
1885
2733
 
@@ -1893,6 +2741,16 @@ def _is_fork(fields: dict) -> bool:
1893
2741
 
1894
2742
 
1895
2743
  def _browse_row(conn: sqlite3.Connection, conversation_key: str, effective_speed: str, fields: dict) -> dict:
2744
+ return _browse_row_from_fields(
2745
+ conversation_key, fields,
2746
+ cost_usd=_conversation_total_cost(conn, conversation_key, effective_speed),
2747
+ parent=_parent_of(conn, conversation_key),
2748
+ )
2749
+
2750
+
2751
+ def _browse_row_from_fields(
2752
+ conversation_key: str, fields: dict, *, cost_usd: float, parent,
2753
+ ) -> dict:
1896
2754
  return {
1897
2755
  "conversation_key": conversation_key,
1898
2756
  "title": _display_chain(fields),
@@ -1901,9 +2759,9 @@ def _browse_row(conn: sqlite3.Connection, conversation_key: str, effective_speed
1901
2759
  "started_utc": fields["started"],
1902
2760
  "last_activity_utc": fields["last"],
1903
2761
  "count": fields["item_count"],
1904
- "cost_usd": _conversation_total_cost(conn, conversation_key, effective_speed),
2762
+ "cost_usd": cost_usd,
1905
2763
  "models": list(fields["models"]),
1906
- "parent": _parent_of(conn, conversation_key),
2764
+ "parent": parent,
1907
2765
  "is_fork": _is_fork(fields),
1908
2766
  }
1909
2767
 
@@ -1948,6 +2806,200 @@ def _paginate_rows(rows: list[dict], *, cursor: str | None, limit: int):
1948
2806
  return window, page
1949
2807
 
1950
2808
 
2809
+ def _stored_rollups_present(conn: sqlite3.Connection) -> bool:
2810
+ """Whether the authoritative stored-rollup branch has been materialized.
2811
+
2812
+ Normal writes update normalized rows and their rollups in one transaction.
2813
+ The only supported no-rollup state is the pre-first-recompute window, where
2814
+ the live branch keeps the rail available. This constant-time probe avoids
2815
+ reintroducing the whole-message-table DISTINCT scan on every cold browse.
2816
+ """
2817
+ return conn.execute(
2818
+ "SELECT 1 FROM codex_conversation_rollups LIMIT 1").fetchone() is not None
2819
+
2820
+
2821
+ def _live_browse_fields(conn: sqlite3.Connection) -> list[tuple[str, dict]]:
2822
+ """Live-recompute fallback used only while no stored rollups exist."""
2823
+ out = []
2824
+ for (conversation_key,) in conn.execute(
2825
+ "SELECT DISTINCT conversation_key FROM codex_conversation_messages"):
2826
+ fields = _rollup_fields(conn, conversation_key)
2827
+ if fields is not None:
2828
+ out.append((conversation_key, fields))
2829
+ return out
2830
+
2831
+
2832
+ def _facets_from_fields(fields_rows: list[tuple[str, dict]]) -> dict:
2833
+ return _browse_facets([
2834
+ {
2835
+ "project_key": fields["project_key"],
2836
+ "project_label": fields["project_label"],
2837
+ "models": list(fields["models"]),
2838
+ }
2839
+ for _conversation_key, fields in fields_rows
2840
+ ])
2841
+
2842
+
2843
+ def _stored_browse_facets(conn: sqlite3.Connection) -> dict:
2844
+ fields_rows = []
2845
+ for conversation_key, project_key, project_label, models_json in conn.execute(
2846
+ "SELECT conversation_key, project_key, project_label, models_json "
2847
+ "FROM codex_conversation_rollups"
2848
+ ):
2849
+ try:
2850
+ parsed = json.loads(models_json) if models_json else []
2851
+ models = parsed if isinstance(parsed, list) else []
2852
+ except (TypeError, json.JSONDecodeError):
2853
+ models = []
2854
+ fields_rows.append((conversation_key, {
2855
+ "project_key": project_key,
2856
+ "project_label": project_label,
2857
+ "models": models,
2858
+ }))
2859
+ return _facets_from_fields(fields_rows)
2860
+
2861
+
2862
+ def _stored_filter_sql(alias: str, project_key: str | None, model: str | None):
2863
+ clauses = []
2864
+ params = []
2865
+ if project_key is not None:
2866
+ clauses.append(f"{alias}.project_key = ?")
2867
+ params.append(project_key)
2868
+ if model is not None:
2869
+ # models_json is the writer's canonical JSON array. Searching for the
2870
+ # complete JSON string literal is exact and does not require JSON1.
2871
+ clauses.append(f"instr(COALESCE({alias}.models_json, ''), ?) > 0")
2872
+ params.append(json.dumps(model))
2873
+ return (" AND ".join(clauses) if clauses else "1"), params
2874
+
2875
+
2876
+ def _page_costs(
2877
+ conn: sqlite3.Connection, conversation_keys: list[str], effective_speed: str,
2878
+ ) -> dict[str, float]:
2879
+ if not conversation_keys:
2880
+ return {}
2881
+ placeholders = ",".join("?" for _ in conversation_keys)
2882
+ totals = {key: 0.0 for key in conversation_keys}
2883
+ for ck, model, inp, cin, out, rout in conn.execute(
2884
+ "SELECT conversation_key, model, input_tokens, cached_input_tokens, "
2885
+ "output_tokens, reasoning_output_tokens FROM codex_session_entries "
2886
+ f"WHERE conversation_key IN ({placeholders}) "
2887
+ "ORDER BY conversation_key, id",
2888
+ conversation_keys,
2889
+ ):
2890
+ totals[ck] += _calculate_codex_entry_cost(
2891
+ model or "", inp or 0, cin or 0, out or 0, rout or 0,
2892
+ speed=effective_speed)
2893
+ return totals
2894
+
2895
+
2896
+ def _stored_browse_page(
2897
+ conn: sqlite3.Connection, *, effective_speed: str,
2898
+ project_key: str | None, model: str | None, limit: int,
2899
+ cursor: str | None,
2900
+ ):
2901
+ where_sql, filter_params = _stored_filter_sql("r", project_key, model)
2902
+ total = conn.execute(
2903
+ f"SELECT COUNT(*) FROM codex_conversation_rollups r WHERE {where_sql}",
2904
+ filter_params,
2905
+ ).fetchone()[0]
2906
+
2907
+ cursor_row = None
2908
+ if cursor is not None:
2909
+ cursor_where, cursor_params = _stored_filter_sql("c", project_key, model)
2910
+ cursor_row = conn.execute(
2911
+ "SELECT COALESCE(c.last_activity_utc, ''), c.conversation_key "
2912
+ "FROM codex_conversation_rollups c "
2913
+ f"WHERE c.conversation_key = ? AND {cursor_where}",
2914
+ [cursor, *cursor_params],
2915
+ ).fetchone()
2916
+
2917
+ page_where = [where_sql]
2918
+ page_params = list(filter_params)
2919
+ if cursor_row is not None:
2920
+ cursor_last, cursor_key = cursor_row
2921
+ page_where.append(
2922
+ "(COALESCE(r.last_activity_utc, '') < ? OR "
2923
+ "(COALESCE(r.last_activity_utc, '') = ? AND r.conversation_key < ?))")
2924
+ page_params.extend((cursor_last, cursor_last, cursor_key))
2925
+
2926
+ sql = (
2927
+ "SELECT r.conversation_key, r.item_count, r.started_utc, "
2928
+ "r.last_activity_utc, r.project_key, r.project_label, r.models_json, "
2929
+ "r.title, r.parent_thread_id, r.source_root_key, t.native_thread_id, "
2930
+ "pt.conversation_key, pr.title, pr.project_label, pt.native_thread_id "
2931
+ "FROM codex_conversation_rollups r "
2932
+ "LEFT JOIN codex_conversation_threads t "
2933
+ "ON t.conversation_key = r.conversation_key "
2934
+ "LEFT JOIN codex_conversation_threads pt "
2935
+ "ON pt.source_root_key = r.source_root_key "
2936
+ "AND pt.native_thread_id = r.parent_thread_id "
2937
+ "AND pt.conversation_key != r.conversation_key "
2938
+ "LEFT JOIN codex_conversation_rollups pr "
2939
+ "ON pr.conversation_key = pt.conversation_key "
2940
+ f"WHERE {' AND '.join(page_where)} "
2941
+ "ORDER BY COALESCE(r.last_activity_utc, '') DESC, r.conversation_key DESC"
2942
+ )
2943
+ if limit:
2944
+ sql += " LIMIT ?"
2945
+ page_params.append(limit + 1)
2946
+ raw_rows = list(conn.execute(sql, page_params))
2947
+ has_more = bool(limit and len(raw_rows) > limit)
2948
+ if has_more:
2949
+ raw_rows = raw_rows[:limit]
2950
+
2951
+ keys = [row[0] for row in raw_rows]
2952
+ costs = _page_costs(conn, keys, effective_speed)
2953
+ rows = []
2954
+ for row in raw_rows:
2955
+ (conversation_key, item_count, started, last, row_project_key,
2956
+ project_label, models_json, title, parent_thread_id, source_root_key,
2957
+ native_thread_id, parent_key, parent_title, parent_project_label,
2958
+ parent_native_thread_id) = row
2959
+ try:
2960
+ parsed = json.loads(models_json) if models_json else []
2961
+ models = parsed if isinstance(parsed, list) else []
2962
+ except (TypeError, json.JSONDecodeError):
2963
+ models = []
2964
+ fields = {
2965
+ "item_count": item_count,
2966
+ "started": started,
2967
+ "last": last,
2968
+ "project_key": row_project_key,
2969
+ "project_label": project_label,
2970
+ "models": models,
2971
+ "title": title,
2972
+ "parent_thread_id": parent_thread_id,
2973
+ "source_root_key": source_root_key,
2974
+ "native_thread_id": native_thread_id,
2975
+ }
2976
+ parent = None
2977
+ if parent_key is not None:
2978
+ parent = {
2979
+ "conversation_key": parent_key,
2980
+ "title": _display_chain({
2981
+ "title": parent_title,
2982
+ "project_label": parent_project_label,
2983
+ "native_thread_id": parent_native_thread_id,
2984
+ }),
2985
+ }
2986
+ rows.append(_browse_row_from_fields(
2987
+ conversation_key, fields, cost_usd=costs.get(conversation_key, 0.0),
2988
+ parent=parent))
2989
+ next_cursor = rows[-1]["conversation_key"] if (rows and has_more) else None
2990
+ return rows, {"total": total, "returned": len(rows), "cursor": next_cursor}
2991
+
2992
+
2993
+ def list_codex_conversation_facets(conn: sqlite3.Connection) -> dict:
2994
+ """Facet-only browse projection; never builds or prices a discarded page."""
2995
+ if not codex_normalization_authoritative(conn):
2996
+ return {"status": "normalization_pending",
2997
+ "facets": {"projects": [], "models": []}}
2998
+ facets = (_stored_browse_facets(conn) if _stored_rollups_present(conn)
2999
+ else _facets_from_fields(_live_browse_fields(conn)))
3000
+ return {"status": "ok", "facets": facets}
3001
+
3002
+
1951
3003
  def list_codex_conversations(
1952
3004
  conn: sqlite3.Connection,
1953
3005
  *,
@@ -1956,6 +3008,7 @@ def list_codex_conversations(
1956
3008
  model: str | None = None,
1957
3009
  limit: int = 50,
1958
3010
  cursor: str | None = None,
3011
+ selected: str | None = None,
1959
3012
  ) -> dict:
1960
3013
  """Browse envelope (§5.6 / §6.1): a page of conversation rows ordered by last
1961
3014
  activity, with project/model facets. Dual-branch — the stored rollup fast
@@ -1966,23 +3019,37 @@ def list_codex_conversations(
1966
3019
  if not codex_normalization_authoritative(conn):
1967
3020
  return {"status": "normalization_pending", "rows": [],
1968
3021
  "facets": {"projects": [], "models": []}, "page": {"total": 0}}
1969
- keys = [r[0] for r in conn.execute(
1970
- "SELECT DISTINCT conversation_key FROM codex_conversation_messages")]
1971
- rows: list[dict] = []
1972
- for conversation_key in keys:
1973
- fields = _rollup_fields(conn, conversation_key)
1974
- if fields is None:
1975
- continue
1976
- rows.append(_browse_row(conn, conversation_key, effective_speed, fields))
1977
- facets = _browse_facets(rows)
1978
- filtered = [
1979
- row for row in rows
1980
- if (project_key is None or row["project_key"] == project_key)
1981
- and (model is None or model in (row["models"] or []))
3022
+ if _stored_rollups_present(conn):
3023
+ facets = _stored_browse_facets(conn)
3024
+ page_rows, page = _stored_browse_page(
3025
+ conn, effective_speed=effective_speed, project_key=project_key,
3026
+ model=model, limit=limit, cursor=cursor)
3027
+ result = {"status": "ok", "rows": page_rows, "facets": facets, "page": page}
3028
+ if selected is not None:
3029
+ fields = _rollup_fields(conn, selected)
3030
+ if fields is not None:
3031
+ result["selected"] = _browse_row(
3032
+ conn, selected, effective_speed, fields)
3033
+ return result
3034
+
3035
+ fields_rows = _live_browse_fields(conn)
3036
+ rows = [
3037
+ _browse_row(conn, conversation_key, effective_speed, fields)
3038
+ for conversation_key, fields in fields_rows
1982
3039
  ]
3040
+ facets = _facets_from_fields(fields_rows)
3041
+ filtered = [row for row in rows
3042
+ if (project_key is None or row["project_key"] == project_key)
3043
+ and (model is None or model in (row["models"] or []))]
1983
3044
  filtered.sort(key=_recent_sort_key, reverse=True)
1984
3045
  page_rows, page = _paginate_rows(filtered, cursor=cursor, limit=limit)
1985
- return {"status": "ok", "rows": page_rows, "facets": facets, "page": page}
3046
+ result = {"status": "ok", "rows": page_rows, "facets": facets, "page": page}
3047
+ if selected is not None:
3048
+ selected_row = next(
3049
+ (row for row in rows if row["conversation_key"] == selected), None)
3050
+ if selected_row is not None:
3051
+ result["selected"] = selected_row
3052
+ return result
1986
3053
 
1987
3054
 
1988
3055
  # ── search (§6.2) ─────────────────────────────────────────────────────────────
@@ -2057,6 +3124,405 @@ def _pos_to_item_key(conn: sqlite3.Connection, conversation_key: str) -> dict:
2057
3124
  return _pos_to_item_key_and_order(conn, conversation_key)[0]
2058
3125
 
2059
3126
 
3127
+ # ── #482 visible render-leaf projection ─────────────────────────────────────
3128
+
3129
+ _COMPLETION_EVENT_TYPES = {
3130
+ "patch_apply_end",
3131
+ "web_search_end",
3132
+ "mcp_tool_call_end",
3133
+ }
3134
+
3135
+
3136
+ def _find_surface(row) -> str | None:
3137
+ if row.kind in {"user", "assistant", "reasoning", "meta"}:
3138
+ return "body"
3139
+ if row.kind == "tool_call":
3140
+ return "call"
3141
+ if row.kind == "tool_output":
3142
+ return "output"
3143
+ if row.kind == "event" and row.event_type in _COMPLETION_EVENT_TYPES:
3144
+ return "completion"
3145
+ return None
3146
+
3147
+
3148
+ def _project_plain_leaves(leaves: list[RenderLeaf]):
3149
+ """Project structured-card leaves with a non-searchable visual boundary.
3150
+
3151
+ Native card fields render in separate block/inline containers. A newline
3152
+ between fields prevents a match from crossing that visual boundary while
3153
+ keeping each leaf's offsets local to the exact string its React component
3154
+ receives.
3155
+ """
3156
+ text_parts: list[str] = []
3157
+ projected: list[ProjectedLeaf] = []
3158
+ cursor = 0
3159
+ for leaf in leaves:
3160
+ if not leaf.text:
3161
+ continue
3162
+ if text_parts:
3163
+ text_parts.append("\n")
3164
+ cursor += 1
3165
+ start = cursor
3166
+ text_parts.append(leaf.text)
3167
+ cursor += len(leaf.text)
3168
+ projected.append(ProjectedLeaf(leaf.key, start, cursor))
3169
+ return "".join(text_parts), tuple(projected)
3170
+
3171
+
3172
+ def _project_markdown_fields(fields: list[tuple[str, str]]):
3173
+ text_parts: list[str] = []
3174
+ leaves: list[ProjectedLeaf] = []
3175
+ cursor = 0
3176
+ for field, source in fields:
3177
+ if not source:
3178
+ continue
3179
+ if text_parts:
3180
+ text_parts.append("\n")
3181
+ cursor += 1
3182
+ projected_text, projected_leaves = project_markdown(source)
3183
+ text_parts.append(projected_text)
3184
+ leaves.extend(
3185
+ ProjectedLeaf(f"{field}/{leaf.key}", cursor + leaf.start, cursor + leaf.end)
3186
+ for leaf in projected_leaves
3187
+ )
3188
+ cursor += len(projected_text)
3189
+ return "".join(text_parts), tuple(leaves)
3190
+
3191
+
3192
+ def _patch_diff_leaves(files) -> list[RenderLeaf]:
3193
+ leaves: list[RenderLeaf] = []
3194
+ for file_index, file in enumerate(files or []):
3195
+ if not isinstance(file, dict):
3196
+ continue
3197
+ for field in ("path", "move_path"):
3198
+ value = file.get(field)
3199
+ if isinstance(value, str) and value:
3200
+ leaves.append(RenderLeaf(f"files.{file_index}.{field}", value))
3201
+ diff = file.get("unified_diff")
3202
+ if not isinstance(diff, str):
3203
+ continue
3204
+ hunk_index = -1
3205
+ row_index = 0
3206
+ for line in diff.replace("\r\n", "\n").replace("\r", "\n").split("\n"):
3207
+ if line.startswith("@@"):
3208
+ hunk_index += 1
3209
+ row_index = 0
3210
+ continue
3211
+ if hunk_index < 0 or not line or line.startswith(("--- ", "+++ ", "\\")):
3212
+ continue
3213
+ if line[0] not in {"+", "-", " "}:
3214
+ continue
3215
+ leaves.append(RenderLeaf(
3216
+ f"files.{file_index}.diff.{hunk_index}.{row_index}", line[1:]))
3217
+ row_index += 1
3218
+ return leaves
3219
+
3220
+
3221
+ def _json_card_text(value) -> str:
3222
+ return value if isinstance(value, str) else json.dumps(
3223
+ value, ensure_ascii=False, indent=2, separators=(",", ": "))
3224
+
3225
+
3226
+ def _project_completion_payload(payload: dict):
3227
+ patch = kern.decode_patch_event_card(payload)
3228
+ if patch is not None:
3229
+ leaves = _patch_diff_leaves(patch.get("files"))
3230
+ for field in ("stdout", "stderr"):
3231
+ value = patch.get(field)
3232
+ if isinstance(value, str) and value:
3233
+ leaves.append(RenderLeaf(field, _strip_ansi(value)))
3234
+ return _project_plain_leaves(leaves) if leaves else None
3235
+
3236
+ completion = kern.decode_secondary_event_card(payload)
3237
+ if completion is None:
3238
+ return None
3239
+ if completion.get("type") == "web_search_completion":
3240
+ leaves: list[RenderLeaf] = []
3241
+ for index, result in enumerate(completion.get("results") or []):
3242
+ if not isinstance(result, dict):
3243
+ continue
3244
+ for field in ("title", "domain", "snippet", "ref_id"):
3245
+ value = result.get(field)
3246
+ if isinstance(value, str) and value:
3247
+ leaves.append(RenderLeaf(f"results.{index}.{field}", value))
3248
+ error = completion.get("error")
3249
+ if error is not None:
3250
+ leaves.append(RenderLeaf("error", _json_card_text(error)))
3251
+ return _project_plain_leaves(leaves) if leaves else None
3252
+ if completion.get("type") == "mcp_completion":
3253
+ leaves = [
3254
+ RenderLeaf("arguments", _json_card_text(completion.get("arguments"))),
3255
+ RenderLeaf("result", _json_card_text(completion.get("result"))),
3256
+ ]
3257
+ return _project_plain_leaves(leaves)
3258
+ return None
3259
+
3260
+
3261
+ def _project_find_row(row, *, payload: dict | None = None, block: dict | None = None):
3262
+ if row.kind == "event" and row.event_type in _COMPLETION_EVENT_TYPES and payload:
3263
+ completion = _project_completion_payload(payload)
3264
+ if completion is not None:
3265
+ return completion
3266
+ text = _row_display(row)
3267
+ if not text:
3268
+ return None
3269
+ if row.kind == "meta":
3270
+ try:
3271
+ detail = json.loads(row.detail_json or "{}")
3272
+ except (TypeError, json.JSONDecodeError):
3273
+ detail = {}
3274
+ meta_kind = detail.get("meta_kind") if isinstance(detail, dict) else None
3275
+ if meta_kind == "command":
3276
+ return project_plain((RenderLeaf("t0", text),))
3277
+ if meta_kind == "context":
3278
+ return project_context(text)
3279
+ return project_markdown(text)
3280
+ if row.kind == "reasoning" and isinstance(block, dict):
3281
+ detail = block.get("detail")
3282
+ reasoning = detail.get("reasoning") if isinstance(detail, dict) else None
3283
+ if isinstance(reasoning, dict):
3284
+ visible_headings = block.get("_find_visible_headings")
3285
+ if isinstance(visible_headings, list):
3286
+ leaves = [
3287
+ RenderLeaf(leaf_key, text)
3288
+ for leaf_key, text in visible_headings
3289
+ if isinstance(leaf_key, str) and isinstance(text, str) and text
3290
+ ]
3291
+ if leaves:
3292
+ return _project_plain_leaves(leaves)
3293
+ if reasoning.get("body") is None:
3294
+ return None
3295
+ fields = [
3296
+ (field, reasoning[field])
3297
+ for field in ("title", "summary", "body")
3298
+ if isinstance(reasoning.get(field), str) and reasoning[field]
3299
+ ]
3300
+ if fields:
3301
+ return _project_markdown_fields(fields)
3302
+ if row.kind in {"user", "assistant", "reasoning"}:
3303
+ return project_markdown(text)
3304
+ if row.kind == "tool_output" and isinstance(block, dict):
3305
+ detail = block.get("detail")
3306
+ card = detail.get("card") if isinstance(detail, dict) else None
3307
+ if isinstance(card, dict) and card.get("type") == "terminal":
3308
+ output = card.get("output")
3309
+ parts = output.get("parts") if isinstance(output, dict) else None
3310
+ if isinstance(parts, list):
3311
+ stdout = "".join(
3312
+ part.get("text", "") for part in parts
3313
+ if isinstance(part, dict) and part.get("type") == "text"
3314
+ and part.get("stream") != "stderr"
3315
+ )
3316
+ stderr = "".join(
3317
+ part.get("text", "") for part in parts
3318
+ if isinstance(part, dict) and part.get("type") == "text"
3319
+ and part.get("stream") == "stderr"
3320
+ )
3321
+ leaves = []
3322
+ if stdout:
3323
+ leaves.append(RenderLeaf("stdout", _strip_ansi(stdout)))
3324
+ if stderr:
3325
+ leaves.append(RenderLeaf("stderr", _strip_ansi(stderr)))
3326
+ leaves.extend(
3327
+ RenderLeaf(f"raw.{index}", part["text"])
3328
+ for index, part in enumerate(parts)
3329
+ if isinstance(part, dict) and part.get("type") == "raw"
3330
+ and isinstance(part.get("text"), str) and part["text"]
3331
+ )
3332
+ if leaves:
3333
+ return _project_plain_leaves(leaves)
3334
+ if row.kind == "tool_call" and isinstance(block, dict):
3335
+ detail = block.get("detail")
3336
+ detail = detail if isinstance(detail, dict) else {}
3337
+ card = detail.get("card")
3338
+ if isinstance(card, dict):
3339
+ if card.get("type") == "patch":
3340
+ return None
3341
+ if card.get("type") == "web_search" and isinstance(card.get("query"), str):
3342
+ return project_plain((RenderLeaf("query", card["query"]),))
3343
+ if card.get("type") == "mcp":
3344
+ return None
3345
+ if card.get("type") == "terminal":
3346
+ commands = [
3347
+ RenderLeaf(f"commands.{index}", entry["command"])
3348
+ for index, entry in enumerate(card.get("commands") or [])
3349
+ if isinstance(entry, dict) and isinstance(entry.get("command"), str)
3350
+ ]
3351
+ if commands:
3352
+ return _project_plain_leaves(commands)
3353
+ args = detail.get("args")
3354
+ if isinstance(args, str) and args:
3355
+ return project_plain((RenderLeaf("t0", args),))
3356
+ return project_plain((RenderLeaf("t0", text),))
3357
+
3358
+
3359
+ def materialize_codex_find_projection(
3360
+ conn: sqlite3.Connection,
3361
+ conversation_keys,
3362
+ ) -> None:
3363
+ """Replace #482 projection rows for the affected conversations.
3364
+
3365
+ The existing item/block builder is the only authority for native folds.
3366
+ Every searchable physical row keeps its own block key; a folded output or
3367
+ completion separately records the visual call block that owns it.
3368
+ """
3369
+ keys = sorted({key for key in conversation_keys if key})
3370
+ if not keys:
3371
+ return
3372
+ for conversation_key in keys:
3373
+ conn.execute(
3374
+ "DELETE FROM codex_find_projection WHERE conversation_key=?",
3375
+ (conversation_key,),
3376
+ )
3377
+ rows = [
3378
+ kern.CodexNormalizedRow(*row)
3379
+ for row in conn.execute(
3380
+ "SELECT " + _ROW_COLS + " FROM codex_conversation_messages "
3381
+ "WHERE conversation_key=? "
3382
+ "ORDER BY timestamp_utc,source_path,line_offset",
3383
+ (conversation_key,),
3384
+ )
3385
+ ]
3386
+ if not rows:
3387
+ continue
3388
+ kept, _suppressed = kern.pair_mirrors(rows)
3389
+ items = kern.canonical_items(kept)
3390
+ payloads = _load_row_payloads(conn, conversation_key)
3391
+ pos_to_item = _pos_to_item_key(conn, conversation_key)
3392
+ row_ids = {
3393
+ (source_path, line_offset): message_id
3394
+ for message_id, source_path, line_offset in conn.execute(
3395
+ "SELECT id,source_path,line_offset "
3396
+ "FROM codex_conversation_messages WHERE conversation_key=?",
3397
+ (conversation_key,),
3398
+ )
3399
+ }
3400
+ render_order = 0
3401
+ seen: set[tuple[str, int]] = set()
3402
+ seen_reasoning_by_turn: dict[str, set[str]] = {}
3403
+
3404
+ def store(row, *, container_block_key: str, block: dict | None = None) -> None:
3405
+ nonlocal render_order
3406
+ position = (row.source_path, row.line_offset)
3407
+ if position in seen:
3408
+ return
3409
+ surface = _find_surface(row)
3410
+ retained = _row_payload(row, payloads)
3411
+ payload = retained[1] if retained is not None else None
3412
+ projected = _project_find_row(row, payload=payload, block=block)
3413
+ message_id = row_ids.get(position)
3414
+ if surface is None or projected is None or message_id is None:
3415
+ return
3416
+ text, leaves = projected
3417
+ if not text:
3418
+ return
3419
+ physical_block_key = _block_key_for_row(row)
3420
+ item_key = pos_to_item.get(position)
3421
+ if item_key is None:
3422
+ return
3423
+ disclosure = (
3424
+ [container_block_key]
3425
+ if row.kind in {"reasoning", "meta"} or surface != "body"
3426
+ else []
3427
+ )
3428
+ conn.execute(
3429
+ "INSERT INTO codex_find_projection "
3430
+ "(message_id,conversation_key,item_key,block_key,"
3431
+ "container_block_key,surface,render_order,projected_text,"
3432
+ "leaves_json,disclosure_json,projection_version) "
3433
+ "VALUES (?,?,?,?,?,?,?,?,?,?,?)",
3434
+ (
3435
+ message_id,
3436
+ conversation_key,
3437
+ item_key,
3438
+ physical_block_key,
3439
+ container_block_key,
3440
+ surface,
3441
+ render_order,
3442
+ text,
3443
+ json.dumps(
3444
+ [
3445
+ {"key": leaf.key, "start": leaf.start, "end": leaf.end}
3446
+ for leaf in leaves
3447
+ ],
3448
+ sort_keys=True,
3449
+ separators=(",", ":"),
3450
+ ),
3451
+ json.dumps(disclosure, separators=(",", ":")),
3452
+ CODEX_FIND_PROJECTION_VERSION,
3453
+ ),
3454
+ )
3455
+ seen.add(position)
3456
+ render_order += 1
3457
+
3458
+ for item in items:
3459
+ built_entries = _item_blocks_with_rows(
3460
+ item, payloads, decompose_headings=True,
3461
+ )
3462
+ completion_owner: dict[str, str] = {}
3463
+ for candidate_block, _candidate_primary, _candidate_output in built_entries:
3464
+ candidate_container = (
3465
+ candidate_block.get("block_key")
3466
+ or _block_key_for_row(_candidate_primary)
3467
+ )
3468
+ candidate_detail = candidate_block.get("detail")
3469
+ candidate_card = (
3470
+ candidate_detail.get("card")
3471
+ if isinstance(candidate_detail, dict) else None
3472
+ )
3473
+ completion = (
3474
+ candidate_card.get("completion")
3475
+ if isinstance(candidate_card, dict) else None
3476
+ )
3477
+ event_key = (
3478
+ completion.get("event_block_key")
3479
+ if isinstance(completion, dict) else None
3480
+ )
3481
+ if isinstance(event_key, str):
3482
+ completion_owner[event_key] = candidate_container
3483
+
3484
+ for block, primary, output in built_entries:
3485
+ container = block.get("block_key") or _block_key_for_row(primary)
3486
+ container = completion_owner.get(_block_key_for_row(primary), container)
3487
+ if primary.kind == "reasoning":
3488
+ detail = block.get("detail")
3489
+ reasoning = (
3490
+ detail.get("reasoning") if isinstance(detail, dict) else None
3491
+ )
3492
+ headings = (
3493
+ reasoning.get("headings")
3494
+ if isinstance(reasoning, dict) else None
3495
+ )
3496
+ if isinstance(headings, list):
3497
+ turn_key = primary.turn_id or item.get("turn_id") or ""
3498
+ prior = seen_reasoning_by_turn.setdefault(turn_key, set())
3499
+ visible = []
3500
+ for heading_index, heading in enumerate(headings):
3501
+ text = heading.get("text") if isinstance(heading, dict) else None
3502
+ if not isinstance(text, str) or text in prior:
3503
+ continue
3504
+ prior.add(text)
3505
+ visible.append((f"headings.{heading_index}", text))
3506
+ block["_find_visible_headings"] = visible
3507
+ store(primary, container_block_key=container, block=block)
3508
+ if output is not None:
3509
+ store(output, container_block_key=container, block=block)
3510
+ # Completion folds that intentionally produce no standalone block
3511
+ # still own a searchable physical surface and point at their call.
3512
+ for row in item["rows"]:
3513
+ if row.event_type not in _COMPLETION_EVENT_TYPES:
3514
+ continue
3515
+ physical_key = _block_key_for_row(row)
3516
+ container = completion_owner.get(physical_key, physical_key)
3517
+ store(row, container_block_key=container)
3518
+
3519
+ conn.execute(
3520
+ "INSERT INTO cache_meta(key,value) VALUES"
3521
+ "('codex_find_projection_generation','1') "
3522
+ "ON CONFLICT(key) DO UPDATE SET value=CAST(value AS INTEGER)+1"
3523
+ )
3524
+
3525
+
2060
3526
  def _fts_query(query: str, column: str | None) -> str:
2061
3527
  """A safe FTS5 query: each whitespace term becomes a quoted phrase, joined by
2062
3528
  implicit AND (term-wise AND — the documented divergence from LIKE's single
@@ -2120,7 +3586,60 @@ def _excerpt(text: str | None) -> str:
2120
3586
  return collapsed[:200]
2121
3587
 
2122
3588
 
2123
- def _collapse_message_hits(conn: sqlite3.Connection, matched_rows: list) -> list[dict]:
3589
+ def _search_display_text(text: str | None) -> str:
3590
+ """Readable search projection for retained structured content arrays.
3591
+
3592
+ Tool outputs must retain their provider JSON in ``search_tool`` so every
3593
+ leaf stays searchable. The rail, however, needs the same ordered text
3594
+ leaves a reader sees—not the serialized wrapper. Unknown JSON and future
3595
+ shapes fall back byte-for-byte to the retained string.
3596
+ """
3597
+ if not text:
3598
+ return ""
3599
+ raw = str(text)
3600
+ if not raw.lstrip().startswith("["):
3601
+ return raw
3602
+ try:
3603
+ parsed = json.loads(raw)
3604
+ except (TypeError, json.JSONDecodeError):
3605
+ # Search columns are capped. A large content array can therefore end
3606
+ # mid-string and cease to be valid JSON even though one or more leading
3607
+ # text parts are complete. `_canonical_json` sorts object keys, so a
3608
+ # text-bearing part starts as `{\"text\":...}`. Decode only those
3609
+ # complete JSON string literals; never regex-unescape provider bytes.
3610
+ parts = []
3611
+ for match in re.finditer(r'(?:\A\[\{|,\{)"text":', raw):
3612
+ try:
3613
+ value, _end = json.JSONDecoder().raw_decode(raw, match.end())
3614
+ except (TypeError, json.JSONDecodeError):
3615
+ continue
3616
+ if isinstance(value, str) and value:
3617
+ parts.append(value)
3618
+ return "\n".join(parts) + ("\n…" if parts else "") or raw
3619
+ joined = kern._join_content_texts(parsed)
3620
+ return joined if joined else raw
3621
+
3622
+
3623
+ def _search_excerpt(text: str | None, query: str, width: int = 200) -> str:
3624
+ """Whitespace-collapsed, match-centred excerpt from readable search text."""
3625
+ collapsed = " ".join(_search_display_text(text).split())
3626
+ if not collapsed:
3627
+ return ""
3628
+ needle = " ".join(query.split())
3629
+ found = collapsed.casefold().find(needle.casefold()) if needle else -1
3630
+ if found < 0 or len(collapsed) <= width:
3631
+ return collapsed[:width]
3632
+ start = max(0, found - (width // 3))
3633
+ end = min(len(collapsed), start + width)
3634
+ start = max(0, end - width)
3635
+ excerpt = collapsed[start:end]
3636
+ return (("… " if start else "") + excerpt
3637
+ + (" …" if end < len(collapsed) else ""))
3638
+
3639
+
3640
+ def _collapse_message_hits(
3641
+ conn: sqlite3.Connection, matched_rows: list, query: str,
3642
+ ) -> list[dict]:
2124
3643
  """Collapse matched physical rows to canonical ``item_key`` BEFORE totals /
2125
3644
  badges (§6.2) — both members of a mirror pair map to one item_key, so mirror
2126
3645
  rows never double-count (turned or unturned)."""
@@ -2143,7 +3662,7 @@ def _collapse_message_hits(conn: sqlite3.Connection, matched_rows: list) -> list
2143
3662
  "last_activity_utc": last_act, "project_label": project_label})
2144
3663
  hit["_badges"].add(_badge_for_kind(kind))
2145
3664
  if hit["snippet"] is None:
2146
- hit["snippet"] = _excerpt(disp)
3665
+ hit["snippet"] = _search_excerpt(disp, query)
2147
3666
  return [
2148
3667
  {"conversation_key": h["conversation_key"], "item_key": h["item_key"],
2149
3668
  "title": h["title"], "snippet": h["snippet"], "badges": sorted(h["_badges"]),
@@ -2154,22 +3673,31 @@ def _collapse_message_hits(conn: sqlite3.Connection, matched_rows: list) -> list
2154
3673
 
2155
3674
  def _search_title(conn: sqlite3.Connection, query: str) -> list[dict]:
2156
3675
  """Title search over the rollup table — identical LIKE semantics in both FTS
2157
- and LIKE modes (§6.2). Conversation-level hits (no item anchor)."""
3676
+ and LIKE modes (§6.2). Conversation-level hits (no item anchor).
3677
+
3678
+ #463 S4 §5.1 — the third read path that needs cleaning, and the one that is
3679
+ user-facing on the CLI: `cctally transcript search --source codex
3680
+ --kind title` prints this `snippet` and emits this `title` in its JSON. The
3681
+ MATCH still runs against the stored value, so a query that names markup
3682
+ still finds its conversation; only what is shown is cleaned.
3683
+ """
2158
3684
  like = f"%{query}%"
2159
3685
  hits = []
2160
3686
  for ck, title, last_act, project_label in conn.execute(
2161
3687
  "SELECT conversation_key, title, last_activity_utc, project_label "
2162
3688
  "FROM codex_conversation_rollups WHERE title LIKE ?", (like,)):
3689
+ cleaned = clean_codex_title(title)
2163
3690
  hits.append(
2164
- {"conversation_key": ck, "item_key": None, "title": title,
2165
- "snippet": _excerpt(title), "badges": ["title"],
3691
+ {"conversation_key": ck, "item_key": None, "title": cleaned,
3692
+ "snippet": _excerpt(cleaned), "badges": ["title"],
2166
3693
  "last_activity_utc": last_act, "project_label": project_label})
2167
3694
  return hits
2168
3695
 
2169
3696
 
2170
3697
  def _search_files(conn: sqlite3.Connection, query: str) -> list[dict]:
2171
3698
  """File-touch search — matches file paths, collapsed to the owning message's
2172
- canonical item_key (§6.2)."""
3699
+ canonical item_key (§6.2). ``message_id`` is an application-level link, so
3700
+ an orphan is skipped and cannot suppress valid rows."""
2173
3701
  like = f"%{query}%"
2174
3702
  pos_cache: dict[str, dict] = {}
2175
3703
  fields_cache: dict[str, tuple] = {}
@@ -2244,7 +3772,7 @@ def search_codex_conversations(
2244
3772
  hits = _search_files(conn, query)
2245
3773
  else:
2246
3774
  hits = _collapse_message_hits(
2247
- conn, _matched_message_rows(conn, query, kind, mode))
3775
+ conn, _matched_message_rows(conn, query, kind, mode), query)
2248
3776
  hits.sort(key=lambda h: (h["conversation_key"], h["item_key"] or ""))
2249
3777
  total = len(hits)
2250
3778
  page_hits, page = _paginate_hits(hits, cursor=cursor, limit=limit)
@@ -2256,6 +3784,402 @@ def search_codex_conversations(
2256
3784
 
2257
3785
  # ── in-conversation find (§3.1) ───────────────────────────────────────────────
2258
3786
 
3787
+ _CODEX_EXACT_FIND_SCHEMA_VERSION = 2
3788
+ _CODEX_EXACT_FIND_DEFAULT_LIMIT = 100
3789
+ _CODEX_EXACT_FIND_MAX_LIMIT = 200
3790
+ _CODEX_EXACT_FIND_CURSOR_PREFIX = "ofc1."
3791
+ _CODEX_EXACT_FIND_QUERY_DOMAIN = b"cctally-codex-find-query-v1\0"
3792
+ _CODEX_EXACT_FIND_OCCURRENCE_DOMAIN = b"cctally-codex-find-occurrence-v1\0"
3793
+
3794
+
3795
+ class InvalidFindCursor(ValueError):
3796
+ """The external exact-find cursor is malformed."""
3797
+
3798
+
3799
+ class StaleFindCursor(ValueError):
3800
+ """The exact-find cursor belongs to another query or projection generation."""
3801
+
3802
+
3803
+ def _exact_find_query_id(
3804
+ query: str, *, regex: bool, case_sensitive: bool, kind: str
3805
+ ) -> str:
3806
+ payload = json.dumps(
3807
+ {
3808
+ "case": case_sensitive,
3809
+ "kind": kind,
3810
+ "projection": CODEX_FIND_PROJECTION_VERSION,
3811
+ "query": query,
3812
+ "regex": regex,
3813
+ },
3814
+ ensure_ascii=False,
3815
+ sort_keys=True,
3816
+ separators=(",", ":"),
3817
+ ).encode("utf-8")
3818
+ return hashlib.sha256(_CODEX_EXACT_FIND_QUERY_DOMAIN + payload).hexdigest()
3819
+
3820
+
3821
+ def _exact_find_occurrence_id(
3822
+ query_id: str,
3823
+ *,
3824
+ block_key: str,
3825
+ surface: str,
3826
+ ordinal: int,
3827
+ start: int,
3828
+ end: int,
3829
+ ) -> str:
3830
+ payload = json.dumps(
3831
+ [
3832
+ CODEX_FIND_PROJECTION_VERSION,
3833
+ query_id,
3834
+ block_key,
3835
+ surface,
3836
+ ordinal,
3837
+ start,
3838
+ end,
3839
+ ],
3840
+ ensure_ascii=False,
3841
+ separators=(",", ":"),
3842
+ ).encode("utf-8")
3843
+ digest = hashlib.sha256(_CODEX_EXACT_FIND_OCCURRENCE_DOMAIN + payload).digest()
3844
+ return "o1." + base64.urlsafe_b64encode(digest).decode("ascii").rstrip("=")
3845
+
3846
+
3847
+ def _encode_exact_find_cursor(
3848
+ *,
3849
+ query_id: str,
3850
+ generation: int,
3851
+ start_index: int,
3852
+ direction: str,
3853
+ boundary: tuple[int, int, str, int],
3854
+ ) -> str:
3855
+ payload = json.dumps(
3856
+ {
3857
+ "b": list(boundary),
3858
+ "d": direction,
3859
+ "g": generation,
3860
+ "i": start_index,
3861
+ "q": query_id,
3862
+ "v": CODEX_FIND_PROJECTION_VERSION,
3863
+ },
3864
+ sort_keys=True,
3865
+ separators=(",", ":"),
3866
+ ).encode("utf-8")
3867
+ return _CODEX_EXACT_FIND_CURSOR_PREFIX + base64.urlsafe_b64encode(
3868
+ payload
3869
+ ).decode("ascii").rstrip("=")
3870
+
3871
+
3872
+ def _decode_exact_find_cursor(cursor: str) -> dict[str, object]:
3873
+ if not isinstance(cursor, str) or not cursor.startswith(
3874
+ _CODEX_EXACT_FIND_CURSOR_PREFIX
3875
+ ):
3876
+ raise InvalidFindCursor(cursor)
3877
+ encoded = cursor[len(_CODEX_EXACT_FIND_CURSOR_PREFIX):]
3878
+ try:
3879
+ raw = base64.urlsafe_b64decode(encoded + "=" * (-len(encoded) % 4))
3880
+ canonical = base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=")
3881
+ if canonical != encoded:
3882
+ raise InvalidFindCursor(cursor)
3883
+ payload = json.loads(raw.decode("utf-8"))
3884
+ except (binascii.Error, ValueError, TypeError, UnicodeDecodeError, json.JSONDecodeError):
3885
+ raise InvalidFindCursor(cursor) from None
3886
+ if not isinstance(payload, dict) or set(payload) != {"b", "d", "g", "i", "q", "v"}:
3887
+ raise InvalidFindCursor(cursor)
3888
+ boundary = payload.get("b")
3889
+ if (
3890
+ payload.get("d") not in {"next", "previous"}
3891
+ or type(payload.get("g")) is not int
3892
+ or type(payload.get("i")) is not int
3893
+ or payload["i"] < 0
3894
+ or not isinstance(payload.get("q"), str)
3895
+ or payload.get("v") != CODEX_FIND_PROJECTION_VERSION
3896
+ or not isinstance(boundary, list)
3897
+ or len(boundary) != 4
3898
+ or type(boundary[0]) is not int
3899
+ or type(boundary[1]) is not int
3900
+ or not isinstance(boundary[2], str)
3901
+ or type(boundary[3]) is not int
3902
+ ):
3903
+ raise InvalidFindCursor(cursor)
3904
+ return payload
3905
+
3906
+
3907
+ def _exact_find_base(
3908
+ query_id: str,
3909
+ *,
3910
+ status: str,
3911
+ regex: bool,
3912
+ kind: str,
3913
+ ) -> dict[str, object]:
3914
+ return {
3915
+ "schema_version": _CODEX_EXACT_FIND_SCHEMA_VERSION,
3916
+ "semantics": "occurrence",
3917
+ "status": status,
3918
+ "query_id": query_id,
3919
+ "selection_stale": False,
3920
+ "mode": "regex" if regex else "literal",
3921
+ "kind": kind,
3922
+ "search_depth": "full",
3923
+ }
3924
+
3925
+
3926
+ def find_occurrences_in_codex_conversation(
3927
+ conn: sqlite3.Connection,
3928
+ conversation_key: str,
3929
+ query: str,
3930
+ *,
3931
+ regex: bool,
3932
+ case_sensitive: bool,
3933
+ kind: str,
3934
+ limit: int = _CODEX_EXACT_FIND_DEFAULT_LIMIT,
3935
+ cursor: str | None = None,
3936
+ direction: str = "next",
3937
+ around: str | None = None,
3938
+ ) -> dict[str, object]:
3939
+ """Return occurrence-exact matches over the materialized visible projection.
3940
+
3941
+ Matching never crosses a physical projection surface. Coordinates are
3942
+ Unicode-scalar offsets into stable render leaves, while paging cursors are
3943
+ bound to both query semantics and the current projection generation.
3944
+ """
3945
+ if kind not in CODEX_FIND_KINDS:
3946
+ raise ValueError(f"unknown kind: {kind}")
3947
+ if not isinstance(limit, int) or not 1 <= limit <= _CODEX_EXACT_FIND_MAX_LIMIT:
3948
+ raise ValueError("find limit must be between 1 and 200")
3949
+ if direction not in {"next", "previous"}:
3950
+ raise ValueError("find direction must be next or previous")
3951
+ if cursor is not None and around is not None:
3952
+ raise ValueError("find cursor and around are mutually exclusive")
3953
+ q = (query or "").strip()
3954
+ query_id = _exact_find_query_id(
3955
+ q, regex=regex, case_sensitive=case_sensitive, kind=kind
3956
+ )
3957
+ exists = conn.execute(
3958
+ "SELECT 1 FROM codex_conversation_messages WHERE conversation_key=? LIMIT 1",
3959
+ (conversation_key,),
3960
+ ).fetchone()
3961
+ if exists is None:
3962
+ return {"status": "not_found", "conversation_key": conversation_key}
3963
+ complete = conn.execute(
3964
+ "SELECT 1 FROM cache_meta WHERE "
3965
+ "key='codex_find_projection_complete_version' AND value=?",
3966
+ (str(CODEX_FIND_PROJECTION_VERSION),),
3967
+ ).fetchone()
3968
+ base = _exact_find_base(query_id, status="ready", regex=regex, kind=kind)
3969
+ empty_page = {
3970
+ "start_index": 0,
3971
+ "previous_cursor": None,
3972
+ "next_cursor": None,
3973
+ "occurrences": [],
3974
+ }
3975
+ if complete is None:
3976
+ return {**base, "status": "indexing", "page": empty_page}
3977
+ generation_row = conn.execute(
3978
+ "SELECT value FROM cache_meta WHERE key='codex_find_projection_generation'"
3979
+ ).fetchone()
3980
+ try:
3981
+ generation = int(generation_row[0]) if generation_row else 0
3982
+ except (TypeError, ValueError):
3983
+ generation = 0
3984
+
3985
+ decoded_cursor = None
3986
+ if cursor is not None:
3987
+ decoded_cursor = _decode_exact_find_cursor(cursor)
3988
+ if (
3989
+ decoded_cursor["q"] != query_id
3990
+ or decoded_cursor["g"] != generation
3991
+ or decoded_cursor["d"] != direction
3992
+ ):
3993
+ raise StaleFindCursor(cursor)
3994
+
3995
+ if not q or (regex and len(q) > _CODEX_FIND_REGEX_MAX_LEN):
3996
+ return {**base, "total": 0, "page": empty_page}
3997
+ pattern = re.compile(q, 0 if case_sensitive else re.IGNORECASE) if regex else None
3998
+ kind_predicate = {
3999
+ "all": "1=1",
4000
+ "prompts": "m.kind='user'",
4001
+ "assistant": "m.kind='assistant'",
4002
+ "tools": "p.surface IN ('call','output','completion')",
4003
+ "thinking": "m.kind='reasoning'",
4004
+ }[kind]
4005
+ rows = conn.execute(
4006
+ "SELECT p.message_id,p.item_key,p.block_key,p.container_block_key,"
4007
+ "p.surface,p.render_order,"
4008
+ "p.projected_text,p.leaves_json,p.disclosure_json,m.kind "
4009
+ "FROM codex_find_projection p "
4010
+ "JOIN codex_conversation_messages m ON m.id=p.message_id "
4011
+ "WHERE p.conversation_key=? AND p.projection_version=? AND "
4012
+ + kind_predicate
4013
+ + " ORDER BY p.render_order,p.message_id,p.surface",
4014
+ (conversation_key, CODEX_FIND_PROJECTION_VERSION),
4015
+ )
4016
+ requested: list[tuple[dict[str, object], tuple[int, int, str, int]]] = []
4017
+ around_page: list[tuple[dict[str, object], tuple[int, int, str, int]]] = []
4018
+ head: list[tuple[dict[str, object], tuple[int, int, str, int]]] = []
4019
+ tail: deque[tuple[dict[str, object], tuple[int, int, str, int]]] = deque(
4020
+ maxlen=limit
4021
+ )
4022
+ head_next = None
4023
+ requested_next = None
4024
+ around_next = None
4025
+ around_index = None
4026
+ cursor_valid = decoded_cursor is None
4027
+ cursor_index = int(decoded_cursor["i"]) if decoded_cursor is not None else None
4028
+ requested_start = None
4029
+ requested_end = None
4030
+ if decoded_cursor is not None:
4031
+ if direction == "previous":
4032
+ requested_end = cursor_index
4033
+ requested_start = max(0, cursor_index - limit)
4034
+ else:
4035
+ requested_start = cursor_index
4036
+ requested_end = cursor_index + limit
4037
+ total = 0
4038
+ for (
4039
+ message_id,
4040
+ item_key,
4041
+ block_key,
4042
+ container_block_key,
4043
+ surface,
4044
+ render_order,
4045
+ text,
4046
+ leaves_json,
4047
+ disclosure_json,
4048
+ row_kind,
4049
+ ) in rows:
4050
+ try:
4051
+ leaves = tuple(ProjectedLeaf(**leaf) for leaf in json.loads(leaves_json))
4052
+ disclosure = json.loads(disclosure_json)
4053
+ except (TypeError, ValueError, json.JSONDecodeError):
4054
+ continue
4055
+ ranges = (
4056
+ iter_regex_ranges(text, pattern)
4057
+ if pattern is not None
4058
+ else iter_literal_ranges(text, q, case_sensitive=case_sensitive)
4059
+ )
4060
+ for ordinal, match in enumerate(ranges):
4061
+ fragments = slice_range_to_leaves(match, leaves)
4062
+ if not fragments:
4063
+ continue
4064
+ occurrence_id = _exact_find_occurrence_id(
4065
+ query_id,
4066
+ block_key=block_key,
4067
+ surface=surface,
4068
+ ordinal=ordinal,
4069
+ start=match.start,
4070
+ end=match.end,
4071
+ )
4072
+ match_kinds = []
4073
+ if surface != "body":
4074
+ match_kinds.append("tool")
4075
+ if row_kind == "reasoning":
4076
+ match_kinds.append("thinking")
4077
+ occurrence = {
4078
+ "occurrence_id": occurrence_id,
4079
+ "item_key": item_key,
4080
+ "block_key": block_key,
4081
+ "container_block_key": container_block_key,
4082
+ "surface": surface,
4083
+ "match_kinds": match_kinds,
4084
+ "disclosure": disclosure if isinstance(disclosure, list) else [],
4085
+ "fragments": [
4086
+ {
4087
+ "leaf_key": fragment.leaf_key,
4088
+ "start": fragment.start,
4089
+ "end": fragment.end,
4090
+ }
4091
+ for fragment in fragments
4092
+ ],
4093
+ }
4094
+ boundary = (render_order, message_id, surface, ordinal)
4095
+ pair = (occurrence, boundary)
4096
+ index = total
4097
+ if len(head) < limit:
4098
+ head.append(pair)
4099
+ elif index == limit:
4100
+ head_next = pair
4101
+ tail.append(pair)
4102
+ if cursor_index == index:
4103
+ if tuple(decoded_cursor["b"]) != boundary:
4104
+ raise StaleFindCursor(cursor)
4105
+ cursor_valid = True
4106
+ if (
4107
+ requested_start is not None
4108
+ and requested_end is not None
4109
+ and requested_start <= index < requested_end
4110
+ ):
4111
+ requested.append(pair)
4112
+ elif requested_end is not None and index == requested_end:
4113
+ requested_next = pair
4114
+ if around is not None and around_index is None:
4115
+ if occurrence["occurrence_id"] == around:
4116
+ around_index = index
4117
+ around_page.append(pair)
4118
+ elif around_index is not None:
4119
+ if len(around_page) < limit:
4120
+ around_page.append(pair)
4121
+ elif index == around_index + limit:
4122
+ around_next = pair
4123
+ total += 1
4124
+
4125
+ if not cursor_valid:
4126
+ raise StaleFindCursor(cursor)
4127
+ selection_stale = around is not None and around_index is None
4128
+ next_pair = None
4129
+ if around is not None and around_index is not None:
4130
+ start_index = around_index
4131
+ page_pairs = around_page
4132
+ next_pair = around_next
4133
+ elif around is not None:
4134
+ start_index = 0
4135
+ page_pairs = head
4136
+ next_pair = head_next
4137
+ elif decoded_cursor is not None:
4138
+ start_index = min(requested_start or 0, total)
4139
+ page_pairs = requested
4140
+ next_pair = requested_next
4141
+ elif direction == "previous":
4142
+ page_pairs = list(tail)
4143
+ start_index = max(0, total - len(page_pairs))
4144
+ else:
4145
+ start_index = 0
4146
+ page_pairs = head
4147
+ next_pair = head_next
4148
+ page_occurrences = [occurrence for occurrence, _boundary in page_pairs]
4149
+
4150
+ def cursor_for(
4151
+ index: int,
4152
+ cursor_direction: str,
4153
+ pair: tuple[dict[str, object], tuple[int, int, str, int]] | None,
4154
+ ) -> str | None:
4155
+ if not 0 <= index < total or pair is None:
4156
+ return None
4157
+ return _encode_exact_find_cursor(
4158
+ query_id=query_id,
4159
+ generation=generation,
4160
+ start_index=index,
4161
+ direction=cursor_direction,
4162
+ boundary=pair[1],
4163
+ )
4164
+
4165
+ previous_cursor = (
4166
+ cursor_for(start_index, "previous", page_pairs[0])
4167
+ if start_index > 0 and page_pairs else None
4168
+ )
4169
+ next_index = start_index + len(page_occurrences)
4170
+ next_cursor = cursor_for(next_index, "next", next_pair)
4171
+ return {
4172
+ **base,
4173
+ "total": total,
4174
+ "selection_stale": selection_stale,
4175
+ "page": {
4176
+ "start_index": start_index,
4177
+ "previous_cursor": previous_cursor,
4178
+ "next_cursor": next_cursor,
4179
+ "occurrences": page_occurrences,
4180
+ },
4181
+ }
4182
+
2259
4183
  # Claude cap parity: the anchor list caps at 500 (bin/_lib_conversation_query.py
2260
4184
  # ::_FIND_ANCHOR_CAP), with anchors_truncated when more anchors exist pre-cap.
2261
4185
  _CODEX_FIND_ANCHOR_CAP = 500
@@ -2607,6 +4531,17 @@ def read_codex_payload(
2607
4531
  response = {"status": "ok", "block_key": block_key, "which": which,
2608
4532
  "content": content, "truncated": truncated}
2609
4533
  if card is not None:
4534
+ # The SAME ordinal substitution the paged assembly applies (spec sections
4535
+ # 4.3 and 6.5). Without it this route published the provider's own
4536
+ # session id while the paged detail published the conversation-local
4537
+ # ordinal, so one field carried two meanings depending on which route
4538
+ # served it — and a client validator can only be written against one.
4539
+ # A session the index does not know becomes `ref: null`, never the raw
4540
+ # id, because `_apply_session_ordinals` fails closed.
4541
+ index_rows, _detail_bytes = _load_conversation_index_rows(
4542
+ conn, conversation_key)
4543
+ _envelope, ordinals = _build_session_index(index_rows)
4544
+ _apply_session_ordinals(card, ordinals)
2610
4545
  response["card"] = card
2611
4546
  return response
2612
4547