cctally 1.108.0 → 1.110.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. package/CHANGELOG.md +147 -0
  2. package/README.md +6 -6
  3. package/bin/_cctally_account.py +4 -0
  4. package/bin/_cctally_alerts.py +6 -2
  5. package/bin/_cctally_cache.py +1652 -166
  6. package/bin/_cctally_config.py +3 -2
  7. package/bin/_cctally_core.py +594 -33
  8. package/bin/_cctally_dashboard.py +1585 -201
  9. package/bin/_cctally_dashboard_conversation.py +494 -48
  10. package/bin/_cctally_dashboard_envelope.py +167 -11
  11. package/bin/_cctally_dashboard_share.py +154 -36
  12. package/bin/_cctally_dashboard_sources.py +2615 -422
  13. package/bin/_cctally_db.py +678 -5
  14. package/bin/_cctally_diagnosis_sources.py +238 -26
  15. package/bin/_cctally_doctor.py +99 -18
  16. package/bin/_cctally_five_hour.py +1295 -40
  17. package/bin/_cctally_forecast.py +124 -28
  18. package/bin/_cctally_journal.py +408 -38
  19. package/bin/_cctally_milestone_history.py +150 -14
  20. package/bin/_cctally_parser.py +5 -0
  21. package/bin/_cctally_percent_breakdown.py +6 -1
  22. package/bin/_cctally_project.py +48 -76
  23. package/bin/_cctally_quota.py +39 -7
  24. package/bin/_cctally_quota_model.py +17 -2
  25. package/bin/_cctally_record.py +1647 -443
  26. package/bin/_cctally_rederive.py +899 -30
  27. package/bin/_cctally_refresh.py +2 -1
  28. package/bin/_cctally_share.py +24 -5
  29. package/bin/_cctally_source_analytics.py +1035 -78
  30. package/bin/_cctally_statusline.py +83 -19
  31. package/bin/_cctally_store.py +61 -4
  32. package/bin/_cctally_transcript.py +27 -2
  33. package/bin/_cctally_tui.py +264 -76
  34. package/bin/_cctally_weekrefs.py +28 -5
  35. package/bin/_lib_blocks.py +127 -20
  36. package/bin/_lib_cache_report.py +20 -5
  37. package/bin/_lib_codex_conversation_query.py +297 -64
  38. package/bin/_lib_codex_hooks.py +102 -0
  39. package/bin/_lib_codex_metadata.py +207 -0
  40. package/bin/_lib_conversation.py +119 -0
  41. package/bin/_lib_conversation_dispatch.py +31 -1
  42. package/bin/_lib_conversation_query.py +315 -51
  43. package/bin/_lib_conversation_retention.py +413 -25
  44. package/bin/_lib_dashboard_sources.py +267 -7
  45. package/bin/_lib_diagnosis.py +10 -0
  46. package/bin/_lib_diff_kernel.py +12 -1
  47. package/bin/_lib_doctor.py +230 -8
  48. package/bin/_lib_ingest_frontier.py +1366 -49
  49. package/bin/_lib_journal.py +19 -2
  50. package/bin/_lib_merge_gate.py +307 -0
  51. package/bin/_lib_pricing.py +588 -29
  52. package/bin/_lib_record.py +130 -12
  53. package/bin/_lib_rederive.py +903 -2
  54. package/bin/_lib_render.py +9 -3
  55. package/bin/_lib_retained_size.py +27 -2
  56. package/bin/_lib_segment_summary.py +66 -8
  57. package/bin/_lib_share_templates.py +28 -18
  58. package/bin/_lib_snapshot_cache.py +115 -1
  59. package/bin/_lib_source_retry.py +320 -0
  60. package/bin/_lib_statusline_candidates.py +400 -36
  61. package/bin/_lib_tick_stats.py +119 -0
  62. package/bin/_lib_view_models.py +256 -76
  63. package/bin/cctally +69 -0
  64. package/dashboard/static/assets/ConversationsView-BB9vUiL6.js +72 -0
  65. package/dashboard/static/assets/{DoctorModal-CWn3U4Wl.js → DoctorModal-DFY2iPDf.js} +1 -1
  66. package/dashboard/static/assets/ModalRoot-DvRKHbQE.js +1 -0
  67. package/dashboard/static/assets/ProjectsDrillPanel-DAv39uSo.js +1 -0
  68. package/dashboard/static/assets/SourceDetailModal-DSAWo2Fp.js +1 -0
  69. package/dashboard/static/assets/{UpdateModal-D3m8GG6V.js → UpdateModal-R_9dOVnh.js} +3 -3
  70. package/dashboard/static/assets/dashboardStream.shared-worker-DbG6Ef0T.js +1 -0
  71. package/dashboard/static/assets/index-COhxe3vU.css +1 -0
  72. package/dashboard/static/assets/index-DdU62sov.js +13 -0
  73. package/dashboard/static/assets/outlineNavigation-BOCLcRa6.js +9 -0
  74. package/dashboard/static/assets/useKeymap-CW7Up0Ay.js +1 -0
  75. package/dashboard/static/dashboard.html +3 -3
  76. package/package.json +4 -1
  77. package/dashboard/static/assets/ConversationsView-BOSaBRtu.js +0 -72
  78. package/dashboard/static/assets/ModalRoot-SR070V6c.js +0 -1
  79. package/dashboard/static/assets/ProjectsDrillPanel-QLI9i5mZ.js +0 -1
  80. package/dashboard/static/assets/SourceDetailModal-pv0wlFxl.js +0 -1
  81. package/dashboard/static/assets/dashboardStream.shared-worker-1XTMV3nr.js +0 -1
  82. package/dashboard/static/assets/index-BHu4mxd8.css +0 -1
  83. package/dashboard/static/assets/index-CIWsbux3.js +0 -13
  84. package/dashboard/static/assets/outlineNavigation-CJvKqmLV.js +0 -9
  85. package/dashboard/static/assets/useKeymap-ffqsS5G0.js +0 -1
@@ -47,6 +47,11 @@ from _lib_codex_find_projection import (
47
47
  regex_ranges,
48
48
  slice_range_to_leaves,
49
49
  )
50
+ from _lib_codex_metadata import (
51
+ codex_metadata_is_malformed,
52
+ decode_codex_project_metadata,
53
+ resolve_codex_threads_table,
54
+ )
50
55
  from _lib_codex_title_clean import clean_codex_title
51
56
  from _lib_conversation import _strip_ansi
52
57
  from _lib_conversation_query import (
@@ -1451,23 +1456,113 @@ def _conversation_total_cost(conn: sqlite3.Connection, conversation_key: str, ef
1451
1456
  # ── rollup fields (dual-branch: stored fast path vs live recompute) ───────────
1452
1457
 
1453
1458
 
1459
+ #: The request-scoped set of conversations whose project metadata cannot be
1460
+ #: read. It is read from the RAW threads table, past any account-scoped TEMP
1461
+ #: view, and exposes nothing but key membership; attribution values keep coming
1462
+ #: from the scoped rollup.
1463
+ _UNDECODABLE_THREAD_KEYS_SQL = """
1464
+ SELECT conversation_key,
1465
+ CAST(cwd AS BLOB) AS cwd_blob,
1466
+ CAST(git_json AS BLOB) AS git_json_blob
1467
+ FROM {threads}
1468
+ """
1469
+
1470
+ #: `_thread_facts` keeps the UNQUALIFIED table name on purpose: it is a reader
1471
+ #: that must honor account scoping, so an account-scoped request sees the empty
1472
+ #: TEMP view exactly as it does today. Only the two columns change, to BLOBs.
1473
+ _THREAD_FACTS_SQL = (
1474
+ "SELECT native_thread_id, root_thread_id, parent_thread_id, source_root_key, "
1475
+ "CAST(cwd AS BLOB) AS cwd_blob, CAST(git_json AS BLOB) AS git_json_blob "
1476
+ "FROM codex_conversation_threads WHERE conversation_key = ?"
1477
+ )
1478
+
1479
+
1480
+ def undecodable_codex_conversation_keys(
1481
+ conn: sqlite3.Connection,
1482
+ ) -> "frozenset[str]":
1483
+ """Conversations whose project metadata the store cannot read (§4.9).
1484
+
1485
+ ONE frozen set per viewer request, materialized at the entry point and
1486
+ threaded through the whole call graph; no reader performs its own check.
1487
+ Membership is ``codex_metadata_is_malformed(direct, None)`` over the
1488
+ thread's own two fields — an undecodable ``cwd``, or an absent or empty
1489
+ ``cwd`` beside an undecodable ``git_json``. A valid ``cwd`` beside an
1490
+ undecodable ``git_json`` is NOT a member, because the viewer's attribution
1491
+ is direct-only through ``_codex_conversation_project_attribution``, which
1492
+ stops at a usable ``cwd``; the rollup writer, the stored path, the live
1493
+ path and this set therefore share one predicate.
1494
+
1495
+ The read goes to the RAW table ``resolve_codex_threads_table`` names, so an
1496
+ account-scoped request observes the same membership as an unscoped one,
1497
+ while attribution values keep coming from the scoped rollup. A store with
1498
+ no threads table at all has no undecodable conversation, so it answers the
1499
+ empty set rather than failing a reader that would otherwise have worked;
1500
+ that case reaches here as ``resolve_codex_threads_table``'s
1501
+ ``RuntimeError``, and it is the ONLY failure this answers with the empty
1502
+ set.
1503
+
1504
+ A ``sqlite3.Error`` is a read the store could not perform, not the answer
1505
+ "nothing is undecodable", so it propagates. Answering the empty set for one
1506
+ would republish each conversation's PRE-CORRUPTION stored attribution on
1507
+ every viewer surface, which is exactly the stale answer this set exists to
1508
+ withhold.
1509
+ """
1510
+ try:
1511
+ threads = resolve_codex_threads_table(conn)
1512
+ except RuntimeError:
1513
+ return frozenset()
1514
+ rows = conn.execute(_UNDECODABLE_THREAD_KEYS_SQL.format(threads=threads))
1515
+ keys = set()
1516
+ for conversation_key, cwd_blob, git_json_blob in rows:
1517
+ if not conversation_key:
1518
+ continue
1519
+ direct = decode_codex_project_metadata(cwd_blob, git_json_blob)
1520
+ if codex_metadata_is_malformed(direct, None):
1521
+ keys.add(str(conversation_key))
1522
+ return frozenset(keys)
1523
+
1524
+
1454
1525
  def _thread_facts(conn: sqlite3.Connection, conversation_key: str):
1455
- """``(native, root, parent, source_root_key, cwd, git_json)`` for a
1456
- conversation's thread, or ``None`` (no thread row / not-yet-linked)."""
1457
- return conn.execute(
1458
- "SELECT native_thread_id, root_thread_id, parent_thread_id, source_root_key, "
1459
- "cwd, git_json FROM codex_conversation_threads WHERE conversation_key = ?",
1460
- (conversation_key,),
1461
- ).fetchone()
1526
+ """``(native, root, parent, source_root_key, cwd, git_json, malformed)``
1527
+ for a conversation's thread, or ``None`` (no thread row / not-yet-linked).
1528
+
1529
+ The two metadata columns arrive as BLOBs and are decoded FIELD BY FIELD, so
1530
+ one undecodable byte no longer raises ``OperationalError`` for the whole
1531
+ statement and the three identity-only callers — the spawn-child link, the
1532
+ live-tail file set and the discovery frontier's walk root — keep working on
1533
+ a corrupted conversation. An undecodable field reads as ``None`` and the
1534
+ seventh member carries the direct-only malformed predicate, which is the
1535
+ same verdict ``_recompute_codex_rollups`` persists NULL attribution for.
1536
+ """
1537
+ row = conn.execute(_THREAD_FACTS_SQL, (conversation_key,)).fetchone()
1538
+ if row is None:
1539
+ return None
1540
+ native, root, parent, source_root_key, cwd_blob, git_json_blob = row
1541
+ direct = decode_codex_project_metadata(cwd_blob, git_json_blob)
1542
+ return (
1543
+ native, root, parent, source_root_key, direct.cwd, direct.git_json,
1544
+ codex_metadata_is_malformed(direct, None),
1545
+ )
1462
1546
 
1463
1547
 
1464
- def _rollup_fields(conn: sqlite3.Connection, conversation_key: str, rows: list | None = None):
1548
+ def _rollup_fields(
1549
+ conn: sqlite3.Connection, conversation_key: str, rows: list | None = None,
1550
+ *, undecodable_keys: "frozenset[str]" = frozenset(),
1551
+ ):
1465
1552
  """Rollup fields for a conversation — the stored rollup row when present
1466
1553
  (fast path), else a LIVE recompute that reproduces ``_recompute_codex_rollups``
1467
1554
  EXACTLY (§3.2 / §6.1): same kernel helpers (``rollup_item_count``,
1468
1555
  ``derive_title``), same min/max/sorted, and the SAME
1469
1556
  ``_codex_conversation_project_attribution`` the writer uses. Returns ``None``
1470
- when the conversation has no normalized rows."""
1557
+ when the conversation has no normalized rows.
1558
+
1559
+ ``undecodable_keys`` is the request's set (§4.9). A member's attribution is
1560
+ nulled on the stored fast path, which issues no query of its own, and the
1561
+ live branch never calls the attribution resolver for one. The thread-update
1562
+ trigger advances only the accounting mutation ledger, so a rollup
1563
+ materialized before a later corruption would otherwise keep that stale
1564
+ attribution on every viewer surface with no write to repair it.
1565
+ """
1471
1566
  stored = conn.execute(
1472
1567
  "SELECT item_count, started_utc, last_activity_utc, project_key, project_label, "
1473
1568
  "models_json, title, parent_thread_id, source_root_key "
@@ -1476,9 +1571,12 @@ def _rollup_fields(conn: sqlite3.Connection, conversation_key: str, rows: list |
1476
1571
  ).fetchone()
1477
1572
  thread = _thread_facts(conn, conversation_key)
1478
1573
  native = thread[0] if thread else None
1574
+ excluded = conversation_key in undecodable_keys
1479
1575
  if stored is not None:
1480
1576
  item_count, started, last, project_key, project_label, models_json, title, parent, srk = stored
1481
1577
  models = json.loads(models_json) if models_json else []
1578
+ if excluded:
1579
+ project_key = project_label = None
1482
1580
  return {
1483
1581
  "item_count": item_count, "started": started, "last": last,
1484
1582
  "project_key": project_key, "project_label": project_label,
@@ -1498,13 +1596,21 @@ def _rollup_fields(conn: sqlite3.Connection, conversation_key: str, rows: list |
1498
1596
  models = sorted({r.model for r in rows if r.model})
1499
1597
  source_root_key = rows[0].source_root_key
1500
1598
  cwd = git_json = parent = None
1599
+ malformed = False
1501
1600
  if thread is not None:
1502
- native, _root, parent, thread_root, cwd, git_json = thread
1601
+ native, _root, parent, thread_root, cwd, git_json, malformed = thread
1503
1602
  if thread_root:
1504
1603
  source_root_key = thread_root
1505
- from _cctally_cache import _codex_conversation_project_attribution
1506
- project_key, project_label = _codex_conversation_project_attribution(
1507
- source_root_key, cwd, git_json)
1604
+ if excluded or malformed:
1605
+ # Mirrors `_recompute_codex_rollups` exactly: a thread whose metadata
1606
+ # the store cannot read is NULL, never `(unassigned)`. `(unassigned)`
1607
+ # is a real answer for a thread with no metadata, and publishing it
1608
+ # here would be a wrong answer every later read prefers.
1609
+ project_key = project_label = None
1610
+ else:
1611
+ from _cctally_cache import _codex_conversation_project_attribution
1612
+ project_key, project_label = _codex_conversation_project_attribution(
1613
+ source_root_key, cwd, git_json)
1508
1614
  return {
1509
1615
  "item_count": item_count, "started": started, "last": last,
1510
1616
  "project_key": project_key, "project_label": project_label,
@@ -1534,20 +1640,28 @@ def _display_chain(fields: dict) -> str:
1534
1640
  or _short_native(fields.get("native_thread_id")) or "")
1535
1641
 
1536
1642
 
1537
- def _conversation_display_title(conn: sqlite3.Connection, conversation_key: str, rows: list | None = None) -> str:
1538
- fields = _rollup_fields(conn, conversation_key, rows=rows)
1643
+ def _conversation_display_title(
1644
+ conn: sqlite3.Connection, conversation_key: str, rows: list | None = None,
1645
+ *, undecodable_keys: "frozenset[str]" = frozenset(),
1646
+ ) -> str:
1647
+ fields = _rollup_fields(
1648
+ conn, conversation_key, rows=rows, undecodable_keys=undecodable_keys)
1539
1649
  if fields is None:
1540
1650
  return ""
1541
1651
  return _display_chain(fields)
1542
1652
 
1543
1653
 
1544
- def _conversation_hit_fields(conn: sqlite3.Connection, conversation_key: str):
1654
+ def _conversation_hit_fields(
1655
+ conn: sqlite3.Connection, conversation_key: str,
1656
+ *, undecodable_keys: "frozenset[str]" = frozenset(),
1657
+ ):
1545
1658
  """``(title, last_activity_utc, project_label)`` for a search hit's conversation
1546
1659
  (§3.7). ONE ``_rollup_fields`` resolution (stored fast path or the identical
1547
1660
  live recompute), so the neutral search hit carries the conversation-level
1548
1661
  last-activity time (explicitly NOT the matched row's own timestamp) and a
1549
1662
  nullable project label without a per-row lookup."""
1550
- fields = _rollup_fields(conn, conversation_key)
1663
+ fields = _rollup_fields(
1664
+ conn, conversation_key, undecodable_keys=undecodable_keys)
1551
1665
  if fields is None:
1552
1666
  return "", None, None
1553
1667
  return _display_chain(fields), fields.get("last"), fields.get("project_label")
@@ -1556,8 +1670,12 @@ def _conversation_hit_fields(conn: sqlite3.Connection, conversation_key: str):
1556
1670
  # ── threading (§5.5) ──────────────────────────────────────────────────────────
1557
1671
 
1558
1672
 
1559
- def _child_summary(conn: sqlite3.Connection, conversation_key: str, effective_speed: str) -> dict:
1560
- fields = _rollup_fields(conn, conversation_key)
1673
+ def _child_summary(
1674
+ conn: sqlite3.Connection, conversation_key: str, effective_speed: str,
1675
+ *, undecodable_keys: "frozenset[str]" = frozenset(),
1676
+ ) -> dict:
1677
+ fields = _rollup_fields(
1678
+ conn, conversation_key, undecodable_keys=undecodable_keys)
1561
1679
  return {
1562
1680
  "conversation_key": conversation_key,
1563
1681
  "title": _display_chain(fields) if fields else "",
@@ -1568,15 +1686,19 @@ def _child_summary(conn: sqlite3.Connection, conversation_key: str, effective_sp
1568
1686
  }
1569
1687
 
1570
1688
 
1571
- def _children_of(conn: sqlite3.Connection, conversation_key: str, effective_speed: str) -> list[dict]:
1689
+ def _children_of(
1690
+ conn: sqlite3.Connection, conversation_key: str, effective_speed: str,
1691
+ *, undecodable_keys: "frozenset[str]" = frozenset(),
1692
+ ) -> list[dict]:
1572
1693
  """Same-root threads whose ``parent_thread_id`` equals this thread's native
1573
1694
  id (§5.5). Never a filename inference — metadata only."""
1574
1695
  thread = _thread_facts(conn, conversation_key)
1575
1696
  if thread is None:
1576
1697
  return []
1577
- native, _root, _parent, source_root_key, _cwd, _git = thread
1698
+ native, _root, _parent, source_root_key, _cwd, _git, _malformed = thread
1578
1699
  children = [
1579
- _child_summary(conn, child_ck, effective_speed)
1700
+ _child_summary(conn, child_ck, effective_speed,
1701
+ undecodable_keys=undecodable_keys)
1580
1702
  for (child_ck,) in conn.execute(
1581
1703
  "SELECT conversation_key FROM codex_conversation_threads "
1582
1704
  "WHERE source_root_key = ? AND parent_thread_id = ? AND conversation_key != ?",
@@ -1587,7 +1709,10 @@ def _children_of(conn: sqlite3.Connection, conversation_key: str, effective_spee
1587
1709
  return children
1588
1710
 
1589
1711
 
1590
- def _parent_of(conn: sqlite3.Connection, conversation_key: str):
1712
+ def _parent_of(
1713
+ conn: sqlite3.Connection, conversation_key: str,
1714
+ *, undecodable_keys: "frozenset[str]" = frozenset(),
1715
+ ):
1591
1716
  """Parent pointer (§5.5): the same-root thread whose native id equals this
1592
1717
  thread's ``parent_thread_id``. A root (parent == self, or absent) has none;
1593
1718
  a fork whose parent is not ingested also returns ``None`` (no key to point
@@ -1595,7 +1720,7 @@ def _parent_of(conn: sqlite3.Connection, conversation_key: str):
1595
1720
  thread = _thread_facts(conn, conversation_key)
1596
1721
  if thread is None:
1597
1722
  return None
1598
- native, _root, parent, source_root_key, _cwd, _git = thread
1723
+ native, _root, parent, source_root_key, _cwd, _git, _malformed = thread
1599
1724
  if not parent or parent == native:
1600
1725
  return None
1601
1726
  prow = conn.execute(
@@ -1606,7 +1731,11 @@ def _parent_of(conn: sqlite3.Connection, conversation_key: str):
1606
1731
  if prow is None:
1607
1732
  return None
1608
1733
  parent_ck = prow[0]
1609
- return {"conversation_key": parent_ck, "title": _conversation_display_title(conn, parent_ck)}
1734
+ return {
1735
+ "conversation_key": parent_ck,
1736
+ "title": _conversation_display_title(
1737
+ conn, parent_ck, undecodable_keys=undecodable_keys),
1738
+ }
1610
1739
 
1611
1740
 
1612
1741
  def _consistent_meta_value(direct, nested):
@@ -1655,7 +1784,7 @@ def _spawn_child_link(
1655
1784
  thread = _thread_facts(conn, conversation_key)
1656
1785
  if thread is None:
1657
1786
  return None
1658
- native, _root, _parent, source_root_key, _cwd, _git = thread
1787
+ native, _root, _parent, source_root_key, _cwd, _git, _malformed = thread
1659
1788
  if not native or not source_root_key:
1660
1789
  return None
1661
1790
  matches: dict[str, list[dict]] = {}
@@ -1741,7 +1870,7 @@ def codex_conversation_source_paths(
1741
1870
  keys = [conversation_key]
1742
1871
  thread = _thread_facts(conn, conversation_key)
1743
1872
  if thread is not None:
1744
- native, _root, _parent, source_root_key, _cwd, _git = thread
1873
+ native, _root, _parent, source_root_key, _cwd, _git, _malformed = thread
1745
1874
  if native is not None:
1746
1875
  keys.extend(
1747
1876
  child_ck
@@ -2211,6 +2340,7 @@ def get_codex_conversation(
2211
2340
  rows, detail_bytes = _load_conversation_index_rows(conn, conversation_key)
2212
2341
  if not rows:
2213
2342
  return {"status": "not_found", "conversation_key": conversation_key}
2343
+ undecodable_keys = undecodable_codex_conversation_keys(conn)
2214
2344
  kept, _suppressed = kern.pair_mirrors(rows)
2215
2345
  items = kern.canonical_items(
2216
2346
  kept, fold_patch_completions=not legacy_export)
@@ -2334,12 +2464,16 @@ def get_codex_conversation(
2334
2464
  # fallback inside _rollup_fields derives the title from it. Passing None
2335
2465
  # keeps the stored fast path unchanged and lets the rare no-rollup case
2336
2466
  # do its own wide read rather than titling the conversation "".
2337
- "title": _conversation_display_title(conn, conversation_key),
2467
+ "title": _conversation_display_title(
2468
+ conn, conversation_key, undecodable_keys=undecodable_keys),
2338
2469
  "items": page_items,
2339
2470
  "page": page,
2340
2471
  "session_index": session_index,
2341
- "children": _children_of(conn, conversation_key, effective_speed),
2342
- "parent": _parent_of(conn, conversation_key),
2472
+ "children": _children_of(
2473
+ conn, conversation_key, effective_speed,
2474
+ undecodable_keys=undecodable_keys),
2475
+ "parent": _parent_of(
2476
+ conn, conversation_key, undecodable_keys=undecodable_keys),
2343
2477
  "total_cost_usd": total,
2344
2478
  "unattributed_cost_usd": unattributed_cost,
2345
2479
  "tokens": _tokens_union(conv_tokens),
@@ -2705,7 +2839,9 @@ def codex_conversation_cache_stats():
2705
2839
  return MappingProxyType(outline)
2706
2840
 
2707
2841
 
2708
- def _codex_outline_memo_key(conn, conversation_key, effective_speed):
2842
+ def _codex_outline_memo_key(
2843
+ conn, conversation_key, effective_speed, undecodable_keys=frozenset(),
2844
+ ):
2709
2845
  if not codex_normalization_authoritative(conn):
2710
2846
  _codex_outline_memo_clear()
2711
2847
  return None
@@ -2752,6 +2888,16 @@ def _codex_outline_memo_key(conn, conversation_key, effective_speed):
2752
2888
  seq_row = None
2753
2889
  accounting_revision = int(seq_row[0]) if seq_row else 0
2754
2890
 
2891
+ # #850 §4.9. The three revisions above are writer-owned, and the
2892
+ # thread-update trigger advances the accounting one only for a thread with
2893
+ # an accounting or alias path — so a message-only child's later corruption,
2894
+ # or its repair, would leave a warm key unchanged and the hit would return
2895
+ # before `_outline_envelope` ran. The digest of the request's set is what
2896
+ # makes both directions miss.
2897
+ undecodable_digest = hashlib.sha256(
2898
+ "\0".join(sorted(undecodable_keys)).encode("utf-8")
2899
+ ).hexdigest()
2900
+
2755
2901
  return (
2756
2902
  database_identity,
2757
2903
  account_scope,
@@ -2761,14 +2907,20 @@ def _codex_outline_memo_key(conn, conversation_key, effective_speed):
2761
2907
  event_max_id,
2762
2908
  accounting_revision,
2763
2909
  effective_speed,
2910
+ undecodable_digest,
2764
2911
  )
2765
2912
 
2766
2913
 
2767
- def _codex_outline_memoized(conn, conversation_key, effective_speed):
2768
- key = _codex_outline_memo_key(conn, conversation_key, effective_speed)
2914
+ def _codex_outline_memoized(
2915
+ conn, conversation_key, effective_speed,
2916
+ undecodable_keys: "frozenset[str]" = frozenset(),
2917
+ ):
2918
+ key = _codex_outline_memo_key(
2919
+ conn, conversation_key, effective_speed, undecodable_keys)
2769
2920
  if key is None:
2770
2921
  return _outline_envelope(
2771
- conn, conversation_key, effective_speed=effective_speed)
2922
+ conn, conversation_key, effective_speed=effective_speed,
2923
+ undecodable_keys=undecodable_keys)
2772
2924
  with _CODEX_OUTLINE_MEMO_LOCK:
2773
2925
  hit = _CODEX_OUTLINE_MEMO.get(key)
2774
2926
  if hit is not None:
@@ -2789,7 +2941,8 @@ def _codex_outline_memoized(conn, conversation_key, effective_speed):
2789
2941
  return flight.result
2790
2942
  try:
2791
2943
  body = _outline_envelope(
2792
- conn, conversation_key, effective_speed=effective_speed)
2944
+ conn, conversation_key, effective_speed=effective_speed,
2945
+ undecodable_keys=undecodable_keys)
2793
2946
  except BaseException as exc:
2794
2947
  with _CODEX_OUTLINE_MEMO_LOCK:
2795
2948
  flight.error = exc
@@ -2860,11 +3013,13 @@ def get_codex_conversation_outline(
2860
3013
  """
2861
3014
  with _read_snapshot(conn):
2862
3015
  return _codex_outline_memoized(
2863
- conn, conversation_key, effective_speed)
3016
+ conn, conversation_key, effective_speed,
3017
+ undecodable_codex_conversation_keys(conn))
2864
3018
 
2865
3019
 
2866
3020
  def _outline_envelope(
2867
- conn: sqlite3.Connection, conversation_key: str, *, effective_speed: str
3021
+ conn: sqlite3.Connection, conversation_key: str, *, effective_speed: str,
3022
+ undecodable_keys: "frozenset[str]" = frozenset(),
2868
3023
  ) -> dict:
2869
3024
  if not codex_normalization_authoritative(conn):
2870
3025
  return {"status": "normalization_pending", "conversation_key": conversation_key,
@@ -2988,7 +3143,9 @@ def _outline_envelope(
2988
3143
  failed_calls, outcome_positions, derivation),
2989
3144
  },
2990
3145
  "files": _conversation_files(segment_index, derivation),
2991
- "children": _children_of(conn, conversation_key, effective_speed),
3146
+ "children": _children_of(
3147
+ conn, conversation_key, effective_speed,
3148
+ undecodable_keys=undecodable_keys),
2992
3149
  }
2993
3150
 
2994
3151
 
@@ -3000,11 +3157,15 @@ def _is_fork(fields: dict) -> bool:
3000
3157
  return bool(parent) and parent != fields.get("native_thread_id")
3001
3158
 
3002
3159
 
3003
- def _browse_row(conn: sqlite3.Connection, conversation_key: str, effective_speed: str, fields: dict) -> dict:
3160
+ def _browse_row(
3161
+ conn: sqlite3.Connection, conversation_key: str, effective_speed: str,
3162
+ fields: dict, *, undecodable_keys: "frozenset[str]" = frozenset(),
3163
+ ) -> dict:
3004
3164
  return _browse_row_from_fields(
3005
3165
  conversation_key, fields,
3006
3166
  cost_usd=_conversation_total_cost(conn, conversation_key, effective_speed),
3007
- parent=_parent_of(conn, conversation_key),
3167
+ parent=_parent_of(
3168
+ conn, conversation_key, undecodable_keys=undecodable_keys),
3008
3169
  )
3009
3170
 
3010
3171
 
@@ -3078,12 +3239,16 @@ def _stored_rollups_present(conn: sqlite3.Connection) -> bool:
3078
3239
  "SELECT 1 FROM codex_conversation_rollups LIMIT 1").fetchone() is not None
3079
3240
 
3080
3241
 
3081
- def _live_browse_fields(conn: sqlite3.Connection) -> list[tuple[str, dict]]:
3242
+ def _live_browse_fields(
3243
+ conn: sqlite3.Connection,
3244
+ *, undecodable_keys: "frozenset[str]" = frozenset(),
3245
+ ) -> list[tuple[str, dict]]:
3082
3246
  """Live-recompute fallback used only while no stored rollups exist."""
3083
3247
  out = []
3084
3248
  for (conversation_key,) in conn.execute(
3085
3249
  "SELECT DISTINCT conversation_key FROM codex_conversation_messages"):
3086
- fields = _rollup_fields(conn, conversation_key)
3250
+ fields = _rollup_fields(
3251
+ conn, conversation_key, undecodable_keys=undecodable_keys)
3087
3252
  if fields is not None:
3088
3253
  out.append((conversation_key, fields))
3089
3254
  return out
@@ -3100,12 +3265,25 @@ def _facets_from_fields(fields_rows: list[tuple[str, dict]]) -> dict:
3100
3265
  ])
3101
3266
 
3102
3267
 
3103
- def _stored_browse_facets(conn: sqlite3.Connection) -> dict:
3268
+ def _stored_browse_facets(
3269
+ conn: sqlite3.Connection,
3270
+ *, undecodable_keys: "frozenset[str]" = frozenset(),
3271
+ ) -> dict:
3272
+ """Project and model facets from the persisted rollups.
3273
+
3274
+ A conversation in the request's set contributes no project facet: its
3275
+ stored attribution predates the corruption and no write repairs it, so
3276
+ counting it would publish a project membership the detail route no longer
3277
+ shows. Its models still count, because the model list is not derived from
3278
+ the thread's metadata.
3279
+ """
3104
3280
  fields_rows = []
3105
3281
  for conversation_key, project_key, project_label, models_json in conn.execute(
3106
3282
  "SELECT conversation_key, project_key, project_label, models_json "
3107
3283
  "FROM codex_conversation_rollups"
3108
3284
  ):
3285
+ if conversation_key in undecodable_keys:
3286
+ project_key = project_label = None
3109
3287
  try:
3110
3288
  parsed = json.loads(models_json) if models_json else []
3111
3289
  models = parsed if isinstance(parsed, list) else []
@@ -3119,12 +3297,28 @@ def _stored_browse_facets(conn: sqlite3.Connection) -> dict:
3119
3297
  return _facets_from_fields(fields_rows)
3120
3298
 
3121
3299
 
3122
- def _stored_filter_sql(alias: str, project_key: str | None, model: str | None):
3300
+ def _stored_filter_sql(
3301
+ alias: str, project_key: str | None, model: str | None,
3302
+ undecodable_keys: "frozenset[str]" = frozenset(),
3303
+ ):
3304
+ """The stored page's WHERE clause, and its exclusion of the request's set.
3305
+
3306
+ The exclusion rides on the PROJECT filter only. Under an active project
3307
+ filter a member must be neither counted under its stale project, nor
3308
+ returned by that filter, nor used as that filter's cursor row; an
3309
+ unfiltered page still lists it, with NULL attribution.
3310
+ """
3123
3311
  clauses = []
3124
3312
  params = []
3125
3313
  if project_key is not None:
3126
3314
  clauses.append(f"{alias}.project_key = ?")
3127
3315
  params.append(project_key)
3316
+ if undecodable_keys:
3317
+ ordered = sorted(undecodable_keys)
3318
+ placeholders = ",".join("?" for _ in ordered)
3319
+ clauses.append(
3320
+ f"{alias}.conversation_key NOT IN ({placeholders})")
3321
+ params.extend(ordered)
3128
3322
  if model is not None:
3129
3323
  # models_json is the writer's canonical JSON array. Searching for the
3130
3324
  # complete JSON string literal is exact and does not require JSON1.
@@ -3156,9 +3350,10 @@ def _page_costs(
3156
3350
  def _stored_browse_page(
3157
3351
  conn: sqlite3.Connection, *, effective_speed: str,
3158
3352
  project_key: str | None, model: str | None, limit: int,
3159
- cursor: str | None,
3353
+ cursor: str | None, undecodable_keys: "frozenset[str]" = frozenset(),
3160
3354
  ):
3161
- where_sql, filter_params = _stored_filter_sql("r", project_key, model)
3355
+ where_sql, filter_params = _stored_filter_sql(
3356
+ "r", project_key, model, undecodable_keys)
3162
3357
  total = conn.execute(
3163
3358
  f"SELECT COUNT(*) FROM codex_conversation_rollups r WHERE {where_sql}",
3164
3359
  filter_params,
@@ -3166,7 +3361,8 @@ def _stored_browse_page(
3166
3361
 
3167
3362
  cursor_row = None
3168
3363
  if cursor is not None:
3169
- cursor_where, cursor_params = _stored_filter_sql("c", project_key, model)
3364
+ cursor_where, cursor_params = _stored_filter_sql(
3365
+ "c", project_key, model, undecodable_keys)
3170
3366
  cursor_row = conn.execute(
3171
3367
  "SELECT COALESCE(c.last_activity_utc, ''), c.conversation_key "
3172
3368
  "FROM codex_conversation_rollups c "
@@ -3221,6 +3417,8 @@ def _stored_browse_page(
3221
3417
  models = parsed if isinstance(parsed, list) else []
3222
3418
  except (TypeError, json.JSONDecodeError):
3223
3419
  models = []
3420
+ if conversation_key in undecodable_keys:
3421
+ row_project_key = project_label = None
3224
3422
  fields = {
3225
3423
  "item_count": item_count,
3226
3424
  "started": started,
@@ -3235,6 +3433,8 @@ def _stored_browse_page(
3235
3433
  }
3236
3434
  parent = None
3237
3435
  if parent_key is not None:
3436
+ if parent_key in undecodable_keys:
3437
+ parent_project_label = None
3238
3438
  parent = {
3239
3439
  "conversation_key": parent_key,
3240
3440
  "title": _display_chain({
@@ -3253,10 +3453,25 @@ def _stored_browse_page(
3253
3453
  def list_codex_conversation_facets(conn: sqlite3.Connection) -> dict:
3254
3454
  """Facet-only browse projection; never builds or prices a discarded page."""
3255
3455
  if not codex_normalization_authoritative(conn):
3456
+ # #717. The empty facets are not new; saying WHY they are empty is.
3457
+ #
3458
+ # Model counts are NOT derivable in this state, and that is a decision
3459
+ # rather than an omission. Claude's model facet folds
3460
+ # `conversation_messages`, which no rollup rebuild touches, so it stays
3461
+ # live. Every Codex facet source — the stored rollups and the live
3462
+ # recompute alike — reads the normalized corpus that the pending
3463
+ # replay is about to rewrite, so deriving counts from it would publish
3464
+ # numbers the replay is going to change.
3256
3465
  return {"status": "normalization_pending",
3257
- "facets": {"projects": [], "models": []}}
3258
- facets = (_stored_browse_facets(conn) if _stored_rollups_present(conn)
3259
- else _facets_from_fields(_live_browse_fields(conn)))
3466
+ "facets": {"projects": [], "models": []},
3467
+ "filter_degraded": True}
3468
+ undecodable_keys = undecodable_codex_conversation_keys(conn)
3469
+ facets = (
3470
+ _stored_browse_facets(conn, undecodable_keys=undecodable_keys)
3471
+ if _stored_rollups_present(conn)
3472
+ else _facets_from_fields(
3473
+ _live_browse_fields(conn, undecodable_keys=undecodable_keys))
3474
+ )
3260
3475
  return {"status": "ok", "facets": facets}
3261
3476
 
3262
3477
 
@@ -3279,22 +3494,27 @@ def list_codex_conversations(
3279
3494
  if not codex_normalization_authoritative(conn):
3280
3495
  return {"status": "normalization_pending", "rows": [],
3281
3496
  "facets": {"projects": [], "models": []}, "page": {"total": 0}}
3497
+ undecodable_keys = undecodable_codex_conversation_keys(conn)
3282
3498
  if _stored_rollups_present(conn):
3283
- facets = _stored_browse_facets(conn)
3499
+ facets = _stored_browse_facets(conn, undecodable_keys=undecodable_keys)
3284
3500
  page_rows, page = _stored_browse_page(
3285
3501
  conn, effective_speed=effective_speed, project_key=project_key,
3286
- model=model, limit=limit, cursor=cursor)
3502
+ model=model, limit=limit, cursor=cursor,
3503
+ undecodable_keys=undecodable_keys)
3287
3504
  result = {"status": "ok", "rows": page_rows, "facets": facets, "page": page}
3288
3505
  if selected is not None:
3289
- fields = _rollup_fields(conn, selected)
3506
+ fields = _rollup_fields(
3507
+ conn, selected, undecodable_keys=undecodable_keys)
3290
3508
  if fields is not None:
3291
3509
  result["selected"] = _browse_row(
3292
- conn, selected, effective_speed, fields)
3510
+ conn, selected, effective_speed, fields,
3511
+ undecodable_keys=undecodable_keys)
3293
3512
  return result
3294
3513
 
3295
- fields_rows = _live_browse_fields(conn)
3514
+ fields_rows = _live_browse_fields(conn, undecodable_keys=undecodable_keys)
3296
3515
  rows = [
3297
- _browse_row(conn, conversation_key, effective_speed, fields)
3516
+ _browse_row(conn, conversation_key, effective_speed, fields,
3517
+ undecodable_keys=undecodable_keys)
3298
3518
  for conversation_key, fields in fields_rows
3299
3519
  ]
3300
3520
  facets = _facets_from_fields(fields_rows)
@@ -3899,6 +4119,7 @@ def _search_excerpt(text: str | None, query: str, width: int = 200) -> str:
3899
4119
 
3900
4120
  def _collapse_message_hits(
3901
4121
  conn: sqlite3.Connection, matched_rows: list, query: str,
4122
+ *, undecodable_keys: "frozenset[str]" = frozenset(),
3902
4123
  ) -> list[dict]:
3903
4124
  """Collapse matched physical rows to canonical ``item_key`` BEFORE totals /
3904
4125
  badges (§6.2) — both members of a mirror pair map to one item_key, so mirror
@@ -3910,7 +4131,8 @@ def _collapse_message_hits(
3910
4131
  collapsed: dict[tuple, dict] = {}
3911
4132
  for ck, mrows in by_conv.items():
3912
4133
  pos_map = _pos_to_item_key(conn, ck)
3913
- title, last_act, project_label = _conversation_hit_fields(conn, ck)
4134
+ title, last_act, project_label = _conversation_hit_fields(
4135
+ conn, ck, undecodable_keys=undecodable_keys)
3914
4136
  for source_path, line_offset, kind, disp in mrows:
3915
4137
  item_key = pos_map.get((source_path, line_offset))
3916
4138
  if item_key is None:
@@ -3931,7 +4153,10 @@ def _collapse_message_hits(
3931
4153
  ]
3932
4154
 
3933
4155
 
3934
- def _search_title(conn: sqlite3.Connection, query: str) -> list[dict]:
4156
+ def _search_title(
4157
+ conn: sqlite3.Connection, query: str,
4158
+ *, undecodable_keys: "frozenset[str]" = frozenset(),
4159
+ ) -> list[dict]:
3935
4160
  """Title search over the rollup table — identical LIKE semantics in both FTS
3936
4161
  and LIKE modes (§6.2). Conversation-level hits (no item anchor).
3937
4162
 
@@ -3950,11 +4175,16 @@ def _search_title(conn: sqlite3.Connection, query: str) -> list[dict]:
3950
4175
  hits.append(
3951
4176
  {"conversation_key": ck, "item_key": None, "title": cleaned,
3952
4177
  "snippet": _excerpt(cleaned), "badges": ["title"],
3953
- "last_activity_utc": last_act, "project_label": project_label})
4178
+ "last_activity_utc": last_act,
4179
+ "project_label": (
4180
+ None if ck in undecodable_keys else project_label)})
3954
4181
  return hits
3955
4182
 
3956
4183
 
3957
- def _search_files(conn: sqlite3.Connection, query: str) -> list[dict]:
4184
+ def _search_files(
4185
+ conn: sqlite3.Connection, query: str,
4186
+ *, undecodable_keys: "frozenset[str]" = frozenset(),
4187
+ ) -> list[dict]:
3958
4188
  """File-touch search — matches file paths, collapsed to the owning message's
3959
4189
  canonical item_key (§6.2). ``message_id`` is an application-level link, so
3960
4190
  an orphan is skipped and cannot suppress valid rows."""
@@ -3973,7 +4203,8 @@ def _search_files(conn: sqlite3.Connection, query: str) -> list[dict]:
3973
4203
  continue
3974
4204
  if ck not in pos_cache:
3975
4205
  pos_cache[ck] = _pos_to_item_key(conn, ck)
3976
- fields_cache[ck] = _conversation_hit_fields(conn, ck)
4206
+ fields_cache[ck] = _conversation_hit_fields(
4207
+ conn, ck, undecodable_keys=undecodable_keys)
3977
4208
  item_key = pos_cache[ck].get((member[0], member[1]))
3978
4209
  title, last_act, project_label = fields_cache[ck]
3979
4210
  hit = collapsed.setdefault(
@@ -4026,13 +4257,15 @@ def search_codex_conversations(
4026
4257
  "total": 0, "mode": mode, "depth": "full"}
4027
4258
  if kind not in CODEX_SEARCH_KINDS:
4028
4259
  kind = "all"
4260
+ undecodable_keys = undecodable_codex_conversation_keys(conn)
4029
4261
  if kind == "title":
4030
- hits = _search_title(conn, query)
4262
+ hits = _search_title(conn, query, undecodable_keys=undecodable_keys)
4031
4263
  elif kind == "files":
4032
- hits = _search_files(conn, query)
4264
+ hits = _search_files(conn, query, undecodable_keys=undecodable_keys)
4033
4265
  else:
4034
4266
  hits = _collapse_message_hits(
4035
- conn, _matched_message_rows(conn, query, kind, mode), query)
4267
+ conn, _matched_message_rows(conn, query, kind, mode), query,
4268
+ undecodable_keys=undecodable_keys)
4036
4269
  hits.sort(key=lambda h: (h["conversation_key"], h["item_key"] or ""))
4037
4270
  total = len(hits)
4038
4271
  page_hits, page = _paginate_hits(hits, cursor=cursor, limit=limit)