cctally 1.91.0 → 1.92.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/CHANGELOG.md +51 -0
  2. package/README.md +4 -2
  3. package/bin/_cctally_cache.py +903 -74
  4. package/bin/_cctally_config.py +57 -0
  5. package/bin/_cctally_core.py +94 -14
  6. package/bin/_cctally_dashboard.py +217 -19
  7. package/bin/_cctally_dashboard_conversation.py +170 -20
  8. package/bin/_cctally_dashboard_envelope.py +2 -0
  9. package/bin/_cctally_db.py +481 -19
  10. package/bin/_cctally_doctor.py +18 -1
  11. package/bin/_cctally_journal.py +1156 -21
  12. package/bin/_cctally_journal_repair.py +6 -0
  13. package/bin/_cctally_parser.py +26 -0
  14. package/bin/_cctally_quota.py +171 -55
  15. package/bin/_cctally_record.py +13 -1
  16. package/bin/_cctally_rederive.py +4 -0
  17. package/bin/_cctally_statusline.py +6 -6
  18. package/bin/_cctally_store.py +1061 -40
  19. package/bin/_cctally_transcript.py +32 -2
  20. package/bin/_cctally_tui.py +54 -6
  21. package/bin/_lib_cache_report.py +8 -3
  22. package/bin/_lib_codex_conversation.py +851 -81
  23. package/bin/_lib_codex_conversation_query.py +2031 -96
  24. package/bin/_lib_codex_find_projection.py +517 -0
  25. package/bin/_lib_codex_harness_preamble.py +176 -0
  26. package/bin/_lib_codex_hooks.py +5 -3
  27. package/bin/_lib_codex_js_scan.py +254 -0
  28. package/bin/_lib_codex_landmarks.py +309 -0
  29. package/bin/_lib_codex_title_clean.py +116 -0
  30. package/bin/_lib_conversation_dispatch.py +168 -22
  31. package/bin/_lib_conversation_query.py +62 -2
  32. package/bin/_lib_conversation_watch.py +4 -2
  33. package/bin/_lib_doctor.py +64 -0
  34. package/bin/_lib_quota_alert_axes.py +31 -34
  35. package/bin/_lib_stats_damage.py +523 -0
  36. package/bin/_lib_stats_publish.py +243 -0
  37. package/bin/cctally +17 -3
  38. package/dashboard/static/assets/index-Dat-mza6.js +97 -0
  39. package/dashboard/static/assets/{index-Dwirao3Y.css → index-DnWdv8um.css} +1 -1
  40. package/dashboard/static/dashboard.html +2 -2
  41. package/package.json +8 -1
  42. package/dashboard/static/assets/index-CILAoEja.js +0 -90
@@ -127,6 +127,7 @@ def _cctally():
127
127
  # for ``eprint`` is deleted.
128
128
  import _cctally_core
129
129
  from _cctally_core import eprint
130
+ from _lib_codex_find_projection import CODEX_FIND_PROJECTION_VERSION
130
131
  from _lib_source_identity import source_root_key
131
132
  # #416 spec §4.2: the pure tolerance-anchored reset kernel. `_lib_quota` imports
132
133
  # only `_lib_accounts` (a stdlib leaf), so binding it here is circular-safe.
@@ -227,6 +228,12 @@ _CONV_INSERT_SQL = (
227
228
  " search_tool,search_thinking)"
228
229
  " VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
229
230
  )
231
+ _CONV_ACCOUNT_INSERT_SQL = _CONV_INSERT_SQL.replace(
232
+ "search_tool,search_thinking)", "search_tool,search_thinking,account_key)"
233
+ ).replace(
234
+ "?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
235
+ "?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
236
+ )
230
237
 
231
238
  # #193: last non-null write wins (ai-title carries no timestamp; see spec S1). NO
232
239
  # byte_offset guard — it can't order a cross-file resumed session. Ordering is
@@ -383,7 +390,7 @@ SESSION_ENTRY_UPSERT_SQL_REWALK = (
383
390
  + _SESSION_ENTRY_REWALK_PHYSICAL_GUARD)
384
391
 
385
392
 
386
- def _conv_row_tuple(m, path_str):
393
+ def _conv_row_tuple(m, path_str, account_key=None, *, include_account=False):
387
394
  """Flatten a ``MessageRow`` into the ``_CONV_INSERT_SQL`` column order.
388
395
 
389
396
  The #177 enrichment fields (stop_reason / attribution_skill /
@@ -393,7 +400,7 @@ def _conv_row_tuple(m, path_str):
393
400
  one tuple. #217 S1 / U7a: the documented-dead ``search_aux`` column is gone
394
401
  from the live schema (dropped by migration 016); the split
395
402
  ``search_tool``/``search_thinking`` columns carry the non-prose index."""
396
- return (
403
+ row = (
397
404
  m.session_id, m.uuid, m.parent_uuid, path_str, m.byte_offset,
398
405
  m.timestamp_utc, m.entry_type, m.text, m.blocks_json, m.model,
399
406
  m.msg_id, m.req_id, m.cwd, m.git_branch, m.is_sidechain,
@@ -401,6 +408,7 @@ def _conv_row_tuple(m, path_str):
401
408
  m.stop_reason, m.attribution_skill, m.attribution_plugin,
402
409
  m.search_tool, m.search_thinking,
403
410
  )
411
+ return row + ((account_key,) if include_account else ())
404
412
 
405
413
 
406
414
  def _iter_sync_entries(
@@ -1300,15 +1308,24 @@ def _codex_provider_roots() -> list[CodexProviderRoot]:
1300
1308
  roots: list[CodexProviderRoot] = []
1301
1309
  seen: set[pathlib.Path] = set()
1302
1310
  for configured in _cctally()._codex_home_roots():
1303
- provider_root = _canonical_codex_path(configured)
1311
+ try:
1312
+ provider_root = _canonical_codex_path(configured)
1313
+ except OSError:
1314
+ # An unreadable configured root is unknown filesystem state. Leave
1315
+ # it out of discovery; the prune scope still retains its configured
1316
+ # identity and will refuse destructive cleanup for that root.
1317
+ continue
1304
1318
  if provider_root in seen:
1305
1319
  continue
1306
1320
  sessions = configured / "sessions"
1307
- if sessions.is_dir():
1308
- walk_root = sessions
1309
- elif configured.is_dir():
1310
- walk_root = configured
1311
- else:
1321
+ try:
1322
+ if sessions.is_dir():
1323
+ walk_root = sessions
1324
+ elif configured.is_dir():
1325
+ walk_root = configured
1326
+ else:
1327
+ continue
1328
+ except OSError:
1312
1329
  continue
1313
1330
  seen.add(provider_root)
1314
1331
  roots.append(CodexProviderRoot(
@@ -1438,6 +1455,214 @@ def _discover_codex_files_with_roots() -> list[CodexDiscoveredFile]:
1438
1455
  return discovered
1439
1456
 
1440
1457
 
1458
+ CODEX_ORPHAN_PRUNE_REFUSED_KEY = "codex_orphan_prune_refused"
1459
+ _CODEX_ROOT_RECOGNITION_TYPES = frozenset({
1460
+ "session_meta", "turn_context", "event_msg", "response_item", "world_state",
1461
+ "compacted",
1462
+ })
1463
+ _CODEX_ROOT_RECOGNITION_MAX_FILES = 32
1464
+ _CODEX_ROOT_RECOGNITION_MAX_BYTES = 64 * 1024
1465
+ _CODEX_ROOT_RECOGNITION_MAX_LINES = 64
1466
+
1467
+
1468
+ def _codex_file_is_recognizable(path: pathlib.Path) -> bool:
1469
+ """Boundedly prove that one JSONL contains a known Codex record envelope.
1470
+
1471
+ Unknown records remain ingestible for forward compatibility, but they are
1472
+ not strong enough evidence to authorize deletion of retained history.
1473
+ """
1474
+ consumed = 0
1475
+ lines = 0
1476
+ try:
1477
+ with path.open("rb") as fh:
1478
+ while (
1479
+ lines < _CODEX_ROOT_RECOGNITION_MAX_LINES
1480
+ and consumed < _CODEX_ROOT_RECOGNITION_MAX_BYTES
1481
+ ):
1482
+ remaining = _CODEX_ROOT_RECOGNITION_MAX_BYTES - consumed
1483
+ raw = fh.readline(remaining + 1)
1484
+ if not raw:
1485
+ break
1486
+ lines += 1
1487
+ consumed += len(raw)
1488
+ if len(raw) > remaining or not raw.endswith(b"\n"):
1489
+ break
1490
+ try:
1491
+ obj = json.loads(
1492
+ raw,
1493
+ parse_constant=_lib_jsonl._reject_nonfinite_json_constant,
1494
+ )
1495
+ except (UnicodeDecodeError, ValueError, TypeError):
1496
+ continue
1497
+ if (
1498
+ isinstance(obj, dict)
1499
+ and obj.get("type") in _CODEX_ROOT_RECOGNITION_TYPES
1500
+ and isinstance(obj.get("payload"), dict)
1501
+ ):
1502
+ return True
1503
+ except OSError:
1504
+ return False
1505
+ return False
1506
+
1507
+
1508
+ @dataclass(frozen=True)
1509
+ class _CodexPruneScope:
1510
+ """Filesystem evidence that may authorize one whole-tree orphan prune.
1511
+
1512
+ A configured root is only *recognized* after a bounded probe finds at least
1513
+ one known Codex record envelope under it. A missing, empty, or unrelated
1514
+ directory therefore remains protected. When a current root is recognized,
1515
+ roots no longer present in the configured set may still be removed — that
1516
+ preserves issue #108's deliberate A -> B cache scoping. Within a recognized
1517
+ root, a missing file is deletion evidence only while its parent directory is
1518
+ still readable; losing an entire mounted subtree fails closed.
1519
+ """
1520
+
1521
+ configured_root_keys: frozenset[str]
1522
+ recognized_root_keys: frozenset[str]
1523
+
1524
+ def refusal_reason(self, source_path: str, root_key: str | None) -> str | None:
1525
+ if not self.recognized_root_keys:
1526
+ return "no_recognizable_roots"
1527
+ if (
1528
+ root_key in self.configured_root_keys
1529
+ and root_key not in self.recognized_root_keys
1530
+ ):
1531
+ return "configured_root_unrecognizable"
1532
+ if root_key in self.recognized_root_keys:
1533
+ parent = pathlib.Path(source_path).parent
1534
+ try:
1535
+ if not parent.is_dir() or not os.access(parent, os.R_OK | os.X_OK):
1536
+ return "source_parent_unavailable"
1537
+ except OSError:
1538
+ return "source_parent_unavailable"
1539
+ return None
1540
+
1541
+ def root_refusal_reason(self, root_key: str) -> str | None:
1542
+ if not self.recognized_root_keys:
1543
+ return "no_recognizable_roots"
1544
+ if (
1545
+ root_key in self.configured_root_keys
1546
+ and root_key not in self.recognized_root_keys
1547
+ ):
1548
+ return "configured_root_unrecognizable"
1549
+ return None
1550
+
1551
+
1552
+ def _codex_prune_scope(files: list[CodexDiscoveredFile]) -> _CodexPruneScope:
1553
+ configured: set[str] = set()
1554
+ for raw_root in _cctally()._codex_home_roots():
1555
+ try:
1556
+ provider_root = _canonical_codex_path(raw_root)
1557
+ configured.add(source_root_key(str(provider_root)))
1558
+ except (OSError, TypeError, ValueError):
1559
+ continue
1560
+ recognized: set[str] = set()
1561
+ checked_per_root: dict[str, int] = {}
1562
+ # Discovery is stable oldest-first for Codex's dated rollout paths. Probe
1563
+ # from the tail so a bounded check sees the current envelope generation,
1564
+ # rather than spending its entire budget on a large legacy prefix. A root
1565
+ # containing only legacy/unknown envelopes still fails closed.
1566
+ for item in reversed(files):
1567
+ if item.source_root_key in recognized:
1568
+ continue
1569
+ checked = checked_per_root.get(item.source_root_key, 0)
1570
+ if checked >= _CODEX_ROOT_RECOGNITION_MAX_FILES:
1571
+ continue
1572
+ checked_per_root[item.source_root_key] = checked + 1
1573
+ if _codex_file_is_recognizable(item.source_path):
1574
+ recognized.add(item.source_root_key)
1575
+ return _CodexPruneScope(
1576
+ configured_root_keys=frozenset(configured),
1577
+ recognized_root_keys=frozenset(recognized),
1578
+ )
1579
+
1580
+
1581
+ def _partition_codex_prune_candidates(
1582
+ scope: _CodexPruneScope,
1583
+ sources: list[tuple[str, str | None]],
1584
+ root_keys: set[str],
1585
+ ) -> tuple[
1586
+ list[tuple[str, str | None]], list[tuple[str, str | None]], set[str], set[str]
1587
+ ]:
1588
+ safe_sources: list[tuple[str, str | None]] = []
1589
+ refused_sources: list[tuple[str, str | None]] = []
1590
+ for source in sources:
1591
+ target = refused_sources if scope.refusal_reason(*source) else safe_sources
1592
+ target.append(source)
1593
+ safe_roots: set[str] = set()
1594
+ refused_roots: set[str] = set()
1595
+ for root_key in root_keys:
1596
+ target = refused_roots if scope.root_refusal_reason(root_key) else safe_roots
1597
+ target.add(root_key)
1598
+ return safe_sources, refused_sources, safe_roots, refused_roots
1599
+
1600
+
1601
+ def _codex_prune_refusal_record(
1602
+ conn: sqlite3.Connection,
1603
+ *,
1604
+ store: str,
1605
+ scope: _CodexPruneScope,
1606
+ refused_sources: list[tuple[str, str | None]],
1607
+ refused_root_keys: set[str],
1608
+ ) -> None:
1609
+ """Persist a privacy-safe refusal signal in the store that declined."""
1610
+ now = (
1611
+ _cctally_core._command_as_of()
1612
+ .astimezone(dt.timezone.utc)
1613
+ .isoformat(timespec="seconds")
1614
+ .replace("+00:00", "Z")
1615
+ )
1616
+ since = now
1617
+ row = conn.execute(
1618
+ "SELECT value FROM cache_meta WHERE key=?",
1619
+ (CODEX_ORPHAN_PRUNE_REFUSED_KEY,),
1620
+ ).fetchone()
1621
+ if row and row[0]:
1622
+ try:
1623
+ previous = json.loads(row[0])
1624
+ if isinstance(previous, dict) and isinstance(previous.get("since"), str):
1625
+ since = previous["since"]
1626
+ except (TypeError, ValueError):
1627
+ pass
1628
+ reasons = sorted({
1629
+ reason
1630
+ for path, root_key in refused_sources
1631
+ if (reason := scope.refusal_reason(path, root_key)) is not None
1632
+ } | {
1633
+ reason
1634
+ for root_key in refused_root_keys
1635
+ if (reason := scope.root_refusal_reason(root_key)) is not None
1636
+ })
1637
+ record = {
1638
+ "schemaVersion": 1,
1639
+ "store": store,
1640
+ "since": since,
1641
+ "at": now,
1642
+ "reasons": reasons,
1643
+ "configuredRootCount": len(scope.configured_root_keys),
1644
+ "recognizedRootCount": len(scope.recognized_root_keys),
1645
+ "preservedFileCount": len({path for path, _root in refused_sources}),
1646
+ "preservedRootCount": len(refused_root_keys),
1647
+ }
1648
+ conn.execute(
1649
+ "INSERT INTO cache_meta(key,value) VALUES(?,?) "
1650
+ "ON CONFLICT(key) DO UPDATE SET value=excluded.value",
1651
+ (CODEX_ORPHAN_PRUNE_REFUSED_KEY, json.dumps(record, sort_keys=True)),
1652
+ )
1653
+ eprint(
1654
+ f"[codex-{store}] orphan prune refused: preserved "
1655
+ f"{record['preservedFileCount']} tracked file(s); "
1656
+ "configured Codex roots were not sufficient deletion evidence"
1657
+ )
1658
+
1659
+
1660
+ def _clear_codex_prune_refusal(conn: sqlite3.Connection) -> None:
1661
+ conn.execute(
1662
+ "DELETE FROM cache_meta WHERE key=?", (CODEX_ORPHAN_PRUNE_REFUSED_KEY,)
1663
+ )
1664
+
1665
+
1441
1666
  def _qualify_codex_targets(only_paths: "set[str]") -> list[CodexDiscoveredFile]:
1442
1667
  """Resolve each requested path through the ordered configured roots exactly
1443
1668
  as full discovery would (spec §5.1) — producing the same per-file facts a
@@ -1603,7 +1828,12 @@ def _clear_codex_derived_rows(conn: sqlite3.Connection) -> bool:
1603
1828
 
1604
1829
 
1605
1830
  def _bump_codex_physical_mutation_seq(conn: sqlite3.Connection) -> None:
1606
- """Advance the dashboard's Codex physical-identity sequence in this txn."""
1831
+ """Advance the shared Codex physical-state invalidation token in this txn.
1832
+
1833
+ Dashboard/TUI snapshot versions and the quota projection certificate both
1834
+ consume this value. Every writer that changes their cache.db inputs must
1835
+ advance it in the same transaction as that change.
1836
+ """
1607
1837
  conn.execute(
1608
1838
  "INSERT INTO cache_meta(key, value) VALUES "
1609
1839
  "('codex_physical_mutation_seq', '1') "
@@ -1622,6 +1852,10 @@ _CODEX_MSG_INSERT_SQL = (
1622
1852
  "INSERT OR IGNORE INTO codex_conversation_messages (" + _CODEX_NORM_COLS + ") "
1623
1853
  "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
1624
1854
  )
1855
+ _CODEX_ACCOUNT_MSG_INSERT_SQL = (
1856
+ "INSERT OR IGNORE INTO codex_conversation_messages (" + _CODEX_NORM_COLS
1857
+ + ", account_key) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
1858
+ )
1625
1859
 
1626
1860
 
1627
1861
  def _codex_conversation_project_attribution(
@@ -1679,17 +1913,29 @@ def _load_codex_normalized_rows(
1679
1913
  ]
1680
1914
 
1681
1915
 
1682
- def _insert_codex_normalized_rows(conn: sqlite3.Connection, rows: list, touches: list) -> None:
1916
+ def _insert_codex_normalized_rows(
1917
+ conn: sqlite3.Connection,
1918
+ rows: list,
1919
+ touches: list,
1920
+ account_by_physical: "dict[tuple[str, int], str | None] | None" = None,
1921
+ ) -> None:
1683
1922
  """Insert normalized rows (INSERT OR IGNORE on the physical key) + their file
1684
1923
  touches (message linkage resolved via (source_path, line_offset))."""
1685
1924
  if rows:
1686
- conn.executemany(_CODEX_MSG_INSERT_SQL, [
1925
+ with_accounts = account_by_physical is not None
1926
+ account_by_physical = account_by_physical or {}
1927
+ conn.executemany(
1928
+ _CODEX_ACCOUNT_MSG_INSERT_SQL if with_accounts else _CODEX_MSG_INSERT_SQL,
1929
+ [
1687
1930
  (r.conversation_key, r.source_root_key, r.source_path, r.line_offset,
1688
1931
  r.timestamp_utc, r.turn_id, r.call_id, r.kind, r.event_type,
1689
1932
  r.record_family, r.model, r.text, r.content_digest, r.content_len,
1690
1933
  r.detail_json, r.search_tool, r.search_thinking)
1934
+ + ((account_by_physical.get((r.source_path, r.line_offset)),)
1935
+ if with_accounts else ())
1691
1936
  for r in rows
1692
- ])
1937
+ ],
1938
+ )
1693
1939
  for touch in touches:
1694
1940
  conn.execute(
1695
1941
  "INSERT OR IGNORE INTO codex_conversation_file_touches "
@@ -1764,26 +2010,125 @@ def _replay_codex_normalization(conn: sqlite3.Connection) -> None:
1764
2010
  the plain rowid alias make a re-run byte-idempotent."""
1765
2011
  kern = _lib_codex_conversation
1766
2012
  events_by_file: dict[str, list] = {}
2013
+ accounts_by_file: dict[str, dict[tuple[str, int], str | None]] = {}
1767
2014
  order: list[str] = []
2015
+ has_account = "account_key" in {
2016
+ str(info[1])
2017
+ for info in conn.execute("PRAGMA table_info(codex_conversation_events)")
2018
+ }
2019
+ account_select = ", account_key" if has_account else ""
1768
2020
  for row in conn.execute(
1769
2021
  "SELECT source_path, line_offset, source_root_key, conversation_key, "
1770
2022
  "native_thread_id, root_thread_id, parent_thread_id, timestamp_utc, "
1771
2023
  "record_type, event_type, turn_id, call_id, payload_json "
2024
+ + account_select + " "
1772
2025
  "FROM codex_conversation_events "
1773
2026
  "ORDER BY source_path ASC, line_offset ASC"
1774
2027
  ):
1775
- event = _lib_jsonl.CodexPhysicalEvent(*row)
2028
+ event = _lib_jsonl.CodexPhysicalEvent(*(row[:-1] if has_account else row))
1776
2029
  if event.source_path not in events_by_file:
1777
2030
  events_by_file[event.source_path] = []
2031
+ accounts_by_file[event.source_path] = {}
1778
2032
  order.append(event.source_path)
1779
2033
  events_by_file[event.source_path].append(event)
2034
+ if has_account:
2035
+ accounts_by_file[event.source_path][
2036
+ (event.source_path, int(event.line_offset))
2037
+ ] = row[-1]
1780
2038
  affected: set = set()
1781
2039
  for source_path in order:
1782
2040
  result = kern.normalize_codex_events(
1783
2041
  events_by_file[source_path], initial=kern.CodexStickyState())
1784
- _insert_codex_normalized_rows(conn, result.rows, result.touches)
2042
+ _insert_codex_normalized_rows(
2043
+ conn, result.rows, result.touches,
2044
+ accounts_by_file[source_path] if has_account else None,
2045
+ )
1785
2046
  affected.update(r.conversation_key for r in result.rows)
1786
2047
  _recompute_codex_rollups(conn, affected)
2048
+ # Migration 025 still replays this helper against legacy cache.db fixtures,
2049
+ # where the conversations-only #482 projection table does not exist. Live
2050
+ # conversations.db replays must refresh the projection, but the historical
2051
+ # cache migration must remain byte-stable and cannot create a newer store's
2052
+ # derivation out of order.
2053
+ projection_exists = conn.execute(
2054
+ "SELECT 1 FROM sqlite_master "
2055
+ "WHERE type='table' AND name='codex_find_projection'"
2056
+ ).fetchone() is not None
2057
+ if projection_exists:
2058
+ import _lib_codex_conversation_query as query
2059
+ query.materialize_codex_find_projection(conn, affected)
2060
+
2061
+
2062
+ def run_codex_find_projection_backfill(
2063
+ conn: sqlite3.Connection,
2064
+ *,
2065
+ batch_size: int = 400,
2066
+ ) -> dict[str, object]:
2067
+ """Consume one resumable #482 projection batch from retained rows.
2068
+
2069
+ The caller owns the Codex conversations provider flock. Progress is keyed
2070
+ by the normalized message rowid, but a selected conversation is always
2071
+ rebuilt as a whole so native folds never straddle a batch boundary.
2072
+ """
2073
+ pending = conn.execute(
2074
+ "SELECT 1 FROM cache_meta "
2075
+ "WHERE key='codex_find_projection_backfill_pending'"
2076
+ ).fetchone()
2077
+ if pending is None:
2078
+ complete = conn.execute(
2079
+ "SELECT 1 FROM cache_meta "
2080
+ "WHERE key='codex_find_projection_complete_version' AND value=?",
2081
+ (str(CODEX_FIND_PROJECTION_VERSION),),
2082
+ ).fetchone() is not None
2083
+ return {"processed": 0, "complete": complete}
2084
+ row = conn.execute(
2085
+ "SELECT value FROM cache_meta "
2086
+ "WHERE key='codex_find_projection_backfill_cursor'"
2087
+ ).fetchone()
2088
+ try:
2089
+ cursor = int(row[0]) if row is not None else 0
2090
+ except (TypeError, ValueError):
2091
+ cursor = 0
2092
+ selected_rows = conn.execute(
2093
+ "SELECT id,conversation_key FROM codex_conversation_messages "
2094
+ "WHERE id>? ORDER BY id LIMIT ?",
2095
+ (cursor, max(1, batch_size)),
2096
+ ).fetchall()
2097
+ selected = {conversation_key for _message_id, conversation_key in selected_rows}
2098
+ if selected_rows:
2099
+ import _lib_codex_conversation_query as query
2100
+ query.materialize_codex_find_projection(conn, selected)
2101
+ # Advance only over raw rows actually consumed by this batch. A
2102
+ # conversation may own later, interleaved row ids; jumping to its MAX
2103
+ # would skip another conversation whose first row lies in between and
2104
+ # could falsely certify the projection complete.
2105
+ cursor = int(selected_rows[-1][0])
2106
+ conn.execute(
2107
+ "INSERT OR REPLACE INTO cache_meta(key,value) VALUES"
2108
+ "('codex_find_projection_backfill_cursor',?)",
2109
+ (str(cursor),),
2110
+ )
2111
+ remaining = conn.execute(
2112
+ "SELECT 1 FROM codex_conversation_messages WHERE id>? LIMIT 1",
2113
+ (cursor,),
2114
+ ).fetchone()
2115
+ complete = remaining is None
2116
+ if complete:
2117
+ conn.execute(
2118
+ "INSERT OR REPLACE INTO cache_meta(key,value) VALUES(?,?)",
2119
+ (
2120
+ "codex_find_projection_complete_version",
2121
+ str(CODEX_FIND_PROJECTION_VERSION),
2122
+ ),
2123
+ )
2124
+ conn.execute(
2125
+ "DELETE FROM cache_meta WHERE key IN "
2126
+ "('codex_find_projection_backfill_pending',"
2127
+ " 'codex_find_projection_backfill_cursor',"
2128
+ " 'codex_find_projection_backfill_version')"
2129
+ )
2130
+ conn.commit()
2131
+ return {"processed": len(selected), "complete": complete}
1787
2132
 
1788
2133
 
1789
2134
  def _repair_codex_turn_ids_for_source(
@@ -1832,27 +2177,18 @@ def _repair_codex_turn_ids_for_source(
1832
2177
  return affected
1833
2178
 
1834
2179
 
1835
- def _collect_inactive_codex_paths_and_roots(
2180
+ def _collect_retained_codex_paths_and_roots(
1836
2181
  conn: sqlite3.Connection,
1837
- current_file_identities: set[tuple[str, str]],
1838
- active_root_keys: set[str],
1839
2182
  ) -> tuple[list[tuple[str, str | None]], set[str]]:
1840
- """Return stale real source identities and their candidate root keys.
2183
+ """Return every retained real source identity and provider root.
1841
2184
 
1842
2185
  A failed/partial prior write can leave any S1 child family without its
1843
- terminal ``codex_session_files`` row. Scope pruning must therefore use
1844
- every physical family, compare each row's path AND provider root, and leave
1845
- relative fixture rows alone.
2186
+ terminal ``codex_session_files`` row. Safety decisions must therefore see
2187
+ every physical family, including a still-discovered path whose root no
2188
+ longer contains recognizable Codex data. Relative fixture rows have no
2189
+ on-disk authority and remain outside filesystem pruning.
1846
2190
  """
1847
- stale_identities: set[tuple[str, str | None]] = set()
1848
- stale_root_keys: set[str] = set()
1849
- current_paths = {path for path, _root_key in current_file_identities}
1850
- terminal_file_identities = {
1851
- (path, root_key)
1852
- for path, root_key in conn.execute(
1853
- "SELECT path, source_root_key FROM codex_session_files"
1854
- )
1855
- }
2191
+ retained_identities: set[tuple[str, str | None]] = set()
1856
2192
  family_queries = (
1857
2193
  "SELECT path, source_root_key FROM codex_session_files",
1858
2194
  "SELECT source_path, source_root_key FROM codex_session_entries",
@@ -1862,29 +2198,59 @@ def _collect_inactive_codex_paths_and_roots(
1862
2198
  )
1863
2199
  for query in family_queries:
1864
2200
  for source_path, root_key in conn.execute(query):
1865
- identity = (source_path, root_key)
1866
- if (
1867
- not os.path.isabs(source_path)
1868
- or identity in current_file_identities
1869
- # An old terminal file at a currently discovered path must
1870
- # reach the normal requalification loop, which resets every
1871
- # family as one file transaction and records the reset stat.
1872
- or (
1873
- source_path in current_paths
1874
- and identity in terminal_file_identities
1875
- )
1876
- ):
2201
+ if not os.path.isabs(source_path):
1877
2202
  continue
1878
- stale_identities.add(identity)
1879
- if root_key is not None:
1880
- stale_root_keys.add(root_key)
1881
- stale_root_keys.update(
2203
+ retained_identities.add((source_path, root_key))
2204
+ retained_root_keys = {
2205
+ root_key for _path, root_key in retained_identities if root_key is not None
2206
+ }
2207
+ retained_root_keys.update(
1882
2208
  root_key
1883
2209
  for (root_key,) in conn.execute(
1884
2210
  "SELECT source_root_key FROM codex_source_roots"
1885
2211
  )
1886
- if root_key not in active_root_keys
1887
2212
  )
2213
+ return (
2214
+ sorted(retained_identities, key=lambda item: (item[0], item[1] or "")),
2215
+ retained_root_keys,
2216
+ )
2217
+
2218
+
2219
+ def _collect_inactive_codex_paths_and_roots(
2220
+ conn: sqlite3.Connection,
2221
+ current_file_identities: set[tuple[str, str]],
2222
+ active_root_keys: set[str],
2223
+ ) -> tuple[list[tuple[str, str | None]], set[str]]:
2224
+ """Return stale retained source identities and candidate root keys."""
2225
+ retained_identities, retained_root_keys = (
2226
+ _collect_retained_codex_paths_and_roots(conn)
2227
+ )
2228
+ current_paths = {path for path, _root_key in current_file_identities}
2229
+ terminal_file_identities = {
2230
+ (path, root_key)
2231
+ for path, root_key in conn.execute(
2232
+ "SELECT path, source_root_key FROM codex_session_files"
2233
+ )
2234
+ }
2235
+ stale_identities = {
2236
+ identity
2237
+ for identity in retained_identities
2238
+ if (
2239
+ identity not in current_file_identities
2240
+ # An old terminal file at a currently discovered path must reach
2241
+ # the normal requalification loop. The caller separately protects
2242
+ # that loop when the configured root is unrecognizable.
2243
+ and not (
2244
+ identity[0] in current_paths
2245
+ and identity in terminal_file_identities
2246
+ )
2247
+ )
2248
+ }
2249
+ stale_root_keys = {
2250
+ root_key
2251
+ for root_key in retained_root_keys
2252
+ if root_key not in active_root_keys
2253
+ }
1888
2254
  return sorted(stale_identities, key=lambda item: (item[0], item[1] or "")), stale_root_keys
1889
2255
 
1890
2256
 
@@ -3793,7 +4159,9 @@ def _reingest_parse_file(jp, path_str):
3793
4159
  return rows
3794
4160
 
3795
4161
 
3796
- def _resumable_reingest_conversation_messages(conn):
4162
+ def _resumable_reingest_conversation_messages(
4163
+ conn, active_account_key: "str | None" = None,
4164
+ ):
3797
4165
  """#179: resumable, lock-friendly replacement for the old global
3798
4166
  ``clear_conversation_messages`` + offset-0 ``backfill_conversation_messages``
3799
4167
  reingest, which re-armed the whole ~2.5min rebuild on any interrupt. Walks
@@ -3841,6 +4209,30 @@ def _resumable_reingest_conversation_messages(conn):
3841
4209
  conn.commit()
3842
4210
  continue
3843
4211
  try:
4212
+ # #347 enrichment replays must not erase first-stamped transcript
4213
+ # ownership. Snapshot by physical offset before the destructive
4214
+ # per-source replace and carry it onto the re-parsed row.
4215
+ account_capable = "account_key" in {
4216
+ str(info[1])
4217
+ for info in conn.execute("PRAGMA table_info(conversation_messages)")
4218
+ }
4219
+ account_by_offset = {
4220
+ int(offset): account_key
4221
+ for offset, account_key in conn.execute(
4222
+ "SELECT byte_offset,account_key FROM conversation_messages "
4223
+ "WHERE source_path=?",
4224
+ (path_str,),
4225
+ )
4226
+ } if account_capable else {}
4227
+ if account_capable:
4228
+ rows = [
4229
+ row + (
4230
+ account_by_offset[int(row[4])]
4231
+ if int(row[4]) in account_by_offset
4232
+ else active_account_key,
4233
+ )
4234
+ for row in rows
4235
+ ]
3844
4236
  # #217 S2 / I-3 (P1-4): conversation_file_touches is derived state
3845
4237
  # keyed by conversation_messages.id, and this per-source reingest
3846
4238
  # DELETEs + re-inserts the file's message rows (bumping autoincrement
@@ -3855,7 +4247,11 @@ def _resumable_reingest_conversation_messages(conn):
3855
4247
  conn.execute("DELETE FROM conversation_messages WHERE source_path=?",
3856
4248
  (path_str,))
3857
4249
  if rows:
3858
- conn.executemany(_CONV_INSERT_SQL, rows)
4250
+ conn.executemany(
4251
+ _CONV_ACCOUNT_INSERT_SQL if account_capable
4252
+ else _CONV_INSERT_SQL,
4253
+ rows,
4254
+ )
3859
4255
  # Refill scoped to this file's just-reinserted physical keys
3860
4256
  # (col 3=source_path, col 4=byte_offset per _conv_row_tuple).
3861
4257
  _fill_file_touches(
@@ -4860,6 +5256,12 @@ class CodexIngestStats:
4860
5256
  # Count of cached files dropped because they fall outside the CURRENT
4861
5257
  # $CODEX_HOME root set (issue #108 — a prior-root purge, not a delta).
4862
5258
  files_pruned: int = 0
5259
+ # #485: a whole-tree walk may discover no trustworthy current Codex root.
5260
+ # In that state absence is not deletion evidence, so ordinary orphan
5261
+ # pruning is refused and retained rows stay untouched. These fields make
5262
+ # the destructive decision directly observable to callers and tests.
5263
+ prune_refused: bool = False
5264
+ prune_refused_files: int = 0
4863
5265
  lock_contended: bool = False
4864
5266
  # #279 S2 F1 parse-health counters — folded from each file's
4865
5267
  # _CodexIterState after its drain. Same vocabulary as the iterator
@@ -5456,6 +5858,7 @@ def sync_codex_cache(
5456
5858
  _cache_root_keys,
5457
5859
  )
5458
5860
  seq_before = codex_physical_mutation_seq(conn)
5861
+ rebuild_prune_scope: _CodexPruneScope | None = None
5459
5862
 
5460
5863
  # #416 spec D1: which rollouts were ALREADY ingested before this rebuild
5461
5864
  # cleared the cursor table. A rebuild re-reads their bytes, and bytes
@@ -5485,6 +5888,46 @@ def sync_codex_cache(
5485
5888
  str(_canonical_codex_path(pathlib.Path(str(_path))))))
5486
5889
  except (ValueError, TypeError, OSError):
5487
5890
  continue
5891
+ if rebuild and not targeted:
5892
+ # #485: rebuild used to clear first and discover second. With an
5893
+ # empty/missing/misconfigured root that made the explicit repair
5894
+ # command itself destroy the only retained copy. Discover before
5895
+ # the clear and require the same positive root evidence as ordinary
5896
+ # pruning whenever real absolute Codex rows already exist.
5897
+ preflight_files = _discover_codex_files_with_roots()
5898
+ preflight_scope = _codex_prune_scope(preflight_files)
5899
+ rebuild_prune_scope = preflight_scope
5900
+ preflight_sources, _preflight_root_keys = (
5901
+ _collect_retained_codex_paths_and_roots(conn)
5902
+ )
5903
+ preflight_source_root_keys = {
5904
+ root_key
5905
+ for _path, root_key in preflight_sources
5906
+ if root_key is not None
5907
+ }
5908
+ (
5909
+ _safe_sources,
5910
+ refused_sources,
5911
+ _safe_roots,
5912
+ refused_root_keys,
5913
+ ) = _partition_codex_prune_candidates(
5914
+ preflight_scope, preflight_sources, preflight_source_root_keys
5915
+ )
5916
+ if refused_sources or refused_root_keys:
5917
+ _codex_prune_refusal_record(
5918
+ conn,
5919
+ store="cache",
5920
+ scope=preflight_scope,
5921
+ refused_sources=refused_sources,
5922
+ refused_root_keys=refused_root_keys,
5923
+ )
5924
+ conn.commit()
5925
+ stats.files_total = len(preflight_files)
5926
+ stats.prune_refused = True
5927
+ stats.prune_refused_files = len({
5928
+ path for path, _root in refused_sources
5929
+ })
5930
+ return stats
5488
5931
  if rebuild:
5489
5932
  # Clear INSIDE the lock — see sync_cache() for the full
5490
5933
  # rationale. Done before the existing SELECT so delta
@@ -5590,19 +6033,13 @@ def sync_codex_cache(
5590
6033
  stats.files_total = len(files)
5591
6034
  _p_disc.set_count(len(files))
5592
6035
 
5593
- # Scope the cache to the CURRENT root set: drop rows ingested under a
5594
- # prior $CODEX_HOME (issue #108). iter_codex_entries() has NO root
5595
- # predicate — it reads every row in range — so without this, reusing
5596
- # the same cache.db across `CODEX_HOME=/A` then `CODEX_HOME=/B` runs
5597
- # returns A+B instead of just B. Prune every real (absolute) row
5598
- # outside the current set, even when that set is empty (an empty
5599
- # current root then prunes the cache to empty): the cache is fully
5600
- # re-derivable, so honoring the override beats retaining unreachable
5601
- # rows. Done INSIDE the lock and committed BEFORE the existing-SELECT
5602
- # + parse loop so no cache.db write lock is held across the read-heavy
5603
- # ingest (same invariant as the --rebuild clear above). Concurrent
5604
- # processes with different $CODEX_HOME would prune each other; the
5605
- # flock serializes them and that is a pathological configuration.
6036
+ # Scope the cache to the CURRENT root set (issue #108), but only from
6037
+ # positive filesystem evidence (#485). A recognized new root still
6038
+ # authorizes the deliberate A -> B scope switch. No recognized root,
6039
+ # or one configured-but-unrecognizable sibling, is unknown filesystem
6040
+ # state rather than proof of deletion and therefore fails closed.
6041
+ protected_paths: set[str] = set()
6042
+ ordinary_prune_scope: _CodexPruneScope | None = None
5606
6043
  if not rebuild and not targeted: # --rebuild already cleared; targeted bypasses
5607
6044
  current_file_identities = {
5608
6045
  (str(item.source_path), item.source_root_key) for item in files
@@ -5619,12 +6056,39 @@ def sync_codex_cache(
5619
6056
  orphan_sources, orphan_root_keys = _collect_inactive_codex_paths_and_roots(
5620
6057
  conn, current_file_identities, active_root_keys,
5621
6058
  )
5622
- if orphan_sources or orphan_root_keys:
6059
+ prune_scope = _codex_prune_scope(files)
6060
+ ordinary_prune_scope = prune_scope
6061
+ (
6062
+ safe_sources,
6063
+ _refused_orphan_sources,
6064
+ safe_root_keys,
6065
+ _refused_orphan_root_keys,
6066
+ ) = _partition_codex_prune_candidates(
6067
+ prune_scope, orphan_sources, orphan_root_keys
6068
+ )
6069
+ retained_sources, _retained_root_keys = (
6070
+ _collect_retained_codex_paths_and_roots(conn)
6071
+ )
6072
+ retained_source_root_keys = {
6073
+ root_key
6074
+ for _path, root_key in retained_sources
6075
+ if root_key is not None
6076
+ }
6077
+ (
6078
+ _safe_retained_sources,
6079
+ refused_sources,
6080
+ _safe_retained_roots,
6081
+ refused_root_keys,
6082
+ ) = _partition_codex_prune_candidates(
6083
+ prune_scope, retained_sources, retained_source_root_keys
6084
+ )
6085
+ protected_paths = {path for path, _root in refused_sources}
6086
+ if safe_sources or safe_root_keys:
5623
6087
  before_prune = conn.total_changes
5624
6088
  # #294 S6: capture the conversation keys the orphan rows belong to
5625
6089
  # BEFORE deleting, so the rollups can be repaired/deleted after.
5626
6090
  orphan_keys: set = set()
5627
- for orphan_path, orphan_root_key in orphan_sources:
6091
+ for orphan_path, orphan_root_key in safe_sources:
5628
6092
  orphan_keys.update(
5629
6093
  row[0] for row in conn.execute(
5630
6094
  "SELECT DISTINCT conversation_key FROM codex_conversation_messages "
@@ -5637,7 +6101,7 @@ def sync_codex_cache(
5637
6101
  )
5638
6102
  _prune_inactive_codex_source_roots(
5639
6103
  conn, active_root_keys,
5640
- candidate_root_keys=orphan_root_keys,
6104
+ candidate_root_keys=safe_root_keys,
5641
6105
  )
5642
6106
  # Recompute-affected-or-delete the rollups the prune touched (§3.2):
5643
6107
  # a conversation with no surviving rows loses its rollup, one that
@@ -5645,8 +6109,24 @@ def sync_codex_cache(
5645
6109
  _recompute_codex_rollups(conn, orphan_keys)
5646
6110
  if conn.total_changes != before_prune:
5647
6111
  _bump_codex_physical_mutation_seq(conn)
6112
+ stats.files_pruned = len({path for path, _root in safe_sources})
6113
+ if refused_sources or refused_root_keys:
6114
+ _codex_prune_refusal_record(
6115
+ conn,
6116
+ store="cache",
6117
+ scope=prune_scope,
6118
+ refused_sources=refused_sources,
6119
+ refused_root_keys=refused_root_keys,
6120
+ )
6121
+ stats.prune_refused = True
6122
+ stats.prune_refused_files = len({
6123
+ path for path, _root in refused_sources
6124
+ })
6125
+ if (
6126
+ safe_sources or safe_root_keys
6127
+ or refused_sources or refused_root_keys
6128
+ ):
5648
6129
  conn.commit()
5649
- stats.files_pruned = len({path for path, _root in orphan_sources})
5650
6130
 
5651
6131
  # This SELECT does NOT open an implicit transaction (Python's
5652
6132
  # sqlite3 module only BEGINs on DML). Do NOT add any INSERT/
@@ -5808,6 +6288,12 @@ def sync_codex_cache(
5808
6288
  break
5809
6289
  jp = discovered.source_path
5810
6290
  path_str = str(jp)
6291
+ if path_str in protected_paths:
6292
+ # The file is still visible, but its configured root did not
6293
+ # yield recognizable Codex data. Treat that as unknown
6294
+ # filesystem state: neither truncation/requalification reset
6295
+ # nor append ingestion may replace the retained generation.
6296
+ continue
5811
6297
  try:
5812
6298
  st = jp.stat()
5813
6299
  except OSError as exc:
@@ -6543,6 +7029,17 @@ def sync_codex_cache(
6543
7029
  and stats.files_failed == 0
6544
7030
  and stats.files_deferred_torn == 0
6545
7031
  ):
7032
+ recovery_scope = rebuild_prune_scope or ordinary_prune_scope
7033
+ if (
7034
+ recovery_scope is not None
7035
+ and recovery_scope.recognized_root_keys
7036
+ and not stats.prune_refused
7037
+ and (
7038
+ stats.files_processed + stats.files_skipped_unchanged
7039
+ == stats.files_total
7040
+ )
7041
+ ):
7042
+ _clear_codex_prune_refusal(conn)
6546
7043
  if replay_pending:
6547
7044
  conn.execute("DELETE FROM cache_meta WHERE key = ?",
6548
7045
  (CODEX_REPLAY_FROM_ZERO_KEY,))
@@ -8095,6 +8592,160 @@ def open_conversations_db(*, attach_cache: bool = True) -> sqlite3.Connection:
8095
8592
  return _conversations_open_guarded(attach_cache=attach_cache)
8096
8593
 
8097
8594
 
8595
+ def scope_conversations_db_to_account(
8596
+ conn: sqlite3.Connection, account_key: str,
8597
+ ) -> None:
8598
+ """Narrow one conversation read connection to ``account_key`` (#347).
8599
+
8600
+ The query kernels intentionally retain their byte-frozen unqualified SQL.
8601
+ A qualified request gets a fresh SQLite connection and this helper shadows
8602
+ every transcript/accounting leaf with TEMP views on that connection only.
8603
+ SQLite resolves TEMP before main/attached schemas, so browse, facets,
8604
+ search, detail, outline, export, payload, media, find, and cost/token joins
8605
+ all inherit the same physical-row predicate without a parallel query stack.
8606
+
8607
+ The two rollups are rebuilt into TEMP tables from the filtered leaves. No
8608
+ persistent row is changed and an unqualified connection takes none of this
8609
+ path, preserving the existing SQL plans and serialized bytes.
8610
+ """
8611
+ import _lib_accounts
8612
+
8613
+ key = str(account_key or "").strip()
8614
+ if not key:
8615
+ raise ValueError("account_key is required")
8616
+ if key == _lib_accounts.UNATTRIBUTED:
8617
+ stored_predicate = "COALESCE(account_key,'unattributed')='unattributed'"
8618
+ else:
8619
+ # The value lives in a one-row TEMP table and reaches every view through
8620
+ # a subquery; never interpolate an opaque account key into DDL.
8621
+ stored_predicate = (
8622
+ "COALESCE(account_key,'unattributed')="
8623
+ "(SELECT account_key FROM _conversation_account_scope)"
8624
+ )
8625
+
8626
+ conn.execute(
8627
+ "CREATE TEMP TABLE _conversation_account_scope "
8628
+ "(account_key TEXT NOT NULL PRIMARY KEY)"
8629
+ )
8630
+ conn.execute(
8631
+ "INSERT INTO _conversation_account_scope(account_key) VALUES(?)", (key,)
8632
+ )
8633
+
8634
+ conn.executescript(
8635
+ f"""
8636
+ CREATE TEMP VIEW conversation_messages AS
8637
+ SELECT * FROM main.conversation_messages WHERE {stored_predicate};
8638
+ CREATE TEMP VIEW conversation_ai_titles AS
8639
+ SELECT t.* FROM main.conversation_ai_titles t
8640
+ WHERE EXISTS (
8641
+ SELECT 1 FROM conversation_messages m
8642
+ WHERE m.source_path=t.source_path
8643
+ AND m.byte_offset=t.byte_offset
8644
+ );
8645
+ CREATE TEMP VIEW conversation_file_touches AS
8646
+ SELECT t.* FROM main.conversation_file_touches t
8647
+ WHERE EXISTS (
8648
+ SELECT 1 FROM conversation_messages m WHERE m.id=t.message_id
8649
+ );
8650
+ CREATE TEMP VIEW session_entries AS
8651
+ SELECT * FROM cache_db.session_entries WHERE {stored_predicate};
8652
+ -- A file-level project path cannot be partitioned when one JSONL
8653
+ -- switches accounts. Scoped anonymization therefore uses only the
8654
+ -- already-filtered physical message CWDs.
8655
+ CREATE TEMP VIEW session_files AS
8656
+ SELECT * FROM cache_db.session_files WHERE 0;
8657
+
8658
+ CREATE TEMP VIEW codex_conversation_events AS
8659
+ SELECT * FROM main.codex_conversation_events WHERE {stored_predicate};
8660
+ CREATE TEMP VIEW codex_conversation_messages AS
8661
+ SELECT * FROM main.codex_conversation_messages WHERE {stored_predicate};
8662
+ CREATE TEMP VIEW codex_conversation_file_touches AS
8663
+ SELECT t.* FROM main.codex_conversation_file_touches t
8664
+ WHERE EXISTS (
8665
+ SELECT 1 FROM codex_conversation_messages m WHERE m.id=t.message_id
8666
+ );
8667
+ CREATE TEMP VIEW codex_find_projection AS
8668
+ SELECT p.* FROM main.codex_find_projection p
8669
+ WHERE EXISTS (
8670
+ SELECT 1 FROM codex_conversation_messages m WHERE m.id=p.message_id
8671
+ );
8672
+ CREATE TEMP VIEW codex_session_entries AS
8673
+ SELECT * FROM cache_db.codex_session_entries WHERE {stored_predicate};
8674
+
8675
+ -- Thread metadata has conversation-level provenance and can predate an
8676
+ -- in-file account switch. Do not expose it in a scoped connection;
8677
+ -- the Codex rollup degrades explicitly to unassigned rather than leaking
8678
+ -- another account's cwd/git metadata.
8679
+ CREATE TEMP VIEW codex_conversation_threads AS
8680
+ SELECT * FROM cache_db.codex_conversation_threads WHERE 0;
8681
+ CREATE TEMP VIEW codex_source_roots AS
8682
+ SELECT * FROM cache_db.codex_source_roots WHERE 0;
8683
+
8684
+ CREATE TEMP TABLE conversation_sessions AS
8685
+ SELECT * FROM main.conversation_sessions WHERE 0;
8686
+ CREATE TEMP TABLE codex_conversation_rollups (
8687
+ conversation_key TEXT NOT NULL PRIMARY KEY,
8688
+ source_root_key TEXT NOT NULL,
8689
+ parent_thread_id TEXT,
8690
+ item_count INTEGER NOT NULL DEFAULT 0,
8691
+ started_utc TEXT,
8692
+ last_activity_utc TEXT,
8693
+ project_key TEXT,
8694
+ project_label TEXT,
8695
+ models_json TEXT,
8696
+ title TEXT
8697
+ );
8698
+ """
8699
+ )
8700
+ _recompute_conversation_sessions(conn)
8701
+ codex_keys = {
8702
+ row[0]
8703
+ for row in conn.execute(
8704
+ "SELECT DISTINCT conversation_key FROM codex_conversation_messages"
8705
+ )
8706
+ if row[0]
8707
+ }
8708
+ safe_project_attribution: dict[str, tuple[str | None, str | None]] = {}
8709
+ for conversation_key in codex_keys:
8710
+ persisted = conn.execute(
8711
+ "SELECT project_key,project_label "
8712
+ "FROM main.codex_conversation_rollups WHERE conversation_key=?",
8713
+ (conversation_key,),
8714
+ ).fetchone()
8715
+ if persisted is not None:
8716
+ safe_project_attribution[conversation_key] = persisted
8717
+ continue
8718
+ thread = conn.execute(
8719
+ "SELECT source_root_key,cwd,git_json "
8720
+ "FROM cache_db.codex_conversation_threads WHERE conversation_key=?",
8721
+ (conversation_key,),
8722
+ ).fetchone()
8723
+ if thread is not None:
8724
+ safe_project_attribution[conversation_key] = (
8725
+ _codex_conversation_project_attribution(*thread)
8726
+ )
8727
+ _recompute_codex_rollups(conn, codex_keys)
8728
+ # Project identity is safe conversation-level enrichment: it is already
8729
+ # visible on the unqualified rail and contains only an opaque key plus the
8730
+ # derived display label. Preserve those two fields for conversations that
8731
+ # survived the physical-row account predicate, without exposing the
8732
+ # conversation-level cwd/git, source-root path, title, or thread topology
8733
+ # that produced them (#497).
8734
+ conn.executemany(
8735
+ "UPDATE codex_conversation_rollups SET project_key=?,project_label=? "
8736
+ "WHERE conversation_key=?",
8737
+ [
8738
+ (project_key, project_label, conversation_key)
8739
+ for conversation_key, (project_key, project_label)
8740
+ in safe_project_attribution.items()
8741
+ ],
8742
+ )
8743
+ # TEMP rollup writes open a transaction. Close it before a long-lived
8744
+ # account-scoped SSE reader starts watching so later provider-writer commits
8745
+ # are visible to the dynamic leaf views and never contend on this setup work.
8746
+ conn.commit()
8747
+
8748
+
8098
8749
  def _open_conversations_db_for_recovery(
8099
8750
  *, attach_cache: bool = True,
8100
8751
  ) -> sqlite3.Connection:
@@ -8669,6 +9320,7 @@ def _prepare_claude_conversation_maintenance(
8669
9320
  *,
8670
9321
  rebuild: bool,
8671
9322
  targeted: bool,
9323
+ active_account_key: "str | None" = None,
8672
9324
  ) -> bool:
8673
9325
  """Consume transcript-only upgrade work under the conversation flock.
8674
9326
 
@@ -8760,7 +9412,9 @@ def _prepare_claude_conversation_maintenance(
8760
9412
  "'conversation_background_mcp_reingest_pending')"
8761
9413
  ).fetchone() is not None
8762
9414
  if reingest:
8763
- _resumable_reingest_conversation_messages(conn)
9415
+ _resumable_reingest_conversation_messages(
9416
+ conn, active_account_key=active_account_key,
9417
+ )
8764
9418
  _set_cache_meta(conn, "conversation_sessions_backfill_pending", "1")
8765
9419
  conn.commit()
8766
9420
 
@@ -8806,6 +9460,20 @@ def sync_claude_conversations(
8806
9460
  stats.lock_contended = True
8807
9461
  return stats
8808
9462
 
9463
+ # #347 observe-and-stamp mirrors the accounting ingest boundary. A
9464
+ # torn credential read is undecided, not unattributed: defer before any
9465
+ # maintenance/rebuild mutation so the cursor remains replayable.
9466
+ import _lib_accounts
9467
+ claude_identity = _cctally_core._resolve_active_claude_identity()
9468
+ if claude_identity.get("status") == "torn":
9469
+ stats.files_deferred_torn += 1
9470
+ stats.deferred_reason = "identity_torn"
9471
+ return stats
9472
+ active_key = claude_identity["account_key"]
9473
+ active_account_key = (
9474
+ None if active_key == _lib_accounts.UNATTRIBUTED else active_key
9475
+ )
9476
+
8809
9477
  targeted = only_paths is not None
8810
9478
  pending_rebuild = conn.execute(
8811
9479
  "SELECT 1 FROM cache_meta "
@@ -8835,10 +9503,19 @@ def sync_claude_conversations(
8835
9503
 
8836
9504
  _report_conversation_progress(progress, "prepare", stats)
8837
9505
  maintenance_replayed = _prepare_claude_conversation_maintenance(
8838
- conn, rebuild=rebuild, targeted=targeted
9506
+ conn, rebuild=rebuild, targeted=targeted,
9507
+ active_account_key=active_account_key,
8839
9508
  )
8840
9509
 
9510
+ rebuild_account_stamps: dict[tuple[str, int], str | None] = {}
8841
9511
  if rebuild:
9512
+ rebuild_account_stamps = {
9513
+ (str(path), int(offset)): account_key
9514
+ for path, offset, account_key in conn.execute(
9515
+ "SELECT source_path,byte_offset,account_key "
9516
+ "FROM conversation_messages"
9517
+ )
9518
+ }
8842
9519
  clear_conversation_messages(conn)
8843
9520
  conn.execute("DELETE FROM conversation_ai_titles")
8844
9521
  conn.execute("DELETE FROM conversation_sessions")
@@ -8915,7 +9592,16 @@ def sync_claude_conversations(
8915
9592
  include_cost=False,
8916
9593
  ):
8917
9594
  if mrow is not None:
8918
- conv_rows.append(_conv_row_tuple(mrow, path_str))
9595
+ account_key = rebuild_account_stamps.get(
9596
+ (path_str, int(mrow.byte_offset)),
9597
+ active_account_key,
9598
+ )
9599
+ conv_rows.append(
9600
+ _conv_row_tuple(
9601
+ mrow, path_str, account_key,
9602
+ include_account=True,
9603
+ )
9604
+ )
8919
9605
  if ai is not None:
8920
9606
  ai_rows.append(
8921
9607
  (ai.session_id, ai.ai_title, path_str, ai.byte_offset)
@@ -8952,7 +9638,7 @@ def sync_claude_conversations(
8952
9638
  )
8953
9639
  stats.files_reset_truncated += 1
8954
9640
  if conv_rows:
8955
- conn.executemany(_CONV_INSERT_SQL, conv_rows)
9641
+ conn.executemany(_CONV_ACCOUNT_INSERT_SQL, conv_rows)
8956
9642
  _fill_file_touches(
8957
9643
  conn, scope=[(row[3], row[4]) for row in conv_rows]
8958
9644
  )
@@ -9059,6 +9745,7 @@ def sync_codex_conversations(
9059
9745
  """Delta-sync Codex events/search rows into conversations.db (#320)."""
9060
9746
  stats = CodexIngestStats()
9061
9747
  did_from_zero_replay = False
9748
+ rebuild_account_stamps: dict[tuple[str, int], str | None] = {}
9062
9749
  _cctally_core.APP_DIR.mkdir(parents=True, exist_ok=True)
9063
9750
  _cctally_core.CONVERSATIONS_LOCK_CODEX_PATH.touch()
9064
9751
  lock_fh = open(_cctally_core.CONVERSATIONS_LOCK_CODEX_PATH, "w")
@@ -9106,7 +9793,51 @@ def sync_codex_conversations(
9106
9793
  return stats
9107
9794
  rebuild = (
9108
9795
  rebuild or pending_rebuild or contract_rebuild or codex_replay_pending)
9796
+ rebuild_prune_scope: _CodexPruneScope | None = None
9109
9797
  if rebuild:
9798
+ rebuild_account_stamps = {
9799
+ (str(path), int(offset)): account_key
9800
+ for path, offset, account_key in conn.execute(
9801
+ "SELECT source_path,line_offset,account_key "
9802
+ "FROM codex_conversation_events"
9803
+ )
9804
+ }
9805
+ # #485: establish positive root evidence before the destructive
9806
+ # clear. A rebuild against an empty or missing CODEX_HOME must keep
9807
+ # the retained transcript store and its pending replay marker.
9808
+ preflight_files = _discover_codex_files_with_roots()
9809
+ preflight_scope = _codex_prune_scope(preflight_files)
9810
+ rebuild_prune_scope = preflight_scope
9811
+ preflight_sources = [
9812
+ (path, root_key)
9813
+ for path, root_key in conn.execute(
9814
+ "SELECT path,source_root_key "
9815
+ "FROM codex_conversation_source_files"
9816
+ )
9817
+ ]
9818
+ (
9819
+ _safe_sources,
9820
+ refused_sources,
9821
+ _safe_roots,
9822
+ _refused_roots,
9823
+ ) = _partition_codex_prune_candidates(
9824
+ preflight_scope, preflight_sources, set()
9825
+ )
9826
+ if refused_sources:
9827
+ _codex_prune_refusal_record(
9828
+ conn,
9829
+ store="conversations",
9830
+ scope=preflight_scope,
9831
+ refused_sources=refused_sources,
9832
+ refused_root_keys=set(),
9833
+ )
9834
+ conn.commit()
9835
+ stats.files_total = len(preflight_files)
9836
+ stats.prune_refused = True
9837
+ stats.prune_refused_files = len({
9838
+ path for path, _root in refused_sources
9839
+ })
9840
+ return stats
9110
9841
  conn.execute(
9111
9842
  "INSERT OR REPLACE INTO cache_meta(key,value) VALUES(?,?)",
9112
9843
  ("conversation_rebuild_codex_pending", "1"),
@@ -9152,9 +9883,36 @@ def sync_codex_conversations(
9152
9883
  if current_size < prev[0]:
9153
9884
  stats.deferred_reason = "truncation"
9154
9885
  return stats
9886
+ protected_paths: set[str] = set()
9887
+ ordinary_prune_scope: _CodexPruneScope | None = None
9155
9888
  if only_paths is None:
9156
9889
  active_paths = {str(item.source_path) for item in files}
9157
- for stale_path in sorted(set(existing) - active_paths):
9890
+ prune_scope = _codex_prune_scope(files)
9891
+ ordinary_prune_scope = prune_scope
9892
+ stale_sources = [
9893
+ (stale_path, existing[stale_path][3])
9894
+ for stale_path in sorted(set(existing) - active_paths)
9895
+ ]
9896
+ safe_sources, _refused_stale, _safe_roots, _refused_roots = (
9897
+ _partition_codex_prune_candidates(
9898
+ prune_scope, stale_sources, set()
9899
+ )
9900
+ )
9901
+ retained_sources = [
9902
+ (path, values[3])
9903
+ for path, values in existing.items()
9904
+ if os.path.isabs(path)
9905
+ ]
9906
+ (
9907
+ _safe_retained,
9908
+ refused_sources,
9909
+ _safe_retained_roots,
9910
+ _refused_retained_roots,
9911
+ ) = _partition_codex_prune_candidates(
9912
+ prune_scope, retained_sources, set()
9913
+ )
9914
+ protected_paths = {path for path, _root in refused_sources}
9915
+ for stale_path, _stale_root_key in safe_sources:
9158
9916
  affected = {
9159
9917
  row[0] for row in conn.execute(
9160
9918
  "SELECT DISTINCT conversation_key "
@@ -9179,11 +9937,30 @@ def sync_codex_conversations(
9179
9937
  (stale_path,),
9180
9938
  )
9181
9939
  _recompute_codex_rollups(conn, affected)
9940
+ import _lib_codex_conversation_query as query
9941
+ query.materialize_codex_find_projection(conn, affected)
9942
+ stats.files_pruned = len({path for path, _root in safe_sources})
9943
+ if refused_sources:
9944
+ _codex_prune_refusal_record(
9945
+ conn,
9946
+ store="conversations",
9947
+ scope=prune_scope,
9948
+ refused_sources=refused_sources,
9949
+ refused_root_keys=set(),
9950
+ )
9951
+ stats.prune_refused = True
9952
+ stats.prune_refused_files = len({
9953
+ path for path, _root in refused_sources
9954
+ })
9182
9955
  conn.commit()
9183
9956
 
9184
9957
  for discovered in files:
9185
9958
  jp = discovered.source_path
9186
9959
  path_str = str(jp)
9960
+ if path_str in protected_paths:
9961
+ # Visible but unrecognizable input is not evidence that the
9962
+ # retained transcript generation should be reset or replaced.
9963
+ continue
9187
9964
  try:
9188
9965
  st = jp.stat()
9189
9966
  except OSError:
@@ -9219,6 +9996,31 @@ def sync_codex_conversations(
9219
9996
  initial_conversation = prev[10] if prev else None
9220
9997
  initial_turn = prev[11] if prev else None
9221
9998
 
9999
+ # #347 uses the accounting cache's journal-backed file-range map as
10000
+ # the Codex transcript oracle. It is already attached read-only,
10001
+ # and core ingest runs first, so rebuilds replay the original
10002
+ # decision instead of consulting the currently-active auth.json.
10003
+ try:
10004
+ file_identity = codex_file_identity(discovered)
10005
+ incarnation_row = conn.execute(
10006
+ "SELECT incarnation FROM cache_db.codex_file_incarnations "
10007
+ "WHERE file_identity=?",
10008
+ (file_identity,),
10009
+ ).fetchone()
10010
+ incarnation = int(incarnation_row[0]) if incarnation_row else 1
10011
+ account_ranges = [
10012
+ (int(off), key)
10013
+ for off, key in conn.execute(
10014
+ "SELECT from_offset,account_key "
10015
+ "FROM cache_db.codex_file_accounts "
10016
+ "WHERE file_identity=? AND incarnation=? "
10017
+ "ORDER BY from_offset",
10018
+ (file_identity, incarnation),
10019
+ )
10020
+ ]
10021
+ except sqlite3.OperationalError:
10022
+ account_ranges = []
10023
+
9222
10024
  state = _CodexIterState(
9223
10025
  session_id=initial_session_id,
9224
10026
  model=initial_model,
@@ -9241,6 +10043,7 @@ def sync_codex_conversations(
9241
10043
  )
9242
10044
  events = []
9243
10045
  event_rows = []
10046
+ account_by_physical: dict[tuple[str, int], str | None] = {}
9244
10047
  yielded = 0
9245
10048
  try:
9246
10049
  with open(jp, "rb") as fh:
@@ -9256,6 +10059,16 @@ def sync_codex_conversations(
9256
10059
  ):
9257
10060
  event = emission.event
9258
10061
  events.append(event)
10062
+ covered, account_key = codex_account_for_offset(
10063
+ account_ranges, int(event.line_offset)
10064
+ )
10065
+ if not covered:
10066
+ account_key = rebuild_account_stamps.get(
10067
+ (event.source_path, int(event.line_offset))
10068
+ )
10069
+ account_by_physical[
10070
+ (event.source_path, int(event.line_offset))
10071
+ ] = account_key
9259
10072
  event_rows.append((
9260
10073
  event.source_path,
9261
10074
  event.line_offset,
@@ -9270,6 +10083,7 @@ def sync_codex_conversations(
9270
10083
  event.turn_id,
9271
10084
  event.call_id,
9272
10085
  event.payload_json,
10086
+ account_key,
9273
10087
  ))
9274
10088
  if emission.accounting is not None:
9275
10089
  yielded += 1
@@ -9327,12 +10141,13 @@ def sync_codex_conversations(
9327
10141
  "INSERT OR IGNORE INTO codex_conversation_events "
9328
10142
  "(source_path,line_offset,source_root_key,conversation_key,"
9329
10143
  "native_thread_id,root_thread_id,parent_thread_id,"
9330
- "timestamp_utc,record_type,event_type,turn_id,call_id,payload_json) "
9331
- "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)",
10144
+ "timestamp_utc,record_type,event_type,turn_id,call_id,payload_json,"
10145
+ "account_key) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
9332
10146
  event_rows,
9333
10147
  )
9334
10148
  _insert_codex_normalized_rows(
9335
- conn, normalized.rows, normalized.touches
10149
+ conn, normalized.rows, normalized.touches,
10150
+ account_by_physical,
9336
10151
  )
9337
10152
  affected_keys.update(
9338
10153
  row.conversation_key for row in normalized.rows
@@ -9345,6 +10160,8 @@ def sync_codex_conversations(
9345
10160
  _repair_codex_turn_ids_for_source(conn, path_str)
9346
10161
  )
9347
10162
  _recompute_codex_rollups(conn, affected_keys)
10163
+ import _lib_codex_conversation_query as query
10164
+ query.materialize_codex_find_projection(conn, affected_keys)
9348
10165
  terminal = state.thread
9349
10166
  conn.execute(
9350
10167
  "INSERT INTO codex_conversation_source_files "
@@ -9392,8 +10209,20 @@ def sync_codex_conversations(
9392
10209
  stats.files_failed += 1
9393
10210
  _report_conversation_progress(progress, "ingest", stats)
9394
10211
 
10212
+ run_codex_find_projection_backfill(conn)
9395
10213
  _report_conversation_progress(progress, "finalize", stats)
9396
10214
  if only_paths is None and stats.files_failed == 0:
10215
+ recovery_scope = rebuild_prune_scope or ordinary_prune_scope
10216
+ if (
10217
+ recovery_scope is not None
10218
+ and recovery_scope.recognized_root_keys
10219
+ and not stats.prune_refused
10220
+ and (
10221
+ stats.files_processed + stats.files_skipped_unchanged
10222
+ == stats.files_total
10223
+ )
10224
+ ):
10225
+ _clear_codex_prune_refusal(conn)
9397
10226
  conn.execute(
9398
10227
  "INSERT OR REPLACE INTO cache_meta(key,value) VALUES(?,?)",
9399
10228
  (