cctally 1.90.1 → 1.92.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/CHANGELOG.md +74 -0
  2. package/README.md +2 -2
  3. package/bin/_cctally_cache.py +863 -74
  4. package/bin/_cctally_config.py +57 -0
  5. package/bin/_cctally_core.py +53 -8
  6. package/bin/_cctally_dashboard.py +146 -5
  7. package/bin/_cctally_dashboard_conversation.py +164 -18
  8. package/bin/_cctally_dashboard_envelope.py +69 -12
  9. package/bin/_cctally_dashboard_sources.py +27 -1
  10. package/bin/_cctally_db.py +372 -10
  11. package/bin/_cctally_doctor.py +18 -1
  12. package/bin/_cctally_journal.py +535 -13
  13. package/bin/_cctally_journal_repair.py +6 -0
  14. package/bin/_cctally_parser.py +6 -0
  15. package/bin/_cctally_quota.py +171 -55
  16. package/bin/_cctally_record.py +13 -1
  17. package/bin/_cctally_rederive.py +4 -0
  18. package/bin/_cctally_store.py +311 -6
  19. package/bin/_cctally_transcript.py +32 -2
  20. package/bin/_lib_cache_report.py +8 -3
  21. package/bin/_lib_cache_report_wire.py +8 -20
  22. package/bin/_lib_codex_conversation.py +959 -81
  23. package/bin/_lib_codex_conversation_query.py +2792 -167
  24. package/bin/_lib_codex_find_projection.py +370 -0
  25. package/bin/_lib_codex_harness_preamble.py +176 -0
  26. package/bin/_lib_codex_hooks.py +5 -3
  27. package/bin/_lib_codex_js_scan.py +254 -0
  28. package/bin/_lib_codex_landmarks.py +309 -0
  29. package/bin/_lib_codex_reasoning_headings.py +73 -0
  30. package/bin/_lib_codex_segments.py +259 -0
  31. package/bin/_lib_codex_title_clean.py +116 -0
  32. package/bin/_lib_conversation_dispatch.py +153 -21
  33. package/bin/_lib_conversation_watch.py +4 -2
  34. package/bin/_lib_dashboard_sources.py +33 -32
  35. package/bin/_lib_doctor.py +64 -0
  36. package/bin/_lib_quota_alert_axes.py +31 -34
  37. package/bin/_lib_stats_damage.py +523 -0
  38. package/bin/cctally +5 -0
  39. package/dashboard/static/assets/index-BEzzJtUd.js +97 -0
  40. package/dashboard/static/assets/index-DnWdv8um.css +1 -0
  41. package/dashboard/static/dashboard.html +2 -2
  42. package/package.json +9 -1
  43. package/dashboard/static/assets/index-Bar8-S1i.css +0 -1
  44. package/dashboard/static/assets/index-CRogVlEC.js +0 -92
@@ -227,6 +227,12 @@ _CONV_INSERT_SQL = (
227
227
  " search_tool,search_thinking)"
228
228
  " VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
229
229
  )
230
+ _CONV_ACCOUNT_INSERT_SQL = _CONV_INSERT_SQL.replace(
231
+ "search_tool,search_thinking)", "search_tool,search_thinking,account_key)"
232
+ ).replace(
233
+ "?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
234
+ "?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
235
+ )
230
236
 
231
237
  # #193: last non-null write wins (ai-title carries no timestamp; see spec S1). NO
232
238
  # byte_offset guard — it can't order a cross-file resumed session. Ordering is
@@ -383,7 +389,7 @@ SESSION_ENTRY_UPSERT_SQL_REWALK = (
383
389
  + _SESSION_ENTRY_REWALK_PHYSICAL_GUARD)
384
390
 
385
391
 
386
- def _conv_row_tuple(m, path_str):
392
+ def _conv_row_tuple(m, path_str, account_key=None, *, include_account=False):
387
393
  """Flatten a ``MessageRow`` into the ``_CONV_INSERT_SQL`` column order.
388
394
 
389
395
  The #177 enrichment fields (stop_reason / attribution_skill /
@@ -393,7 +399,7 @@ def _conv_row_tuple(m, path_str):
393
399
  one tuple. #217 S1 / U7a: the documented-dead ``search_aux`` column is gone
394
400
  from the live schema (dropped by migration 016); the split
395
401
  ``search_tool``/``search_thinking`` columns carry the non-prose index."""
396
- return (
402
+ row = (
397
403
  m.session_id, m.uuid, m.parent_uuid, path_str, m.byte_offset,
398
404
  m.timestamp_utc, m.entry_type, m.text, m.blocks_json, m.model,
399
405
  m.msg_id, m.req_id, m.cwd, m.git_branch, m.is_sidechain,
@@ -401,6 +407,7 @@ def _conv_row_tuple(m, path_str):
401
407
  m.stop_reason, m.attribution_skill, m.attribution_plugin,
402
408
  m.search_tool, m.search_thinking,
403
409
  )
410
+ return row + ((account_key,) if include_account else ())
404
411
 
405
412
 
406
413
  def _iter_sync_entries(
@@ -1300,15 +1307,24 @@ def _codex_provider_roots() -> list[CodexProviderRoot]:
1300
1307
  roots: list[CodexProviderRoot] = []
1301
1308
  seen: set[pathlib.Path] = set()
1302
1309
  for configured in _cctally()._codex_home_roots():
1303
- provider_root = _canonical_codex_path(configured)
1310
+ try:
1311
+ provider_root = _canonical_codex_path(configured)
1312
+ except OSError:
1313
+ # An unreadable configured root is unknown filesystem state. Leave
1314
+ # it out of discovery; the prune scope still retains its configured
1315
+ # identity and will refuse destructive cleanup for that root.
1316
+ continue
1304
1317
  if provider_root in seen:
1305
1318
  continue
1306
1319
  sessions = configured / "sessions"
1307
- if sessions.is_dir():
1308
- walk_root = sessions
1309
- elif configured.is_dir():
1310
- walk_root = configured
1311
- else:
1320
+ try:
1321
+ if sessions.is_dir():
1322
+ walk_root = sessions
1323
+ elif configured.is_dir():
1324
+ walk_root = configured
1325
+ else:
1326
+ continue
1327
+ except OSError:
1312
1328
  continue
1313
1329
  seen.add(provider_root)
1314
1330
  roots.append(CodexProviderRoot(
@@ -1438,6 +1454,214 @@ def _discover_codex_files_with_roots() -> list[CodexDiscoveredFile]:
1438
1454
  return discovered
1439
1455
 
1440
1456
 
1457
+ CODEX_ORPHAN_PRUNE_REFUSED_KEY = "codex_orphan_prune_refused"
1458
+ _CODEX_ROOT_RECOGNITION_TYPES = frozenset({
1459
+ "session_meta", "turn_context", "event_msg", "response_item", "world_state",
1460
+ "compacted",
1461
+ })
1462
+ _CODEX_ROOT_RECOGNITION_MAX_FILES = 32
1463
+ _CODEX_ROOT_RECOGNITION_MAX_BYTES = 64 * 1024
1464
+ _CODEX_ROOT_RECOGNITION_MAX_LINES = 64
1465
+
1466
+
1467
+ def _codex_file_is_recognizable(path: pathlib.Path) -> bool:
1468
+ """Boundedly prove that one JSONL contains a known Codex record envelope.
1469
+
1470
+ Unknown records remain ingestible for forward compatibility, but they are
1471
+ not strong enough evidence to authorize deletion of retained history.
1472
+ """
1473
+ consumed = 0
1474
+ lines = 0
1475
+ try:
1476
+ with path.open("rb") as fh:
1477
+ while (
1478
+ lines < _CODEX_ROOT_RECOGNITION_MAX_LINES
1479
+ and consumed < _CODEX_ROOT_RECOGNITION_MAX_BYTES
1480
+ ):
1481
+ remaining = _CODEX_ROOT_RECOGNITION_MAX_BYTES - consumed
1482
+ raw = fh.readline(remaining + 1)
1483
+ if not raw:
1484
+ break
1485
+ lines += 1
1486
+ consumed += len(raw)
1487
+ if len(raw) > remaining or not raw.endswith(b"\n"):
1488
+ break
1489
+ try:
1490
+ obj = json.loads(
1491
+ raw,
1492
+ parse_constant=_lib_jsonl._reject_nonfinite_json_constant,
1493
+ )
1494
+ except (UnicodeDecodeError, ValueError, TypeError):
1495
+ continue
1496
+ if (
1497
+ isinstance(obj, dict)
1498
+ and obj.get("type") in _CODEX_ROOT_RECOGNITION_TYPES
1499
+ and isinstance(obj.get("payload"), dict)
1500
+ ):
1501
+ return True
1502
+ except OSError:
1503
+ return False
1504
+ return False
1505
+
1506
+
1507
+ @dataclass(frozen=True)
1508
+ class _CodexPruneScope:
1509
+ """Filesystem evidence that may authorize one whole-tree orphan prune.
1510
+
1511
+ A configured root is only *recognized* after a bounded probe finds at least
1512
+ one known Codex record envelope under it. A missing, empty, or unrelated
1513
+ directory therefore remains protected. When a current root is recognized,
1514
+ roots no longer present in the configured set may still be removed — that
1515
+ preserves issue #108's deliberate A -> B cache scoping. Within a recognized
1516
+ root, a missing file is deletion evidence only while its parent directory is
1517
+ still readable; losing an entire mounted subtree fails closed.
1518
+ """
1519
+
1520
+ configured_root_keys: frozenset[str]
1521
+ recognized_root_keys: frozenset[str]
1522
+
1523
+ def refusal_reason(self, source_path: str, root_key: str | None) -> str | None:
1524
+ if not self.recognized_root_keys:
1525
+ return "no_recognizable_roots"
1526
+ if (
1527
+ root_key in self.configured_root_keys
1528
+ and root_key not in self.recognized_root_keys
1529
+ ):
1530
+ return "configured_root_unrecognizable"
1531
+ if root_key in self.recognized_root_keys:
1532
+ parent = pathlib.Path(source_path).parent
1533
+ try:
1534
+ if not parent.is_dir() or not os.access(parent, os.R_OK | os.X_OK):
1535
+ return "source_parent_unavailable"
1536
+ except OSError:
1537
+ return "source_parent_unavailable"
1538
+ return None
1539
+
1540
+ def root_refusal_reason(self, root_key: str) -> str | None:
1541
+ if not self.recognized_root_keys:
1542
+ return "no_recognizable_roots"
1543
+ if (
1544
+ root_key in self.configured_root_keys
1545
+ and root_key not in self.recognized_root_keys
1546
+ ):
1547
+ return "configured_root_unrecognizable"
1548
+ return None
1549
+
1550
+
1551
+ def _codex_prune_scope(files: list[CodexDiscoveredFile]) -> _CodexPruneScope:
1552
+ configured: set[str] = set()
1553
+ for raw_root in _cctally()._codex_home_roots():
1554
+ try:
1555
+ provider_root = _canonical_codex_path(raw_root)
1556
+ configured.add(source_root_key(str(provider_root)))
1557
+ except (OSError, TypeError, ValueError):
1558
+ continue
1559
+ recognized: set[str] = set()
1560
+ checked_per_root: dict[str, int] = {}
1561
+ # Discovery is stable oldest-first for Codex's dated rollout paths. Probe
1562
+ # from the tail so a bounded check sees the current envelope generation,
1563
+ # rather than spending its entire budget on a large legacy prefix. A root
1564
+ # containing only legacy/unknown envelopes still fails closed.
1565
+ for item in reversed(files):
1566
+ if item.source_root_key in recognized:
1567
+ continue
1568
+ checked = checked_per_root.get(item.source_root_key, 0)
1569
+ if checked >= _CODEX_ROOT_RECOGNITION_MAX_FILES:
1570
+ continue
1571
+ checked_per_root[item.source_root_key] = checked + 1
1572
+ if _codex_file_is_recognizable(item.source_path):
1573
+ recognized.add(item.source_root_key)
1574
+ return _CodexPruneScope(
1575
+ configured_root_keys=frozenset(configured),
1576
+ recognized_root_keys=frozenset(recognized),
1577
+ )
1578
+
1579
+
1580
+ def _partition_codex_prune_candidates(
1581
+ scope: _CodexPruneScope,
1582
+ sources: list[tuple[str, str | None]],
1583
+ root_keys: set[str],
1584
+ ) -> tuple[
1585
+ list[tuple[str, str | None]], list[tuple[str, str | None]], set[str], set[str]
1586
+ ]:
1587
+ safe_sources: list[tuple[str, str | None]] = []
1588
+ refused_sources: list[tuple[str, str | None]] = []
1589
+ for source in sources:
1590
+ target = refused_sources if scope.refusal_reason(*source) else safe_sources
1591
+ target.append(source)
1592
+ safe_roots: set[str] = set()
1593
+ refused_roots: set[str] = set()
1594
+ for root_key in root_keys:
1595
+ target = refused_roots if scope.root_refusal_reason(root_key) else safe_roots
1596
+ target.add(root_key)
1597
+ return safe_sources, refused_sources, safe_roots, refused_roots
1598
+
1599
+
1600
+ def _codex_prune_refusal_record(
1601
+ conn: sqlite3.Connection,
1602
+ *,
1603
+ store: str,
1604
+ scope: _CodexPruneScope,
1605
+ refused_sources: list[tuple[str, str | None]],
1606
+ refused_root_keys: set[str],
1607
+ ) -> None:
1608
+ """Persist a privacy-safe refusal signal in the store that declined."""
1609
+ now = (
1610
+ _cctally_core._command_as_of()
1611
+ .astimezone(dt.timezone.utc)
1612
+ .isoformat(timespec="seconds")
1613
+ .replace("+00:00", "Z")
1614
+ )
1615
+ since = now
1616
+ row = conn.execute(
1617
+ "SELECT value FROM cache_meta WHERE key=?",
1618
+ (CODEX_ORPHAN_PRUNE_REFUSED_KEY,),
1619
+ ).fetchone()
1620
+ if row and row[0]:
1621
+ try:
1622
+ previous = json.loads(row[0])
1623
+ if isinstance(previous, dict) and isinstance(previous.get("since"), str):
1624
+ since = previous["since"]
1625
+ except (TypeError, ValueError):
1626
+ pass
1627
+ reasons = sorted({
1628
+ reason
1629
+ for path, root_key in refused_sources
1630
+ if (reason := scope.refusal_reason(path, root_key)) is not None
1631
+ } | {
1632
+ reason
1633
+ for root_key in refused_root_keys
1634
+ if (reason := scope.root_refusal_reason(root_key)) is not None
1635
+ })
1636
+ record = {
1637
+ "schemaVersion": 1,
1638
+ "store": store,
1639
+ "since": since,
1640
+ "at": now,
1641
+ "reasons": reasons,
1642
+ "configuredRootCount": len(scope.configured_root_keys),
1643
+ "recognizedRootCount": len(scope.recognized_root_keys),
1644
+ "preservedFileCount": len({path for path, _root in refused_sources}),
1645
+ "preservedRootCount": len(refused_root_keys),
1646
+ }
1647
+ conn.execute(
1648
+ "INSERT INTO cache_meta(key,value) VALUES(?,?) "
1649
+ "ON CONFLICT(key) DO UPDATE SET value=excluded.value",
1650
+ (CODEX_ORPHAN_PRUNE_REFUSED_KEY, json.dumps(record, sort_keys=True)),
1651
+ )
1652
+ eprint(
1653
+ f"[codex-{store}] orphan prune refused: preserved "
1654
+ f"{record['preservedFileCount']} tracked file(s); "
1655
+ "configured Codex roots were not sufficient deletion evidence"
1656
+ )
1657
+
1658
+
1659
+ def _clear_codex_prune_refusal(conn: sqlite3.Connection) -> None:
1660
+ conn.execute(
1661
+ "DELETE FROM cache_meta WHERE key=?", (CODEX_ORPHAN_PRUNE_REFUSED_KEY,)
1662
+ )
1663
+
1664
+
1441
1665
  def _qualify_codex_targets(only_paths: "set[str]") -> list[CodexDiscoveredFile]:
1442
1666
  """Resolve each requested path through the ordered configured roots exactly
1443
1667
  as full discovery would (spec §5.1) — producing the same per-file facts a
@@ -1603,7 +1827,12 @@ def _clear_codex_derived_rows(conn: sqlite3.Connection) -> bool:
1603
1827
 
1604
1828
 
1605
1829
  def _bump_codex_physical_mutation_seq(conn: sqlite3.Connection) -> None:
1606
- """Advance the dashboard's Codex physical-identity sequence in this txn."""
1830
+ """Advance the shared Codex physical-state invalidation token in this txn.
1831
+
1832
+ Dashboard/TUI snapshot versions and the quota projection certificate both
1833
+ consume this value. Every writer that changes their cache.db inputs must
1834
+ advance it in the same transaction as that change.
1835
+ """
1607
1836
  conn.execute(
1608
1837
  "INSERT INTO cache_meta(key, value) VALUES "
1609
1838
  "('codex_physical_mutation_seq', '1') "
@@ -1622,6 +1851,10 @@ _CODEX_MSG_INSERT_SQL = (
1622
1851
  "INSERT OR IGNORE INTO codex_conversation_messages (" + _CODEX_NORM_COLS + ") "
1623
1852
  "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
1624
1853
  )
1854
+ _CODEX_ACCOUNT_MSG_INSERT_SQL = (
1855
+ "INSERT OR IGNORE INTO codex_conversation_messages (" + _CODEX_NORM_COLS
1856
+ + ", account_key) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
1857
+ )
1625
1858
 
1626
1859
 
1627
1860
  def _codex_conversation_project_attribution(
@@ -1679,17 +1912,29 @@ def _load_codex_normalized_rows(
1679
1912
  ]
1680
1913
 
1681
1914
 
1682
- def _insert_codex_normalized_rows(conn: sqlite3.Connection, rows: list, touches: list) -> None:
1915
+ def _insert_codex_normalized_rows(
1916
+ conn: sqlite3.Connection,
1917
+ rows: list,
1918
+ touches: list,
1919
+ account_by_physical: "dict[tuple[str, int], str | None] | None" = None,
1920
+ ) -> None:
1683
1921
  """Insert normalized rows (INSERT OR IGNORE on the physical key) + their file
1684
1922
  touches (message linkage resolved via (source_path, line_offset))."""
1685
1923
  if rows:
1686
- conn.executemany(_CODEX_MSG_INSERT_SQL, [
1924
+ with_accounts = account_by_physical is not None
1925
+ account_by_physical = account_by_physical or {}
1926
+ conn.executemany(
1927
+ _CODEX_ACCOUNT_MSG_INSERT_SQL if with_accounts else _CODEX_MSG_INSERT_SQL,
1928
+ [
1687
1929
  (r.conversation_key, r.source_root_key, r.source_path, r.line_offset,
1688
1930
  r.timestamp_utc, r.turn_id, r.call_id, r.kind, r.event_type,
1689
1931
  r.record_family, r.model, r.text, r.content_digest, r.content_len,
1690
1932
  r.detail_json, r.search_tool, r.search_thinking)
1933
+ + ((account_by_physical.get((r.source_path, r.line_offset)),)
1934
+ if with_accounts else ())
1691
1935
  for r in rows
1692
- ])
1936
+ ],
1937
+ )
1693
1938
  for touch in touches:
1694
1939
  conn.execute(
1695
1940
  "INSERT OR IGNORE INTO codex_conversation_file_touches "
@@ -1764,26 +2009,120 @@ def _replay_codex_normalization(conn: sqlite3.Connection) -> None:
1764
2009
  the plain rowid alias make a re-run byte-idempotent."""
1765
2010
  kern = _lib_codex_conversation
1766
2011
  events_by_file: dict[str, list] = {}
2012
+ accounts_by_file: dict[str, dict[tuple[str, int], str | None]] = {}
1767
2013
  order: list[str] = []
2014
+ has_account = "account_key" in {
2015
+ str(info[1])
2016
+ for info in conn.execute("PRAGMA table_info(codex_conversation_events)")
2017
+ }
2018
+ account_select = ", account_key" if has_account else ""
1768
2019
  for row in conn.execute(
1769
2020
  "SELECT source_path, line_offset, source_root_key, conversation_key, "
1770
2021
  "native_thread_id, root_thread_id, parent_thread_id, timestamp_utc, "
1771
2022
  "record_type, event_type, turn_id, call_id, payload_json "
2023
+ + account_select + " "
1772
2024
  "FROM codex_conversation_events "
1773
2025
  "ORDER BY source_path ASC, line_offset ASC"
1774
2026
  ):
1775
- event = _lib_jsonl.CodexPhysicalEvent(*row)
2027
+ event = _lib_jsonl.CodexPhysicalEvent(*(row[:-1] if has_account else row))
1776
2028
  if event.source_path not in events_by_file:
1777
2029
  events_by_file[event.source_path] = []
2030
+ accounts_by_file[event.source_path] = {}
1778
2031
  order.append(event.source_path)
1779
2032
  events_by_file[event.source_path].append(event)
2033
+ if has_account:
2034
+ accounts_by_file[event.source_path][
2035
+ (event.source_path, int(event.line_offset))
2036
+ ] = row[-1]
1780
2037
  affected: set = set()
1781
2038
  for source_path in order:
1782
2039
  result = kern.normalize_codex_events(
1783
2040
  events_by_file[source_path], initial=kern.CodexStickyState())
1784
- _insert_codex_normalized_rows(conn, result.rows, result.touches)
2041
+ _insert_codex_normalized_rows(
2042
+ conn, result.rows, result.touches,
2043
+ accounts_by_file[source_path] if has_account else None,
2044
+ )
1785
2045
  affected.update(r.conversation_key for r in result.rows)
1786
2046
  _recompute_codex_rollups(conn, affected)
2047
+ # Migration 025 still replays this helper against legacy cache.db fixtures,
2048
+ # where the conversations-only #482 projection table does not exist. Live
2049
+ # conversations.db replays must refresh the projection, but the historical
2050
+ # cache migration must remain byte-stable and cannot create a newer store's
2051
+ # derivation out of order.
2052
+ projection_exists = conn.execute(
2053
+ "SELECT 1 FROM sqlite_master "
2054
+ "WHERE type='table' AND name='codex_find_projection'"
2055
+ ).fetchone() is not None
2056
+ if projection_exists:
2057
+ import _lib_codex_conversation_query as query
2058
+ query.materialize_codex_find_projection(conn, affected)
2059
+
2060
+
2061
+ def run_codex_find_projection_backfill(
2062
+ conn: sqlite3.Connection,
2063
+ *,
2064
+ batch_size: int = 400,
2065
+ ) -> dict[str, object]:
2066
+ """Consume one resumable #482 projection batch from retained rows.
2067
+
2068
+ The caller owns the Codex conversations provider flock. Progress is keyed
2069
+ by the normalized message rowid, but a selected conversation is always
2070
+ rebuilt as a whole so native folds never straddle a batch boundary.
2071
+ """
2072
+ pending = conn.execute(
2073
+ "SELECT 1 FROM cache_meta "
2074
+ "WHERE key='codex_find_projection_backfill_pending'"
2075
+ ).fetchone()
2076
+ if pending is None:
2077
+ complete = conn.execute(
2078
+ "SELECT 1 FROM cache_meta "
2079
+ "WHERE key='codex_find_projection_complete_version' AND value='1'"
2080
+ ).fetchone() is not None
2081
+ return {"processed": 0, "complete": complete}
2082
+ row = conn.execute(
2083
+ "SELECT value FROM cache_meta "
2084
+ "WHERE key='codex_find_projection_backfill_cursor'"
2085
+ ).fetchone()
2086
+ try:
2087
+ cursor = int(row[0]) if row is not None else 0
2088
+ except (TypeError, ValueError):
2089
+ cursor = 0
2090
+ selected_rows = conn.execute(
2091
+ "SELECT id,conversation_key FROM codex_conversation_messages "
2092
+ "WHERE id>? ORDER BY id LIMIT ?",
2093
+ (cursor, max(1, batch_size)),
2094
+ ).fetchall()
2095
+ selected = {conversation_key for _message_id, conversation_key in selected_rows}
2096
+ if selected_rows:
2097
+ import _lib_codex_conversation_query as query
2098
+ query.materialize_codex_find_projection(conn, selected)
2099
+ # Advance only over raw rows actually consumed by this batch. A
2100
+ # conversation may own later, interleaved row ids; jumping to its MAX
2101
+ # would skip another conversation whose first row lies in between and
2102
+ # could falsely certify the projection complete.
2103
+ cursor = int(selected_rows[-1][0])
2104
+ conn.execute(
2105
+ "INSERT OR REPLACE INTO cache_meta(key,value) VALUES"
2106
+ "('codex_find_projection_backfill_cursor',?)",
2107
+ (str(cursor),),
2108
+ )
2109
+ remaining = conn.execute(
2110
+ "SELECT 1 FROM codex_conversation_messages WHERE id>? LIMIT 1",
2111
+ (cursor,),
2112
+ ).fetchone()
2113
+ complete = remaining is None
2114
+ if complete:
2115
+ conn.execute(
2116
+ "INSERT OR REPLACE INTO cache_meta(key,value) VALUES"
2117
+ "('codex_find_projection_complete_version','1')"
2118
+ )
2119
+ conn.execute(
2120
+ "DELETE FROM cache_meta WHERE key IN "
2121
+ "('codex_find_projection_backfill_pending',"
2122
+ " 'codex_find_projection_backfill_cursor')"
2123
+ )
2124
+ conn.commit()
2125
+ return {"processed": len(selected), "complete": complete}
1787
2126
 
1788
2127
 
1789
2128
  def _repair_codex_turn_ids_for_source(
@@ -1832,27 +2171,18 @@ def _repair_codex_turn_ids_for_source(
1832
2171
  return affected
1833
2172
 
1834
2173
 
1835
- def _collect_inactive_codex_paths_and_roots(
2174
+ def _collect_retained_codex_paths_and_roots(
1836
2175
  conn: sqlite3.Connection,
1837
- current_file_identities: set[tuple[str, str]],
1838
- active_root_keys: set[str],
1839
2176
  ) -> tuple[list[tuple[str, str | None]], set[str]]:
1840
- """Return stale real source identities and their candidate root keys.
2177
+ """Return every retained real source identity and provider root.
1841
2178
 
1842
2179
  A failed/partial prior write can leave any S1 child family without its
1843
- terminal ``codex_session_files`` row. Scope pruning must therefore use
1844
- every physical family, compare each row's path AND provider root, and leave
1845
- relative fixture rows alone.
2180
+ terminal ``codex_session_files`` row. Safety decisions must therefore see
2181
+ every physical family, including a still-discovered path whose root no
2182
+ longer contains recognizable Codex data. Relative fixture rows have no
2183
+ on-disk authority and remain outside filesystem pruning.
1846
2184
  """
1847
- stale_identities: set[tuple[str, str | None]] = set()
1848
- stale_root_keys: set[str] = set()
1849
- current_paths = {path for path, _root_key in current_file_identities}
1850
- terminal_file_identities = {
1851
- (path, root_key)
1852
- for path, root_key in conn.execute(
1853
- "SELECT path, source_root_key FROM codex_session_files"
1854
- )
1855
- }
2185
+ retained_identities: set[tuple[str, str | None]] = set()
1856
2186
  family_queries = (
1857
2187
  "SELECT path, source_root_key FROM codex_session_files",
1858
2188
  "SELECT source_path, source_root_key FROM codex_session_entries",
@@ -1862,29 +2192,59 @@ def _collect_inactive_codex_paths_and_roots(
1862
2192
  )
1863
2193
  for query in family_queries:
1864
2194
  for source_path, root_key in conn.execute(query):
1865
- identity = (source_path, root_key)
1866
- if (
1867
- not os.path.isabs(source_path)
1868
- or identity in current_file_identities
1869
- # An old terminal file at a currently discovered path must
1870
- # reach the normal requalification loop, which resets every
1871
- # family as one file transaction and records the reset stat.
1872
- or (
1873
- source_path in current_paths
1874
- and identity in terminal_file_identities
1875
- )
1876
- ):
2195
+ if not os.path.isabs(source_path):
1877
2196
  continue
1878
- stale_identities.add(identity)
1879
- if root_key is not None:
1880
- stale_root_keys.add(root_key)
1881
- stale_root_keys.update(
2197
+ retained_identities.add((source_path, root_key))
2198
+ retained_root_keys = {
2199
+ root_key for _path, root_key in retained_identities if root_key is not None
2200
+ }
2201
+ retained_root_keys.update(
1882
2202
  root_key
1883
2203
  for (root_key,) in conn.execute(
1884
2204
  "SELECT source_root_key FROM codex_source_roots"
1885
2205
  )
1886
- if root_key not in active_root_keys
1887
2206
  )
2207
+ return (
2208
+ sorted(retained_identities, key=lambda item: (item[0], item[1] or "")),
2209
+ retained_root_keys,
2210
+ )
2211
+
2212
+
2213
+ def _collect_inactive_codex_paths_and_roots(
2214
+ conn: sqlite3.Connection,
2215
+ current_file_identities: set[tuple[str, str]],
2216
+ active_root_keys: set[str],
2217
+ ) -> tuple[list[tuple[str, str | None]], set[str]]:
2218
+ """Return stale retained source identities and candidate root keys."""
2219
+ retained_identities, retained_root_keys = (
2220
+ _collect_retained_codex_paths_and_roots(conn)
2221
+ )
2222
+ current_paths = {path for path, _root_key in current_file_identities}
2223
+ terminal_file_identities = {
2224
+ (path, root_key)
2225
+ for path, root_key in conn.execute(
2226
+ "SELECT path, source_root_key FROM codex_session_files"
2227
+ )
2228
+ }
2229
+ stale_identities = {
2230
+ identity
2231
+ for identity in retained_identities
2232
+ if (
2233
+ identity not in current_file_identities
2234
+ # An old terminal file at a currently discovered path must reach
2235
+ # the normal requalification loop. The caller separately protects
2236
+ # that loop when the configured root is unrecognizable.
2237
+ and not (
2238
+ identity[0] in current_paths
2239
+ and identity in terminal_file_identities
2240
+ )
2241
+ )
2242
+ }
2243
+ stale_root_keys = {
2244
+ root_key
2245
+ for root_key in retained_root_keys
2246
+ if root_key not in active_root_keys
2247
+ }
1888
2248
  return sorted(stale_identities, key=lambda item: (item[0], item[1] or "")), stale_root_keys
1889
2249
 
1890
2250
 
@@ -3793,7 +4153,9 @@ def _reingest_parse_file(jp, path_str):
3793
4153
  return rows
3794
4154
 
3795
4155
 
3796
- def _resumable_reingest_conversation_messages(conn):
4156
+ def _resumable_reingest_conversation_messages(
4157
+ conn, active_account_key: "str | None" = None,
4158
+ ):
3797
4159
  """#179: resumable, lock-friendly replacement for the old global
3798
4160
  ``clear_conversation_messages`` + offset-0 ``backfill_conversation_messages``
3799
4161
  reingest, which re-armed the whole ~2.5min rebuild on any interrupt. Walks
@@ -3841,6 +4203,30 @@ def _resumable_reingest_conversation_messages(conn):
3841
4203
  conn.commit()
3842
4204
  continue
3843
4205
  try:
4206
+ # #347 enrichment replays must not erase first-stamped transcript
4207
+ # ownership. Snapshot by physical offset before the destructive
4208
+ # per-source replace and carry it onto the re-parsed row.
4209
+ account_capable = "account_key" in {
4210
+ str(info[1])
4211
+ for info in conn.execute("PRAGMA table_info(conversation_messages)")
4212
+ }
4213
+ account_by_offset = {
4214
+ int(offset): account_key
4215
+ for offset, account_key in conn.execute(
4216
+ "SELECT byte_offset,account_key FROM conversation_messages "
4217
+ "WHERE source_path=?",
4218
+ (path_str,),
4219
+ )
4220
+ } if account_capable else {}
4221
+ if account_capable:
4222
+ rows = [
4223
+ row + (
4224
+ account_by_offset[int(row[4])]
4225
+ if int(row[4]) in account_by_offset
4226
+ else active_account_key,
4227
+ )
4228
+ for row in rows
4229
+ ]
3844
4230
  # #217 S2 / I-3 (P1-4): conversation_file_touches is derived state
3845
4231
  # keyed by conversation_messages.id, and this per-source reingest
3846
4232
  # DELETEs + re-inserts the file's message rows (bumping autoincrement
@@ -3855,7 +4241,11 @@ def _resumable_reingest_conversation_messages(conn):
3855
4241
  conn.execute("DELETE FROM conversation_messages WHERE source_path=?",
3856
4242
  (path_str,))
3857
4243
  if rows:
3858
- conn.executemany(_CONV_INSERT_SQL, rows)
4244
+ conn.executemany(
4245
+ _CONV_ACCOUNT_INSERT_SQL if account_capable
4246
+ else _CONV_INSERT_SQL,
4247
+ rows,
4248
+ )
3859
4249
  # Refill scoped to this file's just-reinserted physical keys
3860
4250
  # (col 3=source_path, col 4=byte_offset per _conv_row_tuple).
3861
4251
  _fill_file_touches(
@@ -4860,6 +5250,12 @@ class CodexIngestStats:
4860
5250
  # Count of cached files dropped because they fall outside the CURRENT
4861
5251
  # $CODEX_HOME root set (issue #108 — a prior-root purge, not a delta).
4862
5252
  files_pruned: int = 0
5253
+ # #485: a whole-tree walk may discover no trustworthy current Codex root.
5254
+ # In that state absence is not deletion evidence, so ordinary orphan
5255
+ # pruning is refused and retained rows stay untouched. These fields make
5256
+ # the destructive decision directly observable to callers and tests.
5257
+ prune_refused: bool = False
5258
+ prune_refused_files: int = 0
4863
5259
  lock_contended: bool = False
4864
5260
  # #279 S2 F1 parse-health counters — folded from each file's
4865
5261
  # _CodexIterState after its drain. Same vocabulary as the iterator
@@ -5456,6 +5852,7 @@ def sync_codex_cache(
5456
5852
  _cache_root_keys,
5457
5853
  )
5458
5854
  seq_before = codex_physical_mutation_seq(conn)
5855
+ rebuild_prune_scope: _CodexPruneScope | None = None
5459
5856
 
5460
5857
  # #416 spec D1: which rollouts were ALREADY ingested before this rebuild
5461
5858
  # cleared the cursor table. A rebuild re-reads their bytes, and bytes
@@ -5485,6 +5882,46 @@ def sync_codex_cache(
5485
5882
  str(_canonical_codex_path(pathlib.Path(str(_path))))))
5486
5883
  except (ValueError, TypeError, OSError):
5487
5884
  continue
5885
+ if rebuild and not targeted:
5886
+ # #485: rebuild used to clear first and discover second. With an
5887
+ # empty/missing/misconfigured root that made the explicit repair
5888
+ # command itself destroy the only retained copy. Discover before
5889
+ # the clear and require the same positive root evidence as ordinary
5890
+ # pruning whenever real absolute Codex rows already exist.
5891
+ preflight_files = _discover_codex_files_with_roots()
5892
+ preflight_scope = _codex_prune_scope(preflight_files)
5893
+ rebuild_prune_scope = preflight_scope
5894
+ preflight_sources, _preflight_root_keys = (
5895
+ _collect_retained_codex_paths_and_roots(conn)
5896
+ )
5897
+ preflight_source_root_keys = {
5898
+ root_key
5899
+ for _path, root_key in preflight_sources
5900
+ if root_key is not None
5901
+ }
5902
+ (
5903
+ _safe_sources,
5904
+ refused_sources,
5905
+ _safe_roots,
5906
+ refused_root_keys,
5907
+ ) = _partition_codex_prune_candidates(
5908
+ preflight_scope, preflight_sources, preflight_source_root_keys
5909
+ )
5910
+ if refused_sources or refused_root_keys:
5911
+ _codex_prune_refusal_record(
5912
+ conn,
5913
+ store="cache",
5914
+ scope=preflight_scope,
5915
+ refused_sources=refused_sources,
5916
+ refused_root_keys=refused_root_keys,
5917
+ )
5918
+ conn.commit()
5919
+ stats.files_total = len(preflight_files)
5920
+ stats.prune_refused = True
5921
+ stats.prune_refused_files = len({
5922
+ path for path, _root in refused_sources
5923
+ })
5924
+ return stats
5488
5925
  if rebuild:
5489
5926
  # Clear INSIDE the lock — see sync_cache() for the full
5490
5927
  # rationale. Done before the existing SELECT so delta
@@ -5590,19 +6027,13 @@ def sync_codex_cache(
5590
6027
  stats.files_total = len(files)
5591
6028
  _p_disc.set_count(len(files))
5592
6029
 
5593
- # Scope the cache to the CURRENT root set: drop rows ingested under a
5594
- # prior $CODEX_HOME (issue #108). iter_codex_entries() has NO root
5595
- # predicate it reads every row in range so without this, reusing
5596
- # the same cache.db across `CODEX_HOME=/A` then `CODEX_HOME=/B` runs
5597
- # returns A+B instead of just B. Prune every real (absolute) row
5598
- # outside the current set, even when that set is empty (an empty
5599
- # current root then prunes the cache to empty): the cache is fully
5600
- # re-derivable, so honoring the override beats retaining unreachable
5601
- # rows. Done INSIDE the lock and committed BEFORE the existing-SELECT
5602
- # + parse loop so no cache.db write lock is held across the read-heavy
5603
- # ingest (same invariant as the --rebuild clear above). Concurrent
5604
- # processes with different $CODEX_HOME would prune each other; the
5605
- # flock serializes them and that is a pathological configuration.
6030
+ # Scope the cache to the CURRENT root set (issue #108), but only from
6031
+ # positive filesystem evidence (#485). A recognized new root still
6032
+ # authorizes the deliberate A -> B scope switch. No recognized root,
6033
+ # or one configured-but-unrecognizable sibling, is unknown filesystem
6034
+ # state rather than proof of deletion and therefore fails closed.
6035
+ protected_paths: set[str] = set()
6036
+ ordinary_prune_scope: _CodexPruneScope | None = None
5606
6037
  if not rebuild and not targeted: # --rebuild already cleared; targeted bypasses
5607
6038
  current_file_identities = {
5608
6039
  (str(item.source_path), item.source_root_key) for item in files
@@ -5619,12 +6050,39 @@ def sync_codex_cache(
5619
6050
  orphan_sources, orphan_root_keys = _collect_inactive_codex_paths_and_roots(
5620
6051
  conn, current_file_identities, active_root_keys,
5621
6052
  )
5622
- if orphan_sources or orphan_root_keys:
6053
+ prune_scope = _codex_prune_scope(files)
6054
+ ordinary_prune_scope = prune_scope
6055
+ (
6056
+ safe_sources,
6057
+ _refused_orphan_sources,
6058
+ safe_root_keys,
6059
+ _refused_orphan_root_keys,
6060
+ ) = _partition_codex_prune_candidates(
6061
+ prune_scope, orphan_sources, orphan_root_keys
6062
+ )
6063
+ retained_sources, _retained_root_keys = (
6064
+ _collect_retained_codex_paths_and_roots(conn)
6065
+ )
6066
+ retained_source_root_keys = {
6067
+ root_key
6068
+ for _path, root_key in retained_sources
6069
+ if root_key is not None
6070
+ }
6071
+ (
6072
+ _safe_retained_sources,
6073
+ refused_sources,
6074
+ _safe_retained_roots,
6075
+ refused_root_keys,
6076
+ ) = _partition_codex_prune_candidates(
6077
+ prune_scope, retained_sources, retained_source_root_keys
6078
+ )
6079
+ protected_paths = {path for path, _root in refused_sources}
6080
+ if safe_sources or safe_root_keys:
5623
6081
  before_prune = conn.total_changes
5624
6082
  # #294 S6: capture the conversation keys the orphan rows belong to
5625
6083
  # BEFORE deleting, so the rollups can be repaired/deleted after.
5626
6084
  orphan_keys: set = set()
5627
- for orphan_path, orphan_root_key in orphan_sources:
6085
+ for orphan_path, orphan_root_key in safe_sources:
5628
6086
  orphan_keys.update(
5629
6087
  row[0] for row in conn.execute(
5630
6088
  "SELECT DISTINCT conversation_key FROM codex_conversation_messages "
@@ -5637,7 +6095,7 @@ def sync_codex_cache(
5637
6095
  )
5638
6096
  _prune_inactive_codex_source_roots(
5639
6097
  conn, active_root_keys,
5640
- candidate_root_keys=orphan_root_keys,
6098
+ candidate_root_keys=safe_root_keys,
5641
6099
  )
5642
6100
  # Recompute-affected-or-delete the rollups the prune touched (§3.2):
5643
6101
  # a conversation with no surviving rows loses its rollup, one that
@@ -5645,8 +6103,24 @@ def sync_codex_cache(
5645
6103
  _recompute_codex_rollups(conn, orphan_keys)
5646
6104
  if conn.total_changes != before_prune:
5647
6105
  _bump_codex_physical_mutation_seq(conn)
6106
+ stats.files_pruned = len({path for path, _root in safe_sources})
6107
+ if refused_sources or refused_root_keys:
6108
+ _codex_prune_refusal_record(
6109
+ conn,
6110
+ store="cache",
6111
+ scope=prune_scope,
6112
+ refused_sources=refused_sources,
6113
+ refused_root_keys=refused_root_keys,
6114
+ )
6115
+ stats.prune_refused = True
6116
+ stats.prune_refused_files = len({
6117
+ path for path, _root in refused_sources
6118
+ })
6119
+ if (
6120
+ safe_sources or safe_root_keys
6121
+ or refused_sources or refused_root_keys
6122
+ ):
5648
6123
  conn.commit()
5649
- stats.files_pruned = len({path for path, _root in orphan_sources})
5650
6124
 
5651
6125
  # This SELECT does NOT open an implicit transaction (Python's
5652
6126
  # sqlite3 module only BEGINs on DML). Do NOT add any INSERT/
@@ -5808,6 +6282,12 @@ def sync_codex_cache(
5808
6282
  break
5809
6283
  jp = discovered.source_path
5810
6284
  path_str = str(jp)
6285
+ if path_str in protected_paths:
6286
+ # The file is still visible, but its configured root did not
6287
+ # yield recognizable Codex data. Treat that as unknown
6288
+ # filesystem state: neither truncation/requalification reset
6289
+ # nor append ingestion may replace the retained generation.
6290
+ continue
5811
6291
  try:
5812
6292
  st = jp.stat()
5813
6293
  except OSError as exc:
@@ -6543,6 +7023,17 @@ def sync_codex_cache(
6543
7023
  and stats.files_failed == 0
6544
7024
  and stats.files_deferred_torn == 0
6545
7025
  ):
7026
+ recovery_scope = rebuild_prune_scope or ordinary_prune_scope
7027
+ if (
7028
+ recovery_scope is not None
7029
+ and recovery_scope.recognized_root_keys
7030
+ and not stats.prune_refused
7031
+ and (
7032
+ stats.files_processed + stats.files_skipped_unchanged
7033
+ == stats.files_total
7034
+ )
7035
+ ):
7036
+ _clear_codex_prune_refusal(conn)
6546
7037
  if replay_pending:
6547
7038
  conn.execute("DELETE FROM cache_meta WHERE key = ?",
6548
7039
  (CODEX_REPLAY_FROM_ZERO_KEY,))
@@ -8095,6 +8586,126 @@ def open_conversations_db(*, attach_cache: bool = True) -> sqlite3.Connection:
8095
8586
  return _conversations_open_guarded(attach_cache=attach_cache)
8096
8587
 
8097
8588
 
8589
+ def scope_conversations_db_to_account(
8590
+ conn: sqlite3.Connection, account_key: str,
8591
+ ) -> None:
8592
+ """Narrow one conversation read connection to ``account_key`` (#347).
8593
+
8594
+ The query kernels intentionally retain their byte-frozen unqualified SQL.
8595
+ A qualified request gets a fresh SQLite connection and this helper shadows
8596
+ every transcript/accounting leaf with TEMP views on that connection only.
8597
+ SQLite resolves TEMP before main/attached schemas, so browse, facets,
8598
+ search, detail, outline, export, payload, media, find, and cost/token joins
8599
+ all inherit the same physical-row predicate without a parallel query stack.
8600
+
8601
+ The two rollups are rebuilt into TEMP tables from the filtered leaves. No
8602
+ persistent row is changed and an unqualified connection takes none of this
8603
+ path, preserving the existing SQL plans and serialized bytes.
8604
+ """
8605
+ import _lib_accounts
8606
+
8607
+ key = str(account_key or "").strip()
8608
+ if not key:
8609
+ raise ValueError("account_key is required")
8610
+ if key == _lib_accounts.UNATTRIBUTED:
8611
+ stored_predicate = "COALESCE(account_key,'unattributed')='unattributed'"
8612
+ else:
8613
+ # The value lives in a one-row TEMP table and reaches every view through
8614
+ # a subquery; never interpolate an opaque account key into DDL.
8615
+ stored_predicate = (
8616
+ "COALESCE(account_key,'unattributed')="
8617
+ "(SELECT account_key FROM _conversation_account_scope)"
8618
+ )
8619
+
8620
+ conn.execute(
8621
+ "CREATE TEMP TABLE _conversation_account_scope "
8622
+ "(account_key TEXT NOT NULL PRIMARY KEY)"
8623
+ )
8624
+ conn.execute(
8625
+ "INSERT INTO _conversation_account_scope(account_key) VALUES(?)", (key,)
8626
+ )
8627
+
8628
+ conn.executescript(
8629
+ f"""
8630
+ CREATE TEMP VIEW conversation_messages AS
8631
+ SELECT * FROM main.conversation_messages WHERE {stored_predicate};
8632
+ CREATE TEMP VIEW conversation_ai_titles AS
8633
+ SELECT t.* FROM main.conversation_ai_titles t
8634
+ WHERE EXISTS (
8635
+ SELECT 1 FROM conversation_messages m
8636
+ WHERE m.source_path=t.source_path
8637
+ AND m.byte_offset=t.byte_offset
8638
+ );
8639
+ CREATE TEMP VIEW conversation_file_touches AS
8640
+ SELECT t.* FROM main.conversation_file_touches t
8641
+ WHERE EXISTS (
8642
+ SELECT 1 FROM conversation_messages m WHERE m.id=t.message_id
8643
+ );
8644
+ CREATE TEMP VIEW session_entries AS
8645
+ SELECT * FROM cache_db.session_entries WHERE {stored_predicate};
8646
+ -- A file-level project path cannot be partitioned when one JSONL
8647
+ -- switches accounts. Scoped anonymization therefore uses only the
8648
+ -- already-filtered physical message CWDs.
8649
+ CREATE TEMP VIEW session_files AS
8650
+ SELECT * FROM cache_db.session_files WHERE 0;
8651
+
8652
+ CREATE TEMP VIEW codex_conversation_events AS
8653
+ SELECT * FROM main.codex_conversation_events WHERE {stored_predicate};
8654
+ CREATE TEMP VIEW codex_conversation_messages AS
8655
+ SELECT * FROM main.codex_conversation_messages WHERE {stored_predicate};
8656
+ CREATE TEMP VIEW codex_conversation_file_touches AS
8657
+ SELECT t.* FROM main.codex_conversation_file_touches t
8658
+ WHERE EXISTS (
8659
+ SELECT 1 FROM codex_conversation_messages m WHERE m.id=t.message_id
8660
+ );
8661
+ CREATE TEMP VIEW codex_find_projection AS
8662
+ SELECT p.* FROM main.codex_find_projection p
8663
+ WHERE EXISTS (
8664
+ SELECT 1 FROM codex_conversation_messages m WHERE m.id=p.message_id
8665
+ );
8666
+ CREATE TEMP VIEW codex_session_entries AS
8667
+ SELECT * FROM cache_db.codex_session_entries WHERE {stored_predicate};
8668
+
8669
+ -- Thread metadata has conversation-level provenance and can predate an
8670
+ -- in-file account switch. Do not expose it in a scoped connection;
8671
+ -- the Codex rollup degrades explicitly to unassigned rather than leaking
8672
+ -- another account's cwd/git metadata.
8673
+ CREATE TEMP VIEW codex_conversation_threads AS
8674
+ SELECT * FROM cache_db.codex_conversation_threads WHERE 0;
8675
+ CREATE TEMP VIEW codex_source_roots AS
8676
+ SELECT * FROM cache_db.codex_source_roots WHERE 0;
8677
+
8678
+ CREATE TEMP TABLE conversation_sessions AS
8679
+ SELECT * FROM main.conversation_sessions WHERE 0;
8680
+ CREATE TEMP TABLE codex_conversation_rollups (
8681
+ conversation_key TEXT NOT NULL PRIMARY KEY,
8682
+ source_root_key TEXT NOT NULL,
8683
+ parent_thread_id TEXT,
8684
+ item_count INTEGER NOT NULL DEFAULT 0,
8685
+ started_utc TEXT,
8686
+ last_activity_utc TEXT,
8687
+ project_key TEXT,
8688
+ project_label TEXT,
8689
+ models_json TEXT,
8690
+ title TEXT
8691
+ );
8692
+ """
8693
+ )
8694
+ _recompute_conversation_sessions(conn)
8695
+ codex_keys = {
8696
+ row[0]
8697
+ for row in conn.execute(
8698
+ "SELECT DISTINCT conversation_key FROM codex_conversation_messages"
8699
+ )
8700
+ if row[0]
8701
+ }
8702
+ _recompute_codex_rollups(conn, codex_keys)
8703
+ # TEMP rollup writes open a transaction. Close it before a long-lived
8704
+ # account-scoped SSE reader starts watching so later provider-writer commits
8705
+ # are visible to the dynamic leaf views and never contend on this setup work.
8706
+ conn.commit()
8707
+
8708
+
8098
8709
  def _open_conversations_db_for_recovery(
8099
8710
  *, attach_cache: bool = True,
8100
8711
  ) -> sqlite3.Connection:
@@ -8669,6 +9280,7 @@ def _prepare_claude_conversation_maintenance(
8669
9280
  *,
8670
9281
  rebuild: bool,
8671
9282
  targeted: bool,
9283
+ active_account_key: "str | None" = None,
8672
9284
  ) -> bool:
8673
9285
  """Consume transcript-only upgrade work under the conversation flock.
8674
9286
 
@@ -8760,7 +9372,9 @@ def _prepare_claude_conversation_maintenance(
8760
9372
  "'conversation_background_mcp_reingest_pending')"
8761
9373
  ).fetchone() is not None
8762
9374
  if reingest:
8763
- _resumable_reingest_conversation_messages(conn)
9375
+ _resumable_reingest_conversation_messages(
9376
+ conn, active_account_key=active_account_key,
9377
+ )
8764
9378
  _set_cache_meta(conn, "conversation_sessions_backfill_pending", "1")
8765
9379
  conn.commit()
8766
9380
 
@@ -8806,6 +9420,20 @@ def sync_claude_conversations(
8806
9420
  stats.lock_contended = True
8807
9421
  return stats
8808
9422
 
9423
+ # #347 observe-and-stamp mirrors the accounting ingest boundary. A
9424
+ # torn credential read is undecided, not unattributed: defer before any
9425
+ # maintenance/rebuild mutation so the cursor remains replayable.
9426
+ import _lib_accounts
9427
+ claude_identity = _cctally_core._resolve_active_claude_identity()
9428
+ if claude_identity.get("status") == "torn":
9429
+ stats.files_deferred_torn += 1
9430
+ stats.deferred_reason = "identity_torn"
9431
+ return stats
9432
+ active_key = claude_identity["account_key"]
9433
+ active_account_key = (
9434
+ None if active_key == _lib_accounts.UNATTRIBUTED else active_key
9435
+ )
9436
+
8809
9437
  targeted = only_paths is not None
8810
9438
  pending_rebuild = conn.execute(
8811
9439
  "SELECT 1 FROM cache_meta "
@@ -8835,10 +9463,19 @@ def sync_claude_conversations(
8835
9463
 
8836
9464
  _report_conversation_progress(progress, "prepare", stats)
8837
9465
  maintenance_replayed = _prepare_claude_conversation_maintenance(
8838
- conn, rebuild=rebuild, targeted=targeted
9466
+ conn, rebuild=rebuild, targeted=targeted,
9467
+ active_account_key=active_account_key,
8839
9468
  )
8840
9469
 
9470
+ rebuild_account_stamps: dict[tuple[str, int], str | None] = {}
8841
9471
  if rebuild:
9472
+ rebuild_account_stamps = {
9473
+ (str(path), int(offset)): account_key
9474
+ for path, offset, account_key in conn.execute(
9475
+ "SELECT source_path,byte_offset,account_key "
9476
+ "FROM conversation_messages"
9477
+ )
9478
+ }
8842
9479
  clear_conversation_messages(conn)
8843
9480
  conn.execute("DELETE FROM conversation_ai_titles")
8844
9481
  conn.execute("DELETE FROM conversation_sessions")
@@ -8915,7 +9552,16 @@ def sync_claude_conversations(
8915
9552
  include_cost=False,
8916
9553
  ):
8917
9554
  if mrow is not None:
8918
- conv_rows.append(_conv_row_tuple(mrow, path_str))
9555
+ account_key = rebuild_account_stamps.get(
9556
+ (path_str, int(mrow.byte_offset)),
9557
+ active_account_key,
9558
+ )
9559
+ conv_rows.append(
9560
+ _conv_row_tuple(
9561
+ mrow, path_str, account_key,
9562
+ include_account=True,
9563
+ )
9564
+ )
8919
9565
  if ai is not None:
8920
9566
  ai_rows.append(
8921
9567
  (ai.session_id, ai.ai_title, path_str, ai.byte_offset)
@@ -8952,7 +9598,7 @@ def sync_claude_conversations(
8952
9598
  )
8953
9599
  stats.files_reset_truncated += 1
8954
9600
  if conv_rows:
8955
- conn.executemany(_CONV_INSERT_SQL, conv_rows)
9601
+ conn.executemany(_CONV_ACCOUNT_INSERT_SQL, conv_rows)
8956
9602
  _fill_file_touches(
8957
9603
  conn, scope=[(row[3], row[4]) for row in conv_rows]
8958
9604
  )
@@ -9059,6 +9705,7 @@ def sync_codex_conversations(
9059
9705
  """Delta-sync Codex events/search rows into conversations.db (#320)."""
9060
9706
  stats = CodexIngestStats()
9061
9707
  did_from_zero_replay = False
9708
+ rebuild_account_stamps: dict[tuple[str, int], str | None] = {}
9062
9709
  _cctally_core.APP_DIR.mkdir(parents=True, exist_ok=True)
9063
9710
  _cctally_core.CONVERSATIONS_LOCK_CODEX_PATH.touch()
9064
9711
  lock_fh = open(_cctally_core.CONVERSATIONS_LOCK_CODEX_PATH, "w")
@@ -9106,7 +9753,51 @@ def sync_codex_conversations(
9106
9753
  return stats
9107
9754
  rebuild = (
9108
9755
  rebuild or pending_rebuild or contract_rebuild or codex_replay_pending)
9756
+ rebuild_prune_scope: _CodexPruneScope | None = None
9109
9757
  if rebuild:
9758
+ rebuild_account_stamps = {
9759
+ (str(path), int(offset)): account_key
9760
+ for path, offset, account_key in conn.execute(
9761
+ "SELECT source_path,line_offset,account_key "
9762
+ "FROM codex_conversation_events"
9763
+ )
9764
+ }
9765
+ # #485: establish positive root evidence before the destructive
9766
+ # clear. A rebuild against an empty or missing CODEX_HOME must keep
9767
+ # the retained transcript store and its pending replay marker.
9768
+ preflight_files = _discover_codex_files_with_roots()
9769
+ preflight_scope = _codex_prune_scope(preflight_files)
9770
+ rebuild_prune_scope = preflight_scope
9771
+ preflight_sources = [
9772
+ (path, root_key)
9773
+ for path, root_key in conn.execute(
9774
+ "SELECT path,source_root_key "
9775
+ "FROM codex_conversation_source_files"
9776
+ )
9777
+ ]
9778
+ (
9779
+ _safe_sources,
9780
+ refused_sources,
9781
+ _safe_roots,
9782
+ _refused_roots,
9783
+ ) = _partition_codex_prune_candidates(
9784
+ preflight_scope, preflight_sources, set()
9785
+ )
9786
+ if refused_sources:
9787
+ _codex_prune_refusal_record(
9788
+ conn,
9789
+ store="conversations",
9790
+ scope=preflight_scope,
9791
+ refused_sources=refused_sources,
9792
+ refused_root_keys=set(),
9793
+ )
9794
+ conn.commit()
9795
+ stats.files_total = len(preflight_files)
9796
+ stats.prune_refused = True
9797
+ stats.prune_refused_files = len({
9798
+ path for path, _root in refused_sources
9799
+ })
9800
+ return stats
9110
9801
  conn.execute(
9111
9802
  "INSERT OR REPLACE INTO cache_meta(key,value) VALUES(?,?)",
9112
9803
  ("conversation_rebuild_codex_pending", "1"),
@@ -9152,9 +9843,36 @@ def sync_codex_conversations(
9152
9843
  if current_size < prev[0]:
9153
9844
  stats.deferred_reason = "truncation"
9154
9845
  return stats
9846
+ protected_paths: set[str] = set()
9847
+ ordinary_prune_scope: _CodexPruneScope | None = None
9155
9848
  if only_paths is None:
9156
9849
  active_paths = {str(item.source_path) for item in files}
9157
- for stale_path in sorted(set(existing) - active_paths):
9850
+ prune_scope = _codex_prune_scope(files)
9851
+ ordinary_prune_scope = prune_scope
9852
+ stale_sources = [
9853
+ (stale_path, existing[stale_path][3])
9854
+ for stale_path in sorted(set(existing) - active_paths)
9855
+ ]
9856
+ safe_sources, _refused_stale, _safe_roots, _refused_roots = (
9857
+ _partition_codex_prune_candidates(
9858
+ prune_scope, stale_sources, set()
9859
+ )
9860
+ )
9861
+ retained_sources = [
9862
+ (path, values[3])
9863
+ for path, values in existing.items()
9864
+ if os.path.isabs(path)
9865
+ ]
9866
+ (
9867
+ _safe_retained,
9868
+ refused_sources,
9869
+ _safe_retained_roots,
9870
+ _refused_retained_roots,
9871
+ ) = _partition_codex_prune_candidates(
9872
+ prune_scope, retained_sources, set()
9873
+ )
9874
+ protected_paths = {path for path, _root in refused_sources}
9875
+ for stale_path, _stale_root_key in safe_sources:
9158
9876
  affected = {
9159
9877
  row[0] for row in conn.execute(
9160
9878
  "SELECT DISTINCT conversation_key "
@@ -9179,11 +9897,30 @@ def sync_codex_conversations(
9179
9897
  (stale_path,),
9180
9898
  )
9181
9899
  _recompute_codex_rollups(conn, affected)
9900
+ import _lib_codex_conversation_query as query
9901
+ query.materialize_codex_find_projection(conn, affected)
9902
+ stats.files_pruned = len({path for path, _root in safe_sources})
9903
+ if refused_sources:
9904
+ _codex_prune_refusal_record(
9905
+ conn,
9906
+ store="conversations",
9907
+ scope=prune_scope,
9908
+ refused_sources=refused_sources,
9909
+ refused_root_keys=set(),
9910
+ )
9911
+ stats.prune_refused = True
9912
+ stats.prune_refused_files = len({
9913
+ path for path, _root in refused_sources
9914
+ })
9182
9915
  conn.commit()
9183
9916
 
9184
9917
  for discovered in files:
9185
9918
  jp = discovered.source_path
9186
9919
  path_str = str(jp)
9920
+ if path_str in protected_paths:
9921
+ # Visible but unrecognizable input is not evidence that the
9922
+ # retained transcript generation should be reset or replaced.
9923
+ continue
9187
9924
  try:
9188
9925
  st = jp.stat()
9189
9926
  except OSError:
@@ -9219,6 +9956,31 @@ def sync_codex_conversations(
9219
9956
  initial_conversation = prev[10] if prev else None
9220
9957
  initial_turn = prev[11] if prev else None
9221
9958
 
9959
+ # #347 uses the accounting cache's journal-backed file-range map as
9960
+ # the Codex transcript oracle. It is already attached read-only,
9961
+ # and core ingest runs first, so rebuilds replay the original
9962
+ # decision instead of consulting the currently-active auth.json.
9963
+ try:
9964
+ file_identity = codex_file_identity(discovered)
9965
+ incarnation_row = conn.execute(
9966
+ "SELECT incarnation FROM cache_db.codex_file_incarnations "
9967
+ "WHERE file_identity=?",
9968
+ (file_identity,),
9969
+ ).fetchone()
9970
+ incarnation = int(incarnation_row[0]) if incarnation_row else 1
9971
+ account_ranges = [
9972
+ (int(off), key)
9973
+ for off, key in conn.execute(
9974
+ "SELECT from_offset,account_key "
9975
+ "FROM cache_db.codex_file_accounts "
9976
+ "WHERE file_identity=? AND incarnation=? "
9977
+ "ORDER BY from_offset",
9978
+ (file_identity, incarnation),
9979
+ )
9980
+ ]
9981
+ except sqlite3.OperationalError:
9982
+ account_ranges = []
9983
+
9222
9984
  state = _CodexIterState(
9223
9985
  session_id=initial_session_id,
9224
9986
  model=initial_model,
@@ -9241,6 +10003,7 @@ def sync_codex_conversations(
9241
10003
  )
9242
10004
  events = []
9243
10005
  event_rows = []
10006
+ account_by_physical: dict[tuple[str, int], str | None] = {}
9244
10007
  yielded = 0
9245
10008
  try:
9246
10009
  with open(jp, "rb") as fh:
@@ -9256,6 +10019,16 @@ def sync_codex_conversations(
9256
10019
  ):
9257
10020
  event = emission.event
9258
10021
  events.append(event)
10022
+ covered, account_key = codex_account_for_offset(
10023
+ account_ranges, int(event.line_offset)
10024
+ )
10025
+ if not covered:
10026
+ account_key = rebuild_account_stamps.get(
10027
+ (event.source_path, int(event.line_offset))
10028
+ )
10029
+ account_by_physical[
10030
+ (event.source_path, int(event.line_offset))
10031
+ ] = account_key
9259
10032
  event_rows.append((
9260
10033
  event.source_path,
9261
10034
  event.line_offset,
@@ -9270,6 +10043,7 @@ def sync_codex_conversations(
9270
10043
  event.turn_id,
9271
10044
  event.call_id,
9272
10045
  event.payload_json,
10046
+ account_key,
9273
10047
  ))
9274
10048
  if emission.accounting is not None:
9275
10049
  yielded += 1
@@ -9327,12 +10101,13 @@ def sync_codex_conversations(
9327
10101
  "INSERT OR IGNORE INTO codex_conversation_events "
9328
10102
  "(source_path,line_offset,source_root_key,conversation_key,"
9329
10103
  "native_thread_id,root_thread_id,parent_thread_id,"
9330
- "timestamp_utc,record_type,event_type,turn_id,call_id,payload_json) "
9331
- "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)",
10104
+ "timestamp_utc,record_type,event_type,turn_id,call_id,payload_json,"
10105
+ "account_key) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
9332
10106
  event_rows,
9333
10107
  )
9334
10108
  _insert_codex_normalized_rows(
9335
- conn, normalized.rows, normalized.touches
10109
+ conn, normalized.rows, normalized.touches,
10110
+ account_by_physical,
9336
10111
  )
9337
10112
  affected_keys.update(
9338
10113
  row.conversation_key for row in normalized.rows
@@ -9345,6 +10120,8 @@ def sync_codex_conversations(
9345
10120
  _repair_codex_turn_ids_for_source(conn, path_str)
9346
10121
  )
9347
10122
  _recompute_codex_rollups(conn, affected_keys)
10123
+ import _lib_codex_conversation_query as query
10124
+ query.materialize_codex_find_projection(conn, affected_keys)
9348
10125
  terminal = state.thread
9349
10126
  conn.execute(
9350
10127
  "INSERT INTO codex_conversation_source_files "
@@ -9392,8 +10169,20 @@ def sync_codex_conversations(
9392
10169
  stats.files_failed += 1
9393
10170
  _report_conversation_progress(progress, "ingest", stats)
9394
10171
 
10172
+ run_codex_find_projection_backfill(conn)
9395
10173
  _report_conversation_progress(progress, "finalize", stats)
9396
10174
  if only_paths is None and stats.files_failed == 0:
10175
+ recovery_scope = rebuild_prune_scope or ordinary_prune_scope
10176
+ if (
10177
+ recovery_scope is not None
10178
+ and recovery_scope.recognized_root_keys
10179
+ and not stats.prune_refused
10180
+ and (
10181
+ stats.files_processed + stats.files_skipped_unchanged
10182
+ == stats.files_total
10183
+ )
10184
+ ):
10185
+ _clear_codex_prune_refusal(conn)
9397
10186
  conn.execute(
9398
10187
  "INSERT OR REPLACE INTO cache_meta(key,value) VALUES(?,?)",
9399
10188
  (